Merge pull request #2179 from DennisHeimbigner/vlenfix.dmh

Fix various problem around VLEN's

Author: WardF

CMakeLists.txt

@@ -1270,10 +1270,6 @@ IF(ENABLE_TESTS)
 
   OPTION(ENABLE_FAILING_TESTS "Run tests which are known to fail, check to see if any have been fixed." OFF)
 
-  # There is a special class of tests that currently fail because of memory leaks.
-  # As with failing tests, they can be enabled to work on them
-  OPTION(ENABLE_UNFIXED_MEMORY_LEAKS "Run tests with unfixed memory leaks" OFF)
-
   ###
   # Option to turn on unit testing. See
   # https://github.com/Unidata/netcdf-c/pull/1472 for more information.

RELEASE_NOTES.md

@@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.8.2 - TBD
 
+* [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179(https://github.com/Unidata/netcdf-c/pull/2179).
 * [Bug Fix] Make sure that netcdf.h accurately defines the flags in the open/create mode flags. See [Github #2183](https://github.com/Unidata/netcdf-c/pull/2183).
 * [Enhancement] Improve support for msys2+mingw platform. See [Github #2171](https://github.com/Unidata/netcdf-c/pull/2171).
 * [Bug Fix] Clean up the various inter-test dependencies in ncdump for CMake. See [Github #2168](https://github.com/Unidata/netcdf-c/pull/2168).

configure.ac

@@ -299,15 +299,6 @@ if test "x$enable_jna" = xyes ; then
 AC_DEFINE([JNA], [1], [if true, include jna bug workaround code])
 fi
 
-# Should tests with unfixed memory leaks be run?
-AC_MSG_CHECKING([if unfixed tests with memory leaks should be enabled])
-AC_ARG_ENABLE([unfixed-memory-leaks],
-  [AS_HELP_STRING([--enable-unfixed-memory-leaks],
-    [Enable tests in the ncdump directory that have unfixed memory leaks])])
-test "x$enable_unfixed_memory_leaks" = xyes || enable_unfixed_memory_leaks=no
-AC_MSG_RESULT($enable_unfixed_memory_leaks)
-AM_CONDITIONAL([ENABLE_UNFIXED_MEMORY_LEAKS], [test "x$enable_unfixed_memory_leaks" = xyes])
-
 # Does the user want to turn off unit tests (useful for test coverage
 # analysis).
 AC_MSG_CHECKING([if unit tests should be disabled])

docs/Doxyfile.in

@@ -782,6 +782,7 @@ INPUT = \
     @abs_top_srcdir@/libdispatch/derror.c \
     @abs_top_srcdir@/libdispatch/dv2i.c \
     @abs_top_srcdir@/libdispatch/dcopy.c \
+    @abs_top_srcdir@/libdispatch/daux.c \
     @abs_top_srcdir@/libsrc4/nc4var.c \
     @abs_top_srcdir@/libhdf5/nc4hdf.c \
     @abs_top_srcdir@/libsrc4/nc4internal.c \

include/nc4internal.h

@@ -170,8 +170,10 @@ typedef struct NC_ATT_INFO
     nc_type nc_typeid;      /**< NetCDF type of attribute's data. */
     void *format_att_info;  /**< Pointer to format-specific att info. */
     void *data;             /**< The attribute data. */
+#ifdef SEPDATA
     nc_vlen_t *vldata;      /**< VLEN data (only used for vlen types). */
     char **stdata;          /**< String data (only for string type). */
+#endif
 } NC_ATT_INFO_T;
 
 /** This is a struct to handle the var metadata. */
@@ -252,6 +254,7 @@ typedef struct NC_TYPE_INFO
         } e;                        /**< Enum */
         struct Fields {
             NClist* field;        /**< <! NClist<NC_FIELD_INFO_T*> */
+	    int varsized;         /**< <! 1 if this compound is variable sized; 0 if fixed size */
         } c;                      /**< Compound */
         struct {
             nc_type base_nc_typeid; /**< Typeid of the base type. */
@@ -281,7 +284,7 @@ typedef struct NC_GRP_INFO
 #define NC_INDEF  0x01  /**< in define mode, cleared by ncendef */
 
 /** This is the metadata we need to keep track of for each
-  * netcdf-4/HDF5 file. */
+  * netcdf-4/ file; used by libhdf5, libnczarr, and libdap4 */
 
 typedef struct NC_FILE_INFO
 {
@@ -296,7 +299,7 @@ typedef struct NC_FILE_INFO
     nc_bool_t parallel;   /**< True if file is open for parallel access */
     nc_bool_t redef;      /**< True if redefining an existing file */
     nc_bool_t no_attr_create_order; /**< True if the creation order tracking of attributes is disabled (netcdf-4 only) */
-    int fill_mode;        /**< Fill mode for vars - Unused internally currently */
+    int fill_mode;        /**< Fill mode for vars */
     nc_bool_t no_write;   /**< true if nc_open has mode NC_NOWRITE. */
     NC_GRP_INFO_T *root_grp; /**< Pointer to root group. */
     short next_nc_grpid;  /**< Next available group ID. */
@@ -419,14 +422,20 @@ extern int nc4_check_dup_name(NC_GRP_INFO_T *grp, char *norm_name);
 /* Get the fill value for a var. */
 extern int nc4_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp);
 
-/* Find default fill value. */
-extern int nc4_get_default_fill_value(nc_type typecode, void *fill_value);
+/* Find default fill value for atomic type. */
+extern int nc4_get_default_atomic_fill_value(nc_type, void *fill_value);
+
+/* Find default fill value for any type */
+extern int nc4_get_default_fill_value(NC_TYPE_INFO_T*, void *fill_value);
 
 /* Get an att given pointers to file, group, and perhaps ver info. */
 extern int nc4_get_att_ptrs(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var,
                      const char *name, nc_type *xtype, nc_type mem_type,
                      size_t *lenp, int *attnum, void *data);
 
+/* Get variable/fixed size flag for type */
+extern int NC4_inq_type_fixed_size(int ncid, nc_type xtype, int* isfixedsizep);
+
 /* Close the file. */
 extern int nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio);
 

include/nclog.h

@@ -10,7 +10,9 @@
 #include <stdarg.h>
 #include "ncexternl.h"
 
+#ifndef NCCATCH
 #undef NCCATCH
+#endif
 
 #define NCENVLOGGING "NCLOGGING"
 #define NCENVTRACING "NCTRACING"

include/ncoffsets.h

@@ -51,7 +51,10 @@ typedef struct NCtypealignset {
     NCalignment ncvlenalign;	/* nc_vlen_t*/
 } NCtypealignset;
 
-EXTERNL size_t NC_class_alignment(int ncclass);
+EXTERNL int NC_class_alignment(int ncclass, size_t*);
 EXTERNL void NC_compute_alignments(void);
 
+/* From libdispatch/dinstance.c */
+EXTERNL int NC_type_alignment(int ncid, nc_type xtype, size_t*);
+
 #endif /*NCOFFSETS_H*/

include/netcdf.h

@@ -742,7 +742,10 @@ nc_inq_vlen(int ncid, nc_type xtype, char *name, size_t *datum_sizep,
 /* When you read VLEN type the library will actually allocate the
  * storage space for the data. This storage space must be freed, so
  * pass the pointer back to this function, when you're done with the
- * data, and it will free the vlen memory. */
+ * data, and it will free the vlen memory.
+ * These two functions are deprecated in favor of the nc_reclaim_data function.
+ */
+
 EXTERNL int
 nc_free_vlen(nc_vlen_t *vl);
 
@@ -761,7 +764,9 @@ nc_get_vlen_element(int ncid, int typeid1, const void *vlen_element,
 /* When you read the string type the library will allocate the storage
  * space for the data. This storage space must be freed, so pass the
  * pointer back to this function, when you're done with the data, and
- * it will free the string memory. */
+ * it will free the string memory.
+ * This function is deprecated in favor of the nc_reclaim_data function.
+ */
 EXTERNL int
 nc_free_string(size_t len, char **data);
 
@@ -1753,6 +1758,53 @@ nc_put_var_string(int ncid, int varid, const char **op);
 EXTERNL int
 nc_get_var_string(int ncid, int varid, char **ip);
 
+/* Begin recursive instance walking functions */
+
+/**
+Reclaim a vector of instances of arbitrary type.  Intended for
+use with e.g. nc_get_vara or the input to e.g. nc_put_vara.
+This recursively walks the top-level instances to reclaim any
+nested data such as vlen or strings or such.
+
+Assumes it is passed a pointer to count instances of xtype.
+Reclaims any nested data.
+
+WARNING: nc_reclaim_data does not reclaim the top-level
+memory because we do not know how it was allocated.  However
+nc_reclaim_data_all does reclaim top-level memory.
+
+WARNING: all data blocks below the top-level (e.g. string
+instances) will be reclaimed, so do not call if there is any
+static data in the instance.
+
+Should work for any netcdf format.
+*/
+
+EXTERNL int nc_reclaim_data(int ncid, nc_type xtypeid, void* memory, size_t count);
+EXTERNL int nc_reclaim_data_all(int ncid, nc_type xtypeid, void* memory, size_t count);
+
+/**
+
+Copy vector of arbitrary type instances.  This recursively walks
+the top-level instances to copy any nested data such as vlen or
+strings or such.
+
+Assumes it is passed a pointer to count instances of xtype.
+WARNING: nc_copy_data does not copy the top-level memory, but
+assumes a block of proper size was passed in.  However
+nc_copy_data_all does allocate top-level memory copy.
+
+Should work for any netcdf format.
+*/
+
+EXTERNL int nc_copy_data(int ncid, nc_type xtypeid, const void* memory, size_t count, void* copy);
+EXTERNL int nc_copy_data_all(int ncid, nc_type xtypeid, const void* memory, size_t count, void** copyp);
+
+/* Instance dumper for debugging */
+EXTERNL int nc_dump_data(int ncid, nc_type xtypeid, void* memory, size_t count, char** buf);
+
+/* end recursive instance walking functions */
+
 /* Begin Deprecated, same as functions with "_ubyte" replaced by "_uchar" */
 EXTERNL int
 nc_put_att_ubyte(int ncid, int varid, const char *name, nc_type xtype,
@@ -1790,8 +1842,10 @@ nc_get_varm_ubyte(int ncid, int varid, const size_t *startp,
                   const ptrdiff_t * imapp, unsigned char *ip);
 EXTERNL int
 nc_put_var_ubyte(int ncid, int varid, const unsigned char *op);
+
 EXTERNL int
 nc_get_var_ubyte(int ncid, int varid, unsigned char *ip);
+
 /* End Deprecated */
 
 /* Set the log level. 0 shows only errors, 1 only major messages,

include/netcdf_aux.h

@@ -24,35 +24,30 @@ extern "C" {
 
 
 /**
-Reclaim the output tree of data from a call
-to e.g. nc_get_vara or the input to e.g. nc_put_vara.
+Reclaim a vector of instances of arbitrary type.
+Intended for use with e.g. nc_get_vara or the input to e.g. nc_put_vara.
 This recursively walks the top-level instances to
 reclaim any nested data such as vlen or strings or such.
 
 Assumes it is passed a pointer to count instances of xtype.
 Reclaims any nested data.
-WARNING: does not reclaim the top-level memory because
-we do not know how it was allocated.
-Should work for any netcdf format.
-*/
-
-EXTERNL int ncaux_reclaim_data(int ncid, int xtype, void* memory, size_t count);
+WARNING: ncaux_reclaim_data does not reclaim the top-level memory
+because we do not know how it was allocated.
+However ncaux_reclaim_data_all does reclaim top-level memory.
 
+WARNING: all data blocks below the top-level (e.g. string instances)
+will be reclaimed, so do not call if there is any static data in the instance.
 
-EXTERNL int ncaux_begin_compound(int ncid, const char *name, int alignmode, void** tag);
-
-EXTERNL int ncaux_end_compound(void* tag, nc_type* typeid);
-
-EXTERNL int ncaux_abort_compound(void* tag);
-
-EXTERNL int ncaux_add_field(void* tag,  const char *name, nc_type field_type,
-			   int ndims, const int* dimsizes);
+Should work for any netcdf format.
 
-/* Takes any type */
-EXTERNL size_t ncaux_type_alignment(int xtype, int ncid);
+WARNING: deprecated in favor the corresponding function in netcdf.h.
+These are just wrappers for nc_reclaim_data and
+nc_reclaim_data_all and nc_copy_data and nc_copy_data_all and
+are here for back compatibilty.
+*/
 
-/* Takes type classes only */
-EXTERNL size_t ncaux_class_alignment(int ncclass);
+EXTERNL int ncaux_reclaim_data(int ncid, int xtype, void* memory, size_t count);
+EXTERNL int ncaux_reclaim_data_all(int ncid, int xtype, void* memory, size_t count);
 
 /**************************************************/
 /* Capture the id and parameters for a filter
@@ -70,11 +65,30 @@ EXTERNL int ncaux_h5filterspec_parse_parameter(const char* txt, size_t* nuiparam
 EXTERNL void ncaux_h5filterspec_free(struct NC_H5_Filterspec* f);
 EXTERNL void ncaux_h5filterspec_fix8(unsigned char* mem, int decode);
 
+/**************************************************/
 /* Wrappers to export selected functions from libnetcdf */
 
 EXTERNL int ncaux_readfile(const char* filename, size_t* sizep, void** content);
 EXTERNL int ncaux_writefile(const char* filename, size_t size, void* content);
 
+/**************************************************/
+
+/* Takes any type */
+EXTERNL int ncaux_type_alignment(int xtype, int ncid, size_t* alignp);
+EXTERNL int ncaux_class_alignment(int ncclass, size_t* alignp);
+
+/**************************************************/
+/* Takes type classes only */
+
+/* Build compound types and properly handle offset and alignment */
+
+EXTERNL int ncaux_class_alignment(int ncclass, size_t* alignp);
+EXTERNL int ncaux_begin_compound(int ncid, const char *name, int alignmode, void** tag);
+EXTERNL int ncaux_end_compound(void* tag, nc_type* xtypeid);
+EXTERNL int ncaux_abort_compound(void* tag);
+EXTERNL int ncaux_add_field(void* tag,  const char *name, nc_type field_type,
+			   int ndims, const int* dimsizes);
+
 #if defined(__cplusplus)
 }
 #endif