diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 2cc50fb148..79c0c2a66f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,8 +4,10 @@ Release Notes {#RELEASE_NOTES} \brief Release notes file for the netcdf-c package. This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries. + ## 4.8.1 - TBD +* [Bug Fixes] The netcdf-c library was incorrectly determining the scope of types referred to by nc_inq_type_equal. See [Github #1959](https://github.com/Unidata/netcdf-c/pull/1959) for more information. * [Bug Fix] Fix bug in use of XGetopt when building under Mingw. See [Github #2009](https://github.com/Unidata/netcdf-c/issues/2009). * [Enhancement] Improve the error reporting when attempting to use a filter for which no implementation can be found in HDF5_PLUGIN_PATH. See [Github #2000](https://github.com/Unidata/netcdf-c/pull/2000) for more information. * [Bug Fix] Fix `make distcheck` issue in `nczarr_test/` directory. See [Github #2007](https://github.com/Unidata/netcdf-c/issues/2007). diff --git a/include/ncconfigure.h b/include/ncconfigure.h index 32fd1cc9bb..2b4375ea58 100644 --- a/include/ncconfigure.h +++ b/include/ncconfigure.h @@ -16,6 +16,9 @@ #ifdef HAVE_STDIO_H #include #endif +#ifdef HAVE_STDINT_H +#include +#endif /* This is included in bottom diff --git a/include/ncpathmgr.h b/include/ncpathmgr.h index 631327b7e2..e1b33b8fa9 100644 --- a/include/ncpathmgr.h +++ b/include/ncpathmgr.h @@ -102,6 +102,9 @@ EXTERNL int NChasdriveletter(const char* path); /* Canonicalize and make absolute by prefixing the current working directory */ EXTERNL char* NCpathabsolute(const char* name); +/* Check if this path appears to start with a windows drive letter */ +EXTERNL int NChasdriveletter(const char* path); + /* Convert from the local coding (e.g. ANSI) to utf-8; note that this can produce unexpected results for Windows because it first converts to wide character and then to utf8. */ diff --git a/libdispatch/dcopy.c b/libdispatch/dcopy.c index 04c748f7b2..8a73f9d82a 100644 --- a/libdispatch/dcopy.c +++ b/libdispatch/dcopy.c @@ -10,8 +10,13 @@ #include "config.h" #include "ncdispatch.h" #include "nc_logging.h" +#include "nclist.h" #ifdef USE_NETCDF4 + +static int searchgroup(int ncid1, int tid1, int grp, int* tid2); +static int searchgrouptree(int ncid1, int tid1, int grp, int* tid2); + /** * @internal Compare two netcdf types for equality. Must have the * ncids as well, to find user-defined types. @@ -27,8 +32,7 @@ * @author Ed Hartnett */ static int -NC_compare_nc_types(int ncid1, int typeid1, int ncid2, int typeid2, - int *equalp) +NC_compare_nc_types(int ncid1, int typeid1, int ncid2, int typeid2, int *equalp) { int ret = NC_NOERR; @@ -152,8 +156,15 @@ NC_compare_nc_types(int ncid1, int typeid1, int ncid2, int typeid2, } /** - * @internal Recursively hunt for a netCDF type id. (Code from - * nc4internal.c); Return matching typeid or 0 if not found. + * @internal Recursively hunt for a netCDF type id, tid2, that is "equal" to tid1. + * Question is: what search order do we use? Ncgen uses root group tree in pre-order. + * But NC4_inq_typeid uses these rules: + * 1. ncid2 + * 2. parents of ncid2 (up the tree to root) + * 3. root group tree in pre-order. + * We will leave ncgen for another day and use the nc_inq_typeid rule. + * + * Return matching typeid or 0 if not found. * * @param ncid1 File ID. * @param tid1 Type ID. @@ -161,68 +172,35 @@ NC_compare_nc_types(int ncid1, int typeid1, int ncid2, int typeid2, * @param tid2 Pointer that gets type ID of equal type. * * @return ::NC_NOERR No error. - * @author Ed Hartnett + * @author Ed Hartnett, Dennis Heimbigner */ static int NC_rec_find_nc_type(int ncid1, nc_type tid1, int ncid2, nc_type* tid2) { - int i,ret = NC_NOERR; - int nids; - int* ids = NULL; - - /* Get all types in grp ncid2 */ - if(tid2) - *tid2 = 0; - if ((ret = nc_inq_typeids(ncid2, &nids, NULL))) - return ret; - if (nids) - { - if (!(ids = (int *)malloc((size_t)nids * sizeof(int)))) - return NC_ENOMEM; - if ((ret = nc_inq_typeids(ncid2, &nids, ids))) - return ret; - for(i = 0; i < nids; i++) - { - int equal = 0; - if ((ret = NC_compare_nc_types(ncid1, tid1, ncid2, ids[i], &equal))) - return ret; - if(equal) - { - if(tid2) - *tid2 = ids[i]; - free(ids); - return NC_NOERR; - } - } - free(ids); + int ret = NC_NOERR; + int parent; + + if((ret = searchgroup(ncid1,tid1,ncid2,tid2))) + goto done; + if(*tid2 != 0) + goto done; /* found */ + + /* Look in the parents of ncid2 upto the root */ + switch (ret = nc_inq_grp_parent(ncid2,&parent)) { + case NC_NOERR: + /* Recurse up using parent grp */ + ret = NC_rec_find_nc_type(ncid1, tid1, parent, tid2); + break; + case NC_ENOGRP: + /* do the breadth-first pre-order search of the whole tree */ + /* ncid2 should be root group */ + ret = searchgrouptree(ncid1,tid1,ncid2,tid2); + break; + default: break; } - /* recurse */ - if ((ret = nc_inq_grps(ncid1, &nids, NULL))) - return ret; - if (nids) - { - if (!(ids = (int *)malloc((size_t)nids * sizeof(int)))) - return NC_ENOMEM; - if ((ret = nc_inq_grps(ncid1, &nids, ids))) - { - free(ids); - return ret; - } - for (i = 0; i < nids; i++) - { - ret = NC_rec_find_nc_type(ncid1, tid1, ids[i], tid2); - if (ret && ret != NC_EBADTYPE) - break; - if (tid2 && *tid2 != 0) /* found */ - { - free(ids); - return NC_NOERR; - } - } - free(ids); - } - return NC_EBADTYPE; /* not found */ +done: + return ret; } /** @@ -711,3 +689,92 @@ nc_copy_att(int ncid_in, int varid_in, const char *name, return NC_NOERR; } + +#ifdef USE_NETCDF4 + +/* Helper function for NC_rec_find_nc_type(); + search a specified group for matching type. +*/ +static int +searchgroup(int ncid1, int tid1, int grp, int* tid2) +{ + int i,ret = NC_NOERR; + int nids; + int* ids = NULL; + + /* Get all types in grp */ + if(tid2) + *tid2 = 0; + if ((ret = nc_inq_typeids(grp, &nids, NULL))) + goto done; + if (nids) + { + if (!(ids = (int *)malloc((size_t)nids * sizeof(int)))) + {ret = NC_ENOMEM; goto done;} + if ((ret = nc_inq_typeids(grp, &nids, ids))) + goto done; + for(i = 0; i < nids; i++) + { + int equal = 0; + if ((ret = NC_compare_nc_types(ncid1, tid1, grp, ids[i], &equal))) + goto done; + if(equal) + { + if(tid2) + *tid2 = ids[i]; + goto done; + } + } + } + +done: + nullfree(ids); + return ret; +} + +/* Helper function for NC_rec_find_nc_type(); + search a tree of groups for a matching type + using a breadth first queue +*/ +static int +searchgrouptree(int ncid1, int tid1, int grp, int* tid2) +{ + int i,ret = NC_NOERR; + int nids; + int* ids = NULL; + NClist* queue = nclistnew(); + int gid; + uintptr_t id; + + id = grp; + nclistpush(queue,(void*)id); /* prime the queue */ + while(nclistlength(queue) > 0) { + id = (uintptr_t)nclistremove(queue,0); + gid = (int)id; + if((ret = searchgroup(ncid1,tid1,gid,tid2))) + goto done; + if(*tid2 != 0) + goto done; /*we found it*/ + /* Get subgroups of gid and push onto front of the queue (for breadth first) */ + if((ret = nc_inq_grps(gid,&nids,NULL))) + goto done; + if (!(ids = (int *)malloc((size_t)nids * sizeof(int)))) + {ret = NC_ENOMEM; goto done;} + if ((ret = nc_inq_grps(gid, &nids, ids))) + goto done; + /* push onto the end of the queue */ + for(i=0;inc4_info->root_grp->hdr.id | grp->nc4_info->controller->ext_ncid); + int parent = 0; + char* lastname = strrchr(norm_name,'/'); /* break off the last segment: the type name */ + if(lastname == norm_name) + {retval = NC_EINVAL; goto done;} + *lastname++ = '\0'; /* break off the lastsegment */ + if((retval = NC4_inq_grp_full_ncid(rootncid,norm_name,&parent))) + goto done; + /* Get parent info */ + if((retval=nc4_find_nc4_grp(parent,&grp))) + goto done; + /* See if type exists in this group */ + type = (NC_TYPE_INFO_T*)ncindexlookup(grp->type,lastname); + if(type == NULL) + {retval = NC_EBADTYPE; goto done;} + goto done; } + /* Is the type in this group? If not, search parents. */ for (grptwo = grp; grptwo; grptwo = grptwo->parent) { type = (NC_TYPE_INFO_T*)ncindexlookup(grptwo->type,norm_name); @@ -602,13 +622,13 @@ NC4_inq_typeid(int ncid, const char *name, nc_type *typeidp) if (typeidp) *typeidp = type->hdr.id; - free(norm_name); - /* OK, I give up already! */ if (!type) - return NC_EBADTYPE; + {retval = NC_EBADTYPE; goto done;} - return NC_NOERR; +done: + nullfree(norm_name); + return retval; } /** diff --git a/ncdump/CMakeLists.txt b/ncdump/CMakeLists.txt index 462231bb33..b8cfdc5bfc 100644 --- a/ncdump/CMakeLists.txt +++ b/ncdump/CMakeLists.txt @@ -13,6 +13,7 @@ SET(ncdump_FILES ncdump.c vardata.c dumplib.c indent.c nctime0.c utils.c nciter. SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c list.c) SET(ocprint_FILES ocprint.c) SET(ncvalidator_FILES ncvalidator.c) +SET(printfqn_FILES printfqn.c) IF(USE_X_GETOPT) SET(ncdump_FILES ${ncdump_FILES} XGetopt.c) @@ -27,6 +28,7 @@ ADD_EXECUTABLE(ncvalidator ${ncvalidator_FILES}) IF(USE_HDF5) ADD_EXECUTABLE(nc4print nc4print.c nc4printer.c) + ADD_EXECUTABLE(printfqn ${printfqn_FILES}) ENDIF(USE_HDF5) IF(ENABLE_DAP) @@ -39,6 +41,7 @@ TARGET_LINK_LIBRARIES(ncvalidator netcdf ${ALL_TLL_LIBS}) IF(USE_HDF5) TARGET_LINK_LIBRARIES(nc4print netcdf ${ALL_TLL_LIBS}) + TARGET_LINK_LIBRARIES(printfqn netcdf ${ALL_TLL_LIBS}) ENDIF(USE_HDF5) IF(ENABLE_DAP) @@ -81,6 +84,15 @@ IF(MSVC) ${CMAKE_CURRENT_BINARY_DIR}) SET_TARGET_PROPERTIES(ncvalidator PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR}) + + SET_TARGET_PROPERTIES(printfqn PROPERTIES RUNTIME_OUTPUT_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(printfqn PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG + ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(printfqn PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE + ${CMAKE_CURRENT_BINARY_DIR}) + + ENDIF(USE_HDF5) IF(ENABLE_DAP) @@ -315,8 +327,14 @@ IF(ENABLE_TESTS) add_sh_test(ncdump test_keywords) ENDIF() + + IF(USE_HDF5) + add_sh_test(ncdump test_scope) + ENDIF() + add_sh_test(ncdump test_rcmerge) + ENDIF(ENABLE_TESTS) #IF(MSVC) diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index 017a00d994..1eea46a1a0 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -46,6 +46,13 @@ if USE_HDF5 bin_PROGRAMS += nc4print noinst_PROGRAMS += nc4print nc4print_SOURCES = nc4print.c nc4printer.c + +# Create a helper program for tst_scope.sh +# Program prints out the fqn of the type of +# a specified variable in the .nc file. +noinst_PROGRAMS += printfqn +printfqn_SOURCES = printfqn.c + endif # Conditionally build the ocprint program, but do not install @@ -103,7 +110,7 @@ TESTS += tst_fileinfo.sh tst_hdf5_offset.sh tst_inttags4.sh \ tst_netcdf4.sh tst_fillbug.sh tst_netcdf4_4.sh tst_nccopy4.sh \ tst_nccopy5.sh tst_grp_spec.sh tst_mud.sh tst_h_scalar.sh tst_formatx4.sh \ run_utf8_nc4_tests.sh run_back_comp_tests.sh run_ncgen_nc4_tests.sh \ -tst_ncgen4.sh +tst_ncgen4.sh test_scope.sh # Record interscript dependencies so parallel builds work. tst_nccopy4.log: run_ncgen_tests.log tst_output.log tst_ncgen4.log \ @@ -166,10 +173,10 @@ ref_provenance_v1.nc ref_tst_radix.cdl tst_radix.cdl test_radix.sh \ ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh \ ref_no_ncproperty.nc test_unicode_directory.sh \ ref_roman_szip_simple.cdl ref_roman_szip_unlim.cdl ref_tst_perdimspecs.cdl \ -test_keywords.sh ref_keyword1.cdl ref_keyword2.cdl \ -ref_keyword3.cdl ref_keyword4.cdl \ -ref_tst_nofilters.cdl test_unicode_path.sh test_rcmerge.sh \ -ref_rcmerge1.txt ref_rcmerge2.txt ref_rcmerge3.txt +test_keywords.sh ref_keyword1.cdl ref_keyword2.cdl ref_keyword3.cdl ref_keyword4.cdl \ +ref_tst_nofilters.cdl test_scope.sh type_group_only.cdl type_ancestor_only.cdl \ +type_ancestor_subgroup.cdl type_preorder.cdl test_unicode_path.sh \ +test_rcmerge.sh ref_rcmerge1.txt ref_rcmerge2.txt ref_rcmerge3.txt # The L512.bin file is file containing exactly 512 bytes each of value 0. # It is used for creating hdf5 files with varying offsets for testing. @@ -206,9 +213,11 @@ ctest64.c nccopy3_subset_out.nc camrun.c tst_ncf213.cdl tst_ncf213.nc \ tst_radix.nc tmp_radix.cdl ctest_small_3.c ctest_small_4.c \ ctest_special_atts_4.c tst_roman_szip_simple.cdl \ tst_roman_szip_unlim.cdl tst_perdimpspecs.nc tmppds.* \ -keyword1.nc keyword2.nc keyword3.nc keyword4.nc \ -tmp_keyword1.cdl tmp_keyword2.cdl tmp_keyword3.cdl tmp_keyword4.cdl +keyword1.nc keyword2.nc keyword3.nc keyword4.nc \ +tmp_keyword1.cdl tmp_keyword2.cdl tmp_keyword3.cdl tmp_keyword4.cdl \ +type_*.nc copy_type_*.cdl # Remove directories clean-local: rm -fr rcmergedir + diff --git a/ncdump/printfqn.c b/ncdump/printfqn.c new file mode 100644 index 0000000000..2392c460c4 --- /dev/null +++ b/ncdump/printfqn.c @@ -0,0 +1,143 @@ +#include +#include +#include +#include +#include + +#define CHECK(err) {if(err) report(err,__LINE__);} + +static void +report(int err, int lineno) +{ + fprintf(stderr,"Error: %d: %s\n", lineno, nc_strerror(err)); + exit(1); +} + +void +usage(void) +{ + fprintf(stderr,"usage: printfqn \n"); + exit(0); +} + +int +get_type_parent(int ncid, int tid, int* parentp) +{ + int stat = NC_NOERR; + int i; + int nids; + int ids[4096]; + + /* Does this group have the type we are searching for? */ + if((stat=nc_inq_typeids(ncid,&nids,ids))) goto done; + assert(nids < 4096); + + /* Search for this typeid */ + for(i=0;i copy_$1.cdl +diff -wB ${srcdir}/$1.cdl ${execdir}/copy_$1.cdl +REFT=`${execdir}/printfqn ${execdir}/$1.nc test_variable` +COPYT=`${execdir}/printfqn ${execdir}/$1_copy.nc test_variable` +if test "x$REFT" != "x$COPYT" ; then + echo "***Fail: ref=${REFT} copy=${COPYT}" +fi +} + +if test "x$SETUP" = x1 ; then +for t in $TSTS ; do + setup $t +done +fi + +for t in $TSTS ; do + testscope $t +done + +exit 0 + + diff --git a/ncdump/type_ancestor_only.cdl b/ncdump/type_ancestor_only.cdl new file mode 100644 index 0000000000..51e64711df --- /dev/null +++ b/ncdump/type_ancestor_only.cdl @@ -0,0 +1,21 @@ +netcdf type_ancestor_only { +types: + ubyte enum test_enum_type {OPTION1 = 0, OPTION2 = 1, OPTION3 = 2} ; + +group: test_group { + variables: + test_enum_type test_variable ; + test_enum_type test_variable:_FillValue = OPTION1 ; + + group: test_subgroup1 { + + group: test_subsubgroup { + } // group test_subsubgroup + + } // group test_subgroup1 + + group: test_subgroup2 { + } // group test_subgroup2 + + } // group test_group +} diff --git a/ncdump/type_ancestor_subgroup.cdl b/ncdump/type_ancestor_subgroup.cdl new file mode 100644 index 0000000000..b213d5beaa --- /dev/null +++ b/ncdump/type_ancestor_subgroup.cdl @@ -0,0 +1,15 @@ +netcdf type_ancestor_subgroup { +types: + ubyte enum test_enum_type {OPTION1 = 0, OPTION2 = 1, OPTION3 = 2} ; + +group: test_group { + variables: + test_enum_type test_variable ; + test_enum_type test_variable:_FillValue = OPTION1 ; + group: sub_group { + types: + ubyte enum test_enum_type {OPTION1 = 0, OPTION2 = 1, OPTION3 = 2} ; + } // group sub_group + + } // group test_group +} diff --git a/ncdump/type_group_only.cdl b/ncdump/type_group_only.cdl new file mode 100644 index 0000000000..b19f38cfce --- /dev/null +++ b/ncdump/type_group_only.cdl @@ -0,0 +1,10 @@ +netcdf type_group_only { + +group: test_group { + types: + ubyte enum test_enum_type {OPTION1 = 0, OPTION2 = 1, OPTION3 = 2} ; + variables: + test_enum_type test_variable ; + test_enum_type test_variable:_FillValue = OPTION1 ; + } // group test_group +} diff --git a/ncdump/type_preorder.cdl b/ncdump/type_preorder.cdl new file mode 100644 index 0000000000..4b888fc6c3 --- /dev/null +++ b/ncdump/type_preorder.cdl @@ -0,0 +1,13 @@ +netcdf type_preorder { + +group: preorder { +types: + ubyte enum test_enum_type {OPTION1 = 0, OPTION2 = 1, OPTION3 = 2} ; +} // group preorder + +group: test_group { + variables: + /preorder/test_enum_type test_variable ; + /preorder/test_enum_type test_variable:_FillValue = OPTION1 ; + } // group test_group +}