Update to main

Author: DennisHeimbigner

.github/workflows/run_tests_win_cygwin.yml

@@ -38,11 +38,16 @@ jobs:
           platform: x86_64
           install-dir: 'C:\cygwin'
           packages: >-
-            git automake libtool autoconf2.5 make libhdf5-devel
+            git automake libtool autoconf2.5 make
             libhdf4-devel zipinfo libxml2-devel perl zlib-devel
             libzstd-devel libbz2-devel libaec-devel libzip-devel
             libdeflate-devel gcc-core gcc-g++ libcurl-devel libiconv-devel
-            libssl-devel libcrypt-devel attr libattr-devel unzip zip
+            libssl-devel libcrypt-devel attr libattr-devel unzip zip dash
+
+      - name: Install HDF5 1.14.6-1
+        shell: pwsh
+        run: |
+          C:\setup.exe -q -P libhdf5-devel=1.14.6-1
 
       - name: (Autotools) Run autoconf and friends
         run: |
@@ -130,16 +135,22 @@ jobs:
         shell: pwsh
       - uses: actions/checkout@v4
       - uses: cygwin/cygwin-install-action@v4
+        id: install-cygwin
         with:
           platform: x86_64
           install-dir: 'C:\cygwin'
           packages: >-
-            git automake libtool autoconf2.5 make libhdf5-devel
+            git automake libtool autoconf2.5 make 
             libhdf4-devel zipinfo libxml2-devel perl zlib-devel
             libzstd-devel libbz2-devel libaec-devel libzip-devel
             libdeflate-devel gcc-core gcc-g++ libcurl-devel libiconv-devel
             libssl-devel libcrypt-devel cmake ninja make m4 diffutils zip unzip
 
+      - name: Install HDF5 1.14.6-1
+        shell: pwsh
+        run: |
+          C:\setup.exe -q -P libhdf5-devel=1.14.6-1
+
 ###
 # Configure and build
 ###

.github/workflows/run_tests_win_mingw.yml

@@ -35,10 +35,21 @@ jobs:
         with:
           msystem: ${{ matrix.msystem }}
           update: true
-          pacboy: cc:p autotools:p hdf5:p curl:p libxml2:p zlib:p blosc:p bzip2:p
+          pacboy: cc:p autotools:p curl:p libxml2:p zlib:p blosc:p bzip2:p
           install: diffutils m4 git unzip
 
 ###
+# Downgrade hdf5 
+###
+
+      - name: Downgrade hdf5
+        run: |
+          if [ "${{ matrix.msystem }}" == "MINGW64" ]; then
+            pacman -U https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-hdf5-1.14.6-1-any.pkg.tar.zst --noconfirm
+          else
+            pacman -U https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-hdf5-1.14.6-1-any.pkg.tar.zst --noconfirm
+          fi
+###
 # Configure and build
 ###
 
@@ -95,10 +106,21 @@ jobs:
         with:
           msystem: ${{ matrix.msystem }}
           update: true
-          pacboy: cc:p cmake:p hdf5:p curl:p libxml2:p zlib:p blosc:p bzip2:p
+          pacboy: cc:p cmake:p curl:p libxml2:p zlib:p blosc:p bzip2:p
           install: make m4 diffutils git unzip
 
 ###
+# Downgrade hdf5 
+###
+
+      - name: Downgrade hdf5
+        run: |
+          if [ "${{ matrix.msystem }}" == "MINGW64" ]; then
+            pacman -U https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-hdf5-1.14.6-1-any.pkg.tar.zst --noconfirm
+          else
+            pacman -U https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-hdf5-1.14.6-1-any.pkg.tar.zst --noconfirm
+          fi
+###
 # Configure and build
 ###
 

RELEASE_NOTES.md

@@ -10,10 +10,10 @@ This file contains a high-level description of this package's evolution. Release
 * Add authentication support for session tokens for HDF5 ROS3
 driver. WARNING: this PR compiles and builds but the basic token
 functionality has not been tested because we have no access to a
-server that requires session tokens. See [Github
-????](https://github.com/Unidata/netcdf-c/issues/????) for more
-information.
-* Regularize, cleanup, and refactor various AWS features, especially regularizing AWS-related constants. See [Github ????](https://github.com/Unidata/netcdf-c/issues/????) for more information. 
+server that requires session tokens. See [Github????](https://github.com/Unidata/netcdf-c/issues/????) for more information.
+* Regularize, cleanup, and refactor various AWS features, especially regularizing AWS-related constants. See [Github 3229](https://github.com/Unidata/netcdf-c/issues/3229) for more information. 
+* Introduce consolidated metadata [Github #3225](https://github.com/Unidata/netcdf-c/pull/3225) via `mode=consolidated` or `NCZARR_CONSOLIDATED`
+* Fix the H5FD_class_t problems. See [Github 3202](https://github.com/Unidata/netcdf-c/issues/3202) for more information. 
 * Begin the consolidation of global state into two files: libdispatch/dglobal.c and include/ncglobal.h. See [Github 3197](https://github.com/Unidata/netcdf-c/issues/3197) for more information. 
 * Modify the way xarray attribute sets are handled. See [Github 3218](https://github.com/Unidata/netcdf-c/issues/3218) for more information. 
 * Fix Issue with Numcodecs encoding problems where integer filter parameters are being encoded as strings. See [Github 3201](https://github.com/Unidata/netcdf-c/issues/3201) for more information. 

docs/nczarr.md

@@ -119,11 +119,12 @@ The fragment part of a URL is used to specify information that is interpreted to
 For reading, _key=value_ pairs are provided for specifying the storage format.
 - mode=nczarr|zarr
 
-Additional pairs are provided to specify the Zarr version.
-- mode=v2<!--|v3-->
+Additional pairs are provided to specify
+- Zarr version `mode=v2`<!--|v3-->
 
-Additional pairs are provided to specify the storage medium: Amazon S3 vs File tree vs Zip file.
-- mode=file|zip|s3
+- Storage medium: S3, File or Zip `mode=file|zip|s3`
+
+- Additional options like consolidate(d) metadata `mode=consolidated`
 
 Note that when reading, an attempt will be made to infer the
 format and Zarr version and storage medium format by probing the
@@ -165,6 +166,12 @@ are also in the root group will have an attribute called
 *\_ARRAY\_DIMENSIONS* that stores those dimension names.
 The _noxarray_ mode tells the library to disable the XArray support.
 
+### Consolidated Metadata
+
+In the zarr specification, there is no mention to consolidated metadata. However the python implementation introduced 2 functions, `open_consolidated` and `consolidate` that given a dataset, read/write all the metadata from/to a single object (`/.zmetadata` for zarr 2). This was introduced mainly to improve the performance when accessing data remotely.
+
+The current NetCDF's zarr implementation supports consolidated operations via url fragments containing `mode=zarr,consolidated` or via environment variable `NCZARR_CONSOLIDATED`
+
 # NCZarr Map Implementation {#nczarr_mapimpl}
 
 Internally, the nczarr implementation has a map abstraction that allows different storage formats to be used.
@@ -866,6 +873,9 @@ intended to be a detailed chronology. Rather, it provides highlights
 that will be of interest to NCZarr users. In order to see exact changes,
 It is necessary to use the 'git diff' command.
 
+## 15/12/2025
+1. Include consolidated metadata.
+
 ## 03/31/2024
 1. Document the change to V2 to using attributes to hold NCZarr metadata.
 

include/CMakeLists.txt

@@ -65,7 +65,7 @@ file(READ ncjson.h JSONH0)
 STRING(REPLACE "NCJSON_H" "NETCDF_JSON_H" JSONH1 "${JSONH0}")
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/netcdf_json.h "${JSONH1}")
 file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/netcdf_json.h "#ifdef NETCDF_JSON_H\n")
-file(READ ${CMAKE_SOURCE_DIR}/libdispatch/ncjson.c JSONC0)
+file(READ ${netCDF_SOURCE_DIR}/libdispatch/ncjson.c JSONC0)
 STRING(REGEX REPLACE "#include[ ]+\"ncjson.h\"" "" JSONC1 "${JSONC0}")
 file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/netcdf_json.h "${JSONC1}")
 file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/netcdf_json.h "#endif /*NETCDF_JSON_H*/\n")
@@ -75,7 +75,7 @@ file(READ ncproplist.h PLISTH0)
 STRING(REPLACE "NCPROPLIST_H" "NETCDF_PROPLIST_H" PLISTH1 "${PLISTH0}")
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/netcdf_proplist.h "${PLISTH1}")
 file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/netcdf_proplist.h "#ifdef NETCDF_PROPLIST_H\n")
-file(READ ${CMAKE_SOURCE_DIR}/libdispatch/ncproplist.c PLISTC0)
+file(READ ${netCDF_SOURCE_DIR}/libdispatch/ncproplist.c PLISTC0)
 STRING(REGEX REPLACE "#include[ ]+\"ncproplist.h\"" "" PLISTC1 "${PLISTC0}")
 file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/netcdf_proplist.h "${PLISTC1}")
 file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/netcdf_proplist.h "#endif /*NETCDF_PROPLIST_H*/\n")

include/ncconfigure.h

@@ -67,6 +67,14 @@ char* strdup(const char*);
 #endif
 #endif
 
+#ifndef __cplusplus
+# ifndef HAVE_STRNDUP
+#  ifndef strndup
+char *strndup(const char *s, size_t len);
+#  endif
+# endif
+#endif
+
 #ifndef HAVE_STRLCAT
 #ifndef strlcat
 #define strlcat nc_strlcat

include/ncuri.h

@@ -59,6 +59,7 @@ typedef struct NCURI {
 extern "C" {
 #endif
 
+
 EXTERNL int ncuriparse(const char* s, NCURI** ncuri);
 EXTERNL void ncurifree(NCURI* ncuri);
 

include/netcdf.h

@@ -531,8 +531,10 @@ by the desired type. */
 #define NC_EOBJECT       (-140)    /**< Some object exists when it should not */
 #define NC_ENOOBJECT     (-141)    /**< Some object not found */
 #define NC_EPLUGIN       (-142)    /**< Unclassified failure in accessing a dynamically loaded plugin> */
+#define NC_ENOTZARR      (-143)    /**< Malformed (NC)Zarr file */
+#define NC_EZARRMETA     (-144)    /**< Invalid (NC)Zarr file consolidated metadata */
 
-#define NC4_LAST_ERROR   (-142)    /**< @internal All netCDF errors > this. */
+#define NC4_LAST_ERROR   (-144)    /**< @internal All netCDF errors > this. */
 
 /*
  * Don't forget to update docs/all-error-codes.md if adding new error codes here!
@@ -964,7 +966,7 @@ nc_inq_var_endian(int ncid, int varid, int *endianp);
 
 /* Define a filter for a variable */
 EXTERNL int
-nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* params);
+nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* parms);
 
 /* Learn about the first filter on a variable */
 EXTERNL int

libdispatch/dmissing.c

@@ -43,6 +43,20 @@ strdup(const char* s)
 }
 #endif
 
+#ifndef HAVE_STRNDUP
+char*
+strndup(const char* s, size_t len)
+{
+    char* dup;
+    if(s == NULL) return NULL;
+    dup = (char*)malloc(len+1);
+    if(dup == NULL) return NULL;
+    memcpy((void*)dup,s,len);
+    dup[len] = '\0';
+    return dup;
+}
+#endif
+
 #if !defined(_MSC_VER) && !defined(WIN32)
 
 #ifndef HAVE_STRLCAT

libdispatch/ncuri.c

@@ -78,21 +78,6 @@ static const char* queryallow =
 static const char* userpwdallow =
 "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!$&'()*+,-.;=_~?#/";
 
-#ifndef HAVE_STRNDUP
-#define strndup ncstrndup
-/* Not all systems have strndup, so provide one*/
-char*
-ncstrndup(const char* s, size_t len)
-{
-    char* dup;
-    if(s == NULL) return NULL;
-    dup = (char*)malloc(len+1);
-    if(dup == NULL) return NULL;
-    memcpy((void*)dup,s,len);
-    dup[len] = '\0';
-    return dup;
-}
-#endif
 /* Forward */
 static int collectprefixparams(char* text, char** nextp);
 static void freestringlist(NClist* list);