Merge pull request #3159 from DennisHeimbigner/s3forward.dmh

General Cleanup to provide a clean base for future PRs.

Author: WardF

.github/workflows/run_tests_win_cygwin.yml

@@ -155,7 +155,7 @@ jobs:
           -S . \
           -DCMAKE_INSTALL_PREFIX=/tmp \
           -DBUILD_SHARED_LIBS=ON \
-          -DNETCDF_ENABLE_NETCDF_4=ON \
+          -DNETCDF_ENABLE_HDF5=ON \
           -DNETCDF_BUILD_UTILITIES=ON \
           -DNETCDF_ENABLE_TESTS=ON \
           -DNETCDF_ENABLE_HDF5=ON \
@@ -185,5 +185,5 @@ jobs:
 
       - name: (CMake) Verbose output of CTest failures
         run: >-
-          PATH=$PWD/build:$PATH ctest --test-dir build --output-on-failure -j$(nproc) --rerun-failed -VV
+          PATH=$PWD/build:$PATH ctest --test-dir build --output-on-failure -j$(nproc) --rerun-failed -VV  
         if: ${{ failure() }}

.github/workflows/run_tests_win_mingw.yml

@@ -111,7 +111,7 @@ jobs:
           -S . \
           -DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX} \
           -DBUILD_SHARED_LIBS=ON \
-          -DNETCDF_ENABLE_NETCDF_4=ON \
+          -DNETCDF_ENABLE_HDF5=ON \
           -DNETCDF_ENABLE_DAP=ON \
           -DNETCDF_BUILD_UTILITIES=ON \
           -DNETCDF_ENABLE_TESTS=ON \

CMakeLists.txt

@@ -1760,6 +1760,10 @@ endif()
 # WARNING: this must match the value in configure.ac
 set(S3TESTBUCKET "unidata-zarr-test-data" CACHE STRING "S3 test bucket")
 
+# Additional S3 Test Endpoing
+set(S3ENDPOINT "s3.us-east-1.amazonaws.com" CACHE STRING "S3 endpoint")
+
+
 # The working S3 path tree within the Unidata bucket.
 # WARNING: this must match the value in configure.ac
 set(S3TESTSUBTREE "netcdf-c" CACHE STRING "Working S3 path.")

RELEASE_NOTES.md

@@ -7,17 +7,17 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.10.0 - TBD
 
+* Rebuild the S3-related code and other changes necessary to build cleanly on github actions. See [Github #3159](https://github.com/Unidata/netcdf-c/pull/3159) for more information.
 * Fix the problems around ncdap_test/test_manyurls.c. See [Github ????](https://github.com/Unidata/netcdf-c/issues/????) for more information. 
 * Fix bug in ncdump when printing FQNs ([Issue 3184](https://github.com/Unidata/netcdf-c/issues/3184)). See [Github ????](https://github.com/Unidata/netcdf-c/issues/????) for more information. 
+
 * Update `macOS` github runners from macos-13 to macos-14, due to deprecation. 
 * Fix an error compiling netCDF with AWS-S3-SDK support using cmake. See [Github 3155](https://github.com/Unidata/netcdf-c/issues/3155) for more information. 
 * Add new environmental logging variable for `netCDF4` related logging subsystem, `NC4LOGGING`.  If `libnetcdf` is compiled with logging enabled, logs can be enabled at runtime by setting this environmental variable to the desired log level.  
-* Clean up the S3 API for all non-libnczarr code. This continues the splitting of PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068).
-See [Github #3090](https://github.com/Unidata/netcdf-c/pull/3090) for more information.
-* Step 1 in splitting PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068). Update ncjson.[ch] and ncproplist.[ch]. Also fix references to old API. Also fix include/netcdf_ncjson.h and include/netcdf_proplist.h builds. See [Github #3086](https://github.com/Unidata/netcdf-c/pull/3086) for more information.
+* Update ncjson.[ch] and ncproplist.[ch]. Also fix references to old API. Also fix include/netcdf_ncjson.h and include/netcdf_proplist.h builds. See [Github #3086](https://github.com/Unidata/netcdf-c/pull/3086) for more information.
 * Refactor drc.c to move many of its purely utility functions into dutil.c. Also change the NC_mktmp signature. Change other files to match. See [Github #3094](https://github.com/Unidata/netcdf-c/pull/3094) for more information.
 * Provide an auxilliary function, `ncaux_parse_provenance()`, that allows users to parse the _NCProperties attribute into a collection of character pointers. See [Github #3088](https://github.com/Unidata/netcdf-c/pull/3088) for more information.
-* Step 2 in splitting PR [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068). Fix a namespace problem in tinyxml2.cpp. Note that this is a visual studio problem hence use of _MSC_VER. Also turn off DAP4 tests against Hyrax server until DAP4 spec problems are fixed. See [Github #3089](https://github.com/Unidata/netcdf-c/pull/3089) for more information.
+* Fix a namespace problem in tinyxml2.cpp. Note that this is a visual studio problem hence use of _MSC_VER. Also turn off DAP4 tests against Hyrax server until DAP4 spec problems are fixed. See [Github #3089](https://github.com/Unidata/netcdf-c/pull/3089) for more information.
 
 
 ## 4.9.3 - February 7, 2025

cmake/dependencies.cmake

@@ -453,7 +453,7 @@ if(NETCDF_ENABLE_S3)
     if(NOT OpenSSL_FOUND)
       message(FATAL_ERROR "Can't find an ssl library, required by S3_INTERNAL")
     endif(NOT OpenSSL_FOUND)
-    
+
     #find_package(Crypto REQUIRED)
     #if(NOT CRYPTO_LIB)
     #  message(FATAL_ERROR "Can't find a crypto library, required by S3_INTERNAL")

cmake/netcdf_functions_macros.cmake

@@ -337,3 +337,21 @@ function(getlastdir s ret_val)
   set(${ret_val} "${last}" PARENT_SCOPE)
 endfunction()
 
+function(buildnczarrtest src)
+foreach(_BT ${ARGN})
+    SET(BTBSH "${CMAKE_CURRENT_BINARY_DIR}/run_${_BT}.sh")
+    SET(BTBTMP "${CMAKE_CURRENT_BINARY_DIR}/run_${_BT}.tmp")
+    SET(BTBSRCSH "${CMAKE_CURRENT_SOURCE_DIR}/../${src}/tst_${_BT}.sh")
+    file(REMOVE "${BTBSH}")
+    file(REMOVE "${BTBTMP}")
+    file(APPEND "${BTBTMP}" "#!/bin/bash\n")
+    file(APPEND "${BTBTMP}" "TESTNCZARR=1\n")
+    file(READ "${BTBSRCSH}" _TSTFILE)
+    string(REPLACE "\r" "" _TSTFILENOR "${_TSTFILE}")
+    file(APPEND "${BTBTMP}" "${_TSTFILENOR}")
+#    file(COPY "${BTBTMP}" DESTINATION "${BTBSH}" FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE)
+    configure_file("${BTBTMP}" "${BTBSH}" @ONLY NEWLINE_STYLE LF)
+    file(CHMOD "${BTBSH}" FILE_PERMISSIONS  OWNER_READ OWNER_WRITE OWNER_EXECUTE)
+    file(REMOVE "${BTBTMP}")
+endforeach()
+endfunction()

configure.ac

@@ -2150,6 +2150,11 @@ AC_SUBST([S3TESTBUCKET],["unidata-zarr-test-data"])
 AC_DEFINE([S3TESTSUBTREE], ["netcdf-c"], [S3 test path prefix])
 AC_SUBST([S3TESTSUBTREE],[netcdf-c])
 
+# Additional S3 Test Endpoint
+# WARNING: this must match the value in CMakeLists.txt
+AC_DEFINE([S3ENDPOINT], ["s3.us-east-1.amazonaws.com"], [S3 test endpoint])
+AC_SUBST([S3ENDPOINT],["s3.us-east-1.amazonaws.com"])
+
 # Build a small unique id to avoid interference on same platform
 PLATFORMUID="$RANDOM"
 # Make sure uid > 0

include/CMakeLists.txt

@@ -31,6 +31,10 @@ INSTALL(FILES ${netCDF_SOURCE_DIR}/include/netcdf_filter_build.h
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
     COMPONENT headers)
 
+INSTALL(FILES ${netCDF_SOURCE_DIR}/include/netcdf_vutils.h
+    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+    COMPONENT headers)
+
 INSTALL(FILES ${netCDF_BINARY_DIR}/include/netcdf_dispatch.h
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
     COMPONENT headers)

include/Makefile.am

@@ -12,6 +12,7 @@ include_HEADERS = netcdf.h netcdf_meta.h netcdf_mem.h netcdf_aux.h	\
 netcdf_filter.h netcdf_filter_build.h netcdf_filter_hdf5_build.h 	\
 netcdf_dispatch.h netcdf_vutils.h
 
+# Built headers
 include_HEADERS += netcdf_json.h netcdf_proplist.h
 
 if BUILD_PARALLEL

include/ncbytes.h

@@ -5,10 +5,10 @@
 #define NCBYTES_H 1
 
 typedef struct NCbytes {
-  int nonextendible; /* 1 => fail if an attempt is made to extend this buffer*/
+  char* content;
   unsigned long alloc;
   unsigned long length;
-  char* content;
+  int extendible; /* 0 => fail if an attempt is made to extend this buffer*/
 } NCbytes;
 
 #include "ncexternl.h"
@@ -47,8 +47,8 @@ EXTERNL int ncbytesremove(NCbytes*,unsigned long);
 /* Concatenate a null-terminated string to the end of the buffer */
 EXTERNL int ncbytescat(NCbytes*,const char*);
 
-/* Set the contents of the buffer; mark the buffer as non-extendible */
-EXTERNL int ncbytessetcontents(NCbytes*, void*, unsigned long);
+/* Set the contents of the buffer; mark the buffer extendible = false */
+EXTERNL int ncbytessetcontents(NCbytes*, void*, unsigned long, unsigned long length);
 
 /* Following are always "in-lined"*/
 #define ncbyteslength(bb) ((bb)!=NULL?(bb)->length:0)