Merge pull request #2114 from DennisHeimbigner/zarrs3.dmh

Significantly Improve Amazon S3 Cloud Storage Support

Author: WardF

CMakeLists.txt

@@ -1148,7 +1148,7 @@ ENDIF(ENABLE_NCZARR_S3)
 
 IF(NOT ENABLE_S3_SDK)
   IF(ENABLE_NCZARR_S3 OR ENABLE_NCZARR_S3_TESTS)
-    message(FATAL_ERROR "AWS S3 libraries not found; please specify option DENABLE_NCZARR_S3=NO")
+    message(FATAL_ERROR "S3 support library not found; please specify option DENABLE_NCZARR_S3=NO")
     SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "NCZARR S3 support" FORCE)
     SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE)
   ENDIF()

Makefile.am

@@ -113,7 +113,8 @@ endif
 # and run. ncgen must come before ncdump, because their tests
 # depend on it.
 SUBDIRS = include $(H5_TEST_DIR) libdispatch libsrc $(LIBSRC4_DIR)	\
-$(LIBSRCP) $(LIBHDF4) $(LIBHDF5) $(OCLIB) $(DAP2) ${DAP4} ${NCPOCO} ${ZARR} liblib \
+$(LIBSRCP) $(LIBHDF4) $(LIBHDF5) $(OCLIB) $(DAP2) ${DAP4} \
+${NCPOCO} ${ZARR} liblib \
 $(NCGEN3) $(NCGEN) $(NCDUMP) ${PLUGIN_DIR} $(TESTDIRS) docs		\
 $(EXAMPLES)
 

RELEASE_NOTES.md

@@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.8.2 - TBD
 
+* [Enhancement] Support Amazon S3 access for NCZarr. Also support use of the existing Amazon SDK credentials system. See [Github #2114](https://github.com/Unidata/netcdf-c/pull/2114)
 * [Bug Fix] Fix string allocation error in H5FDhttp.c. See [Github #2127](https://github.com/Unidata/netcdf-c/pull/2127).
 * [Bug Fix] Apply patches for ezxml and for selected  oss-fuzz detected errors. See [Github #2125](https://github.com/Unidata/netcdf-c/pull/2125).
 * [Bug Fix] Ensure that internal Fortran APIs are always defined. See [Github #2098](https://github.com/Unidata/netcdf-c/pull/2098).

configure.ac

@@ -642,25 +642,27 @@ AC_MSG_RESULT($enable_nczarr_s3)
 # Note we check for the library after checking for enable_nczarr_s3
 # because for some reason this screws up if we unconditionally test for sdk
 # and it is not available. Fix someday
-have_aws=no
+enable_s3_sdk=no
 if test "x$enable_nczarr_s3" = xyes ; then
 # See if we have the s3 aws library
 # Check for the AWS S3 SDK library
 AC_LANG_PUSH([C++])
-AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [have_aws=yes],[have_aws=no])
+AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [enable_s3_sdk=yes],[enable_s3_sdk=no])
 AC_LANG_POP
 fi
 
 AC_MSG_CHECKING([whether AWS S3 SDK library is available])
-AC_MSG_RESULT([$have_aws])
+AC_MSG_RESULT([$enable_s3_sdk])
 
-if test "x$have_aws" = xno ; then
-AC_MSG_WARN([AWS SDK not found; disabling S3 support])
+if test "x$enable_s3_sdk" = xno ; then
+AC_MSG_WARN([No S3 library available; disabling S3 support])
 enable_nczarr_s3=no
-else
+fi
+
+if test "x$enable_s3_sdk" = xyes ; then
 AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
 fi
-AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes])
+AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes])
 
 # Check for enabling S3 testing
 AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
@@ -1265,10 +1267,10 @@ if test "x$enable_hdf5" = xyes; then
 
    # Check for the main hdf5 and hdf5_hl library.
 
-   AC_SEARCH_LIBS([H5Fflush], [hdf5dll hdf5], [],
+   AC_SEARCH_LIBS([H5Fflush], [hdf5 hdf5.dll], [],
+   [AC_MSG_ERROR([Can't find or link to the hdf5 library. Use --disable-hdf5, or see config.log for errors.])])
+   AC_SEARCH_LIBS([H5DSis_scale], [hdf5_hl hdf5_hl.dll], [],
    [AC_MSG_ERROR([Can't find or link to the hdf5 library. Use --disable-hdf5, or see config.log for errors.])])
-   AC_SEARCH_LIBS([H5DSis_scale], [hdf5_hldll hdf5_hl], [],
-   [AC_MSG_ERROR([Can't find or link to the hdf5 high-level. Use --disable-hdf5, or see config.log for errors.])])
 
    AC_CHECK_HEADERS([hdf5.h], [], [AC_MSG_ERROR([Compiling a test with HDF5 failed.  Either hdf5.h cannot be found, or config.log should be checked for other reason.])])
 
@@ -1600,7 +1602,7 @@ AM_CONDITIONAL(ENABLE_BYTERANGE, [test "x$enable_byterange" = xyes])
 AM_CONDITIONAL(RELAX_COORD_BOUND, [test "xyes" = xyes])
 AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ])
 AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
-AM_CONDITIONAL(HAVE_AWS, [test "x$have_aws" = xyes])
+AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes])
 AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
 AM_CONDITIONAL(ENABLE_BLOSC, [test "x$enable_blosc" = xyes])
 AM_CONDITIONAL(ENABLE_SZIP, [test "x$enable_szip" = xyes])

dap4_test/baselinethredds/CMakeLists.txt

@@ -6,6 +6,7 @@
 # See netcdf-c/COPYRIGHT file for more info.
 
 
+FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds)
 FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*)
 FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE)
 

docs/CMakeLists.txt

@@ -92,6 +92,6 @@ obsolete/fan_utils.html bestpractices.md filters.md indexing.md
 inmemory.md DAP2.dox attribute_conventions.md FAQ.md
 file_format_specifications.md known_problems.md
 COPYRIGHT.dox user_defined_formats.md DAP4.md DAP4.dox
-testserver.dox byterange.dox filters.md nczarr.md)
+testserver.dox byterange.dox filters.md nczarr.md auth.md)
 
 ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

docs/Doxyfile.in

@@ -742,8 +742,9 @@ INPUT = \
     @abs_top_srcdir@/docs/windows-binaries.md \
     @abs_top_srcdir@/docs/attribute_conventions.md \
     @abs_top_srcdir@/docs/file_format_specifications.md \
-    @abs_top_srcdir@/docs/byterange.dox \
     @abs_top_srcdir@/docs/inmemory.md \
+    @abs_top_srcdir@/docs/filters.md \
+    @abs_top_srcdir@/docs/byterange.dox \
     @abs_top_srcdir@/docs/auth.md \
     @abs_top_srcdir@/docs/nczarr.md \
     @abs_top_srcdir@/docs/notes.md \

docs/Doxyfile.user

@@ -751,6 +751,7 @@ INPUT = \
     ./docs/byterange.dox \
     ./docs/inmemory.md \
     ./docs/auth.md \
+    ./docs/filters.md \
     ./docs/notes.md \
     ./docs/all-error-codes.md \
     ./docs/building-with-cmake.md \

docs/Makefile.am

@@ -11,7 +11,7 @@ EXTRA_DIST = netcdf.m4 DoxygenLayout.xml Doxyfile.in footer.html	\
 mainpage.dox tutorial.dox cdl.dox			\
 architecture.dox internal.dox windows-binaries.md			\
 building-with-cmake.md CMakeLists.txt groups.dox notes.md	\
-install-fortran.md all-error-codes.md credits.md auth.md		\
+install-fortran.md all-error-codes.md credits.md auth.md filters.md \
 obsolete/fan_utils.html indexing.dox	\
 inmemory.md attribute_conventions.md FAQ.md	                \
 file_format_specifications.md known_problems.md COPYRIGHT.md		\