NetCDF-C version 4.9.0 is required

* NetCDF-4 uses the dimension scale feature which is part of HDF5
  high-level APIs, but HDF5 high-level APIs are not well tested for
  parallel I/O. See Unidata/netcdf-c#2251 and
  Unidata/netcdf-c#1822
* NetCDF PR #2161 adds a new flag NC_NODIMSCALE_ATTACH to allow users to
  disable dimension scale, which resolves the problem for e3sm-io. See
  Unidata/netcdf-c#2161
* NetCDF team indicates PR #2161 will appear in version 4.9.0.

Author: wkliao

INSTALL.md

@@ -15,7 +15,7 @@
   + Experimental software developed as part of the Datalib project
 * (Optional) [ADIOS 2.7.1](https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.7.1.tar.gz)
   + Configured with parallel I/O support (cmake with `-DADIOS2_USE_MPI=ON` is required)
-* (Optional) [NetCDF-C 4.8.1](https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.8.1.tar.gz)
+* (Optional) [NetCDF-C 4.9.0](https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.0.tar.gz)
   + Configured with parallel HDF5 support (i.e. `--enable-netcdf4`)
   + Note currently this option fails to run due to a
     [bug](https://github.com/Unidata/netcdf-c/issues/2251) in NetCDF-C.
@@ -79,12 +79,12 @@
   + Configure NetCDF-C with parallel HDF5 I/O enabled.
   + Run `make install`
   + Example build commands are given below. This example will install
-    the NetCDF library under the folder `${HOME}/NetCDF/4.8.1`.
+    the NetCDF library under the folder `${HOME}/NetCDF/4.9.0`.
     ```
-    % wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.8.1.tar.gz
-    % tar -zxf v4.8.1.tar.gz
-    % cd netcdf-c-4.8.1
-    % ./configure --prefix=${HOME}/NetCDF/4.8.1 \
+    % wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.0.tar.gz
+    % tar -zxf v4.9.0.tar.gz
+    % cd netcdf-c-4.9.0
+    % ./configure --prefix=${HOME}/NetCDF/4.9.0 \
                   CC=mpicc \
                   CPPFLAGS=-I${HOME}/HDF5/1.13.0/include \
                   LDFLAGS=-L${HOME}/HDF5/1.13.0/lib \
@@ -121,6 +121,7 @@
                   --with-hdf5=${HOME}/HDF5/1.13.0 \
                   --with-logvol=${HOME}/Log_VOL/1.1.0 \
                   --with-adios2=${HOME}/ADIOS2/2.7.1 \
+                  --with-netcdf4=${HOME}/NetCDF/4.9.0 \
                   CC=mpicc CXX=mpicxx
     % make
     ```
@@ -208,13 +209,13 @@
 
 ### Current supported APIs (option `-a`) and I/O strategies (option `-x`)
   + Table below lists the supported combinations.
-     |           | pnetcdf | hdf5 | hdf5_log | netcdf4 | adios |
-     |-----------|:-------:|:----:|:--------:|:-------:|:-----:|
-     | canonical | yes     | yes  | no       | yes     | no    |
-     | log       | no      | yes  | yes      | yes*    | no    |
-     | blob      | yes     | yes  | no       | no      | yes   |
+     |           | pnetcdf | hdf5 | hdf5_log | netcdf4* | adios |
+     |-----------|:-------:|:----:|:--------:|:--------:|:-----:|
+     | canonical | yes     | yes  | no       | yes      | no    |
+     | log       | no      | yes  | yes      | yes      | no    |
+     | blob      | yes     | yes  | no       | no       | yes   |
      
-     `*` Requires setting of 2 VOL environment variables. See description below.
+     `*` NetCDF-C version 4.9.0 or newer is required.
 
   + **-a pnetcdf -x canonical**
     * A single NetCDF file in the classic CDF5 format will be created. All

configure.ac

@@ -434,6 +434,7 @@ AC_ARG_WITH([netcdf4],
    if test "x$netcdf4_config" != x ; then
       netcdf4_cflags=`$netcdf4_config --cflags`
       netcdf4_lflags=`$netcdf4_config --libs`
+      netcdf4_version=`$netcdf4_config --version | cut -d' ' -f2 | cut -d'-' -f1`
    else
       netcdf4_cflags="-I$netcdf4_inc"
       netcdf4_lflags="-L$netcdf4_lib"
@@ -454,17 +455,48 @@ AC_ARG_WITH([netcdf4],
          LDFLAGS="$LDFLAGS $netcdf4_lflags"
       fi
    fi
-   AC_CHECK_HEADER([netcdf.h], [have_netcdf4=yes], [have_netcdf4=no])
+   AC_CHECK_HEADERS([netcdf.h netcdf_meta.h], [have_netcdf4=yes], [have_netcdf4=no])
    if test "x$have_netcdf4" = xno ; then
       AC_MSG_WARN([
       -----------------------------------------------------------------------
-      Missing NetCDF-header files 'netcdf4_c.h' required to build E3SM_IO. Use
-      configure command-line option --with-netcdf4=/path/to/implementation
-      to specify the location of NetCDF installation. Disable NetCDF 4 support.
+      Missing NetCDF-header files 'netcdf.h' or 'netcdf_meta.h' which are
+      required to build E3SM_IO. Use configure command-line option
+      --with-netcdf4=/path/to/implementation to specify the location of
+      NetCDF-4 installation. Disable NetCDF-4 support.
       -----------------------------------------------------------------------])
    fi
 
-   AC_MSG_CHECKING([if NetCDF $netcdf_version is configured with parallel I/O enabled])
+   if test "x$have_netcdf4" = xyes ; then
+      AC_MSG_CHECKING([NetCDF-C version])
+      if test "x$netcdf4_version" = x ; then
+         nc_version_major=`${EGREP} NC_VERSION_MAJOR $netcdf4_inc/netcdf_meta.h| head -1 | cut -d' ' -f3`
+         nc_version_minor=`${EGREP} NC_VERSION_MINOR $netcdf4_inc/netcdf_meta.h| head -1 | cut -d' ' -f3`
+         nc_version_patch=`${EGREP} NC_VERSION_PATCH $netcdf4_inc/netcdf_meta.h| head -1 | cut -d' ' -f3`
+         netcdf4_version="${nc_version_major}.${nc_version_minor}.${nc_version_patch}"
+      fi
+      AC_MSG_RESULT([$netcdf4_version])
+
+      dnl Check if NetCDF version is 4.9.0 or later
+      AC_MSG_CHECKING([whether NetCDF-C version is 4.9.0 or later])
+      AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+#include <netcdf_meta.h>
+#define VER_NUM (NC_VERSION_MAJOR*1000000 + NC_VERSION_MINOR*1000 + NC_VERSION_PATCH)
+#if (VER_NUM < 4009000)
+#error NetCDF version is older than 4.9.0
+#endif
+      ]])], [netcdf_ge_4_9_0=yes], [netcdf_ge_4_9_0=no])
+      AC_MSG_RESULT([$netcdf_ge_4_9_0])
+
+      if test "x${netcdf_ge_4_9_0}" = xno; then
+         AC_MSG_ERROR([
+         ------------------------------------------------------------
+         Supplied NetCDF-4 version is $netcdf4_version, but 4.9.0 or later
+         is required. Abort.
+         ------------------------------------------------------------])
+      fi
+   fi
+
+   AC_MSG_CHECKING([if NetCDF-4 is configured with parallel I/O enabled])
    if test "x$netcdf4_config" != x ; then
       netcdf4_has_par=`$netcdf4_config --has-parallel4`
    else

src/drivers/e3sm_io_driver_nc4.cpp

@@ -73,7 +73,7 @@ int e3sm_io_driver_nc4::create (std::string path, MPI_Comm comm, MPI_Info info,
     E3SM_IO_TIMER_START (E3SM_IO_TIMER_NC4)
     E3SM_IO_TIMER_START (E3SM_IO_TIMER_NC4_OPEN)
 
-    err = nc_create_par (path.c_str (), NC_CLOBBER | NC_NETCDF4, comm, info, fid);
+    err = nc_create_par (path.c_str (), NC_CLOBBER | NC_NETCDF4 | NC_NODIMSCALE_ATTACH, comm, info, fid);
     CHECK_NCERR
 
     /* turn off fill mode for the entire file */

test.sh

@@ -48,13 +48,6 @@ if test "x${ENABLE_ADIOS2}" = x1 ; then
 fi
 
 if test "x${ENABLE_NETCDF4}" = x1 ; then
-   if test "x$#" = x0 ; then :; else
-      echo "==================================================================="
-      echo "Warning: skip NetCDF-4 parallel tests due to a bug in NetCDF-C"
-      echo "         version 4.8.1 and prior. See bug issue"
-      echo "         https://github.com/Unidata/netcdf-c/issues/2251"
-      echo "==================================================================="
-   fi
    APIS+=("netcdf4 canonical")
    export LD_LIBRARY_PATH=${HDF5_LIB_PATH}:${NETCDF4_LIB_PATH}:${LD_LIBRARY_PATH}
    if test "x${ENABLE_LOGVOL}" = x1 ; then
@@ -107,7 +100,6 @@ for API in "${APIS[@]}" ; do
            IN_FILE="${srcdir}/${IN_FILE}.nc"
            OUT_FILE+=".nc"
         elif test "x${ap[0]}" = xnetcdf4 ; then
-           if test "x$#" = x0 ; then :; else continue; fi
            OUT_FILE+=".nc4"
            if test "x${ap[1]}" = xlog ; then
               # This option requires the two VOL environment variables to be set.

tests/icase_def.c

@@ -93,23 +93,31 @@ err = nc_def_dim(ncid, "levdcmp", 15, &dimids[18]); ERR
 err = nc_def_dim(ncid, "levtrc", 10, &dimids[19]); ERR
 err = nc_def_dim(ncid, "hist_interval", 2, &dimids[20]); ERR
 
+#ifdef DIM_SCALE
 err = nc_def_var(ncid, "levgrnd", NC_FLOAT, 1, dimids, &varid); ERR
 err = nc_put_att(ncid, varid, "long_name", NC_CHAR, 22, "coordinate soil levels"); ERR
 err = nc_put_att(ncid, varid, "units", NC_CHAR, 1, "m"); ERR
+#endif
 
+#ifdef DIM_SCALE
 err = nc_def_var(ncid, "levlak", NC_FLOAT, 1, dimids, &varid); ERR
 err = nc_put_att(ncid, varid, "long_name", NC_CHAR, 22, "coordinate lake levels"); ERR
 err = nc_put_att(ncid, varid, "units", NC_CHAR, 1, "m"); ERR
+#endif
 
+#ifdef DIM_SCALE
 err = nc_def_var(ncid, "levdcmp", NC_FLOAT, 1, dimids, &varid); ERR
 err = nc_put_att(ncid, varid, "long_name", NC_CHAR, 22, "coordinate soil levels"); ERR
 err = nc_put_att(ncid, varid, "units", NC_CHAR, 1, "m"); ERR
+#endif
 
+#ifdef DIM_SCALE
 err = nc_def_var(ncid, "time", NC_FLOAT, 1, dimids, &varid); ERR
 err = nc_put_att(ncid, varid, "long_name", NC_CHAR, 4, "time"); ERR
 err = nc_put_att(ncid, varid, "units", NC_CHAR, 30, "days since 0001-01-01 00:00:00"); ERR
 err = nc_put_att(ncid, varid, "calendar", NC_CHAR, 6, "noleap"); ERR
 err = nc_put_att(ncid, varid, "bounds", NC_CHAR, 11, "time_bounds"); ERR
+#endif
 
 err = nc_def_var(ncid, "mcdate", NC_INT, 1, dimids, &varid); ERR
 err = nc_put_att(ncid, varid, "long_name", NC_CHAR, 23, "current date (YYYYMMDD)"); ERR
@@ -134,17 +142,21 @@ err = nc_def_var(ncid, "date_written", NC_CHAR, 2, dimids, &varid); ERR
 
 err = nc_def_var(ncid, "time_written", NC_CHAR, 2, dimids, &varid); ERR
 
+#ifdef DIM_SCALE
 err = nc_def_var(ncid, "lon", NC_FLOAT, 1, dimids, &varid); ERR
 err = nc_put_att(ncid, varid, "long_name", NC_CHAR, 20, "coordinate longitude"); ERR
 err = nc_put_att(ncid, varid, "units", NC_CHAR, 12, "degrees_east"); ERR
 err = nc_put_att(ncid, varid, "_FillValue", NC_FLOAT, 1, buf); ERR
 err = nc_put_att(ncid, varid, "missing_value", NC_FLOAT, 1, buf); ERR
+#endif
 
+#ifdef DIM_SCALE
 err = nc_def_var(ncid, "lat", NC_FLOAT, 1, dimids, &varid); ERR
 err = nc_put_att(ncid, varid, "long_name", NC_CHAR, 19, "coordinate latitude"); ERR
 err = nc_put_att(ncid, varid, "units", NC_CHAR, 13, "degrees_north"); ERR
 err = nc_put_att(ncid, varid, "_FillValue", NC_FLOAT, 1, buf); ERR
 err = nc_put_att(ncid, varid, "missing_value", NC_FLOAT, 1, buf); ERR
+#endif
 
 err = nc_def_var(ncid, "area", NC_FLOAT, 2, dimids, &varid); ERR
 err = nc_put_att(ncid, varid, "long_name", NC_CHAR, 15, "grid cell areas"); ERR