Merge branch 'csz_bitround' of https://github.com/nco/netcdf-c into gh2232.wif

Author: WardF

include/netcdf.h

@@ -331,19 +331,23 @@ there. */
 #define NC_NOQUANTIZE 0 /**< No quantization in use. */    
 #define NC_QUANTIZE_BITGROOM 1 /**< Use BitGroom quantization. */
 #define NC_QUANTIZE_GRANULARBR 2 /**< Use Granular BitRound quantization. */
+#define NC_QUANTIZE_BITROUND 3 /**< Use BitRound quantization. */
 
 /** When quantization is used for a variable, an attribute of the
  * appropriate name is added. */
-#define NC_QUANTIZE_BITGROOM_ATT_NAME "_QuantizeBitgroomNumberOfSignificantDigits"
+#define NC_QUANTIZE_BITGROOM_ATT_NAME "_QuantizeBitGroomNumberOfSignificantDigits"
 #define NC_QUANTIZE_GRANULARBR_ATT_NAME "_QuantizeGranularBitRoundNumberOfSignificantDigits"
+#define NC_QUANTIZE_BITROUND_ATT_NAME "_QuantizeBitRoundNumberOfSignificantBits"
 
 /** For quantization, the allowed value of number of significant
- * digits for float. */
+ * decimal and binary digits, respectively, for float. */
 #define NC_QUANTIZE_MAX_FLOAT_NSD (7)
+#define NC_QUANTIZE_MAX_FLOAT_NSB (23)
 
 /** For quantization, the allowed value of number of significant
- * digits for double. */
+ * decimal and binary digits, respectively, for double. */
 #define NC_QUANTIZE_MAX_DOUBLE_NSD (15)
+#define NC_QUANTIZE_MAX_DOUBLE_NSB (52)
 
 /** The netcdf version 3 functions all return integer error status.
  * These are the possible values, in addition to certain values from

libdispatch/dvar.c

@@ -469,7 +469,7 @@ nc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_le
   
    The data are quantized by setting unneeded bits to zeros or ones
    so that they may compress well. BitGroom sets bits alternately to 1/0, 
-   while Granular BitRound (GBR) sets (more) bits to zeros.
+   while BitRound and Granular BitRound (GBR) round (more) bits to zeros
    Quantization is lossy (data are irretrievably altered), and it 
    improves the compression ratio provided by a subsequent lossless 
    compression filter. Quantization alone will not reduce the data size.
@@ -479,18 +479,17 @@ nc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_le
    compression will result in significant improvent in the final data
    size.
 
-   This data quantization used the BitGroom algorithm. A notable
-   feature of BitGroom is that the data it processes remain in IEEE754
-   format after quantization. Therefore the BitGroom algorithm does
+   A notable feature of all the quantization algorithms is data remain 
+   in IEEE754 format afterwards. Therefore quantization algorithms do
    nothing when data are read.
   
    Quantization is only available for variables of type NC_FLOAT or
    NC_DOUBLE. Attempts to set quantization for other variable
    types return an error (NC_EINVAL). 
 
    Variables that use quantize will have added an attribute with name
-   ::NC_QUANTIZE_ATT_NAME, which will contain the number of
-   significant digits. Users should not delete or change this
+   ::NC_QUANTIZE_[ALGORITHM_NAME]_ATT_NAME, which will contain the 
+   number of significant digits. Users should not delete or change this
    attribute. This is the only record that quantize has been applied
    to the data.
 
@@ -529,11 +528,16 @@ nc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_le
    @param ncid File ID.
    @param varid Variable ID. ::NC_GLOBAL may not be used.
    @param quantize_mode Quantization mode. May be ::NC_NOQUANTIZE or
-   ::NC_QUANTIZE_BITGROOM or ::NC_QUANTIZE_GRANULARBR.
-   @param nsd Number of significant digits. May be any integer from 1
-   to ::NC_QUANTIZE_MAX_FLOAT_NSD (for variables of type ::NC_FLOAT)
-   or ::NC_QUANTIZE_MAX_DOUBLE_NSD (for variables of type
-   ::NC_DOUBLE). Ignored if quantize_mode = NC_NOQUANTIZE.
+   ::NC_QUANTIZE_BITGROOM or ::NC_QUANTIZE_GRANULARBR or
+   ::NC_QUANTIZE_BITROUND.
+   @param nsd Number of significant digits (either decimal or binary). 
+   May be any integer from 1 to ::NC_QUANTIZE_MAX_FLOAT_NSD (for variables 
+   of type ::NC_FLOAT) or ::NC_QUANTIZE_MAX_DOUBLE_NSD (for variables 
+   of type ::NC_DOUBLE) for mode ::NC_QUANTIZE_BITGROOM and mode
+   ::NC_QUANTIZE_GRANULARBR. May be any integer from 1 to 
+   ::NC_QUANTIZE_MAX_FLOAT_NSB (for variables of type ::NC_FLOAT) or 
+   ::NC_QUANTIZE_MAX_DOUBLE_NSB (for variables of type ::NC_DOUBLE) 
+   for mode ::NC_QUANTIZE_BITROUND. Ignored if quantize_mode = NC_NOQUANTIZE.
    
    @return ::NC_NOERR No error.
    @return ::NC_EGLOBAL Can't use ::NC_GLOBAL with this function.

libhdf5/hdf5open.c

@@ -1218,7 +1218,16 @@ static int get_quantize_info(NC_VAR_INFO_T *var)
 	attid = H5Aopen_by_name(datasetid, ".", NC_QUANTIZE_GRANULARBR_ATT_NAME,
 			    H5P_DEFAULT, H5P_DEFAULT);
 	if (attid > 0)
+	  {
 	    var->quantize_mode = NC_QUANTIZE_GRANULARBR;
+	  }
+	else
+	  {
+	    attid = H5Aopen_by_name(datasetid, ".", NC_QUANTIZE_BITROUND_ATT_NAME,
+				    H5P_DEFAULT, H5P_DEFAULT);
+	    if (attid > 0)
+	      var->quantize_mode = NC_QUANTIZE_BITROUND;
+	  }
       }
 
     /* If there is an attribute, read it for the nsd. */
@@ -2307,7 +2316,7 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name)
  * for both global and variable attributes.
  *
  * @param loc_id HDF5 attribute ID.
- * @param att_name Name of the attrigute.
+ * @param att_name Name of the attribute.
  * @param ainfo HDF5 info struct for attribute.
  * @param att_data Pointer to an att_iter_info struct, which contains
  * pointers to the NC_GRP_INFO_T and (for variable attributes) the

libhdf5/hdf5var.c

@@ -715,36 +715,54 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
     }
 
     /* Remember quantization settings. They will be used when data are
-     * written. */
+     * written.
+     * Code block is identical to one in zvar.c---consider functionalizing */
     if (quantize_mode)
     {
-	/* Only two valid mode settings. */
+	/* Only four valid mode settings. */
 	if (*quantize_mode != NC_NOQUANTIZE &&
 	    *quantize_mode != NC_QUANTIZE_BITGROOM &&
-	    *quantize_mode != NC_QUANTIZE_GRANULARBR)
+	    *quantize_mode != NC_QUANTIZE_GRANULARBR &&
+	    *quantize_mode != NC_QUANTIZE_BITROUND)
 	    return NC_EINVAL;
 
-	if (*quantize_mode == NC_QUANTIZE_BITGROOM || *quantize_mode == NC_QUANTIZE_GRANULARBR)
+	if (*quantize_mode == NC_QUANTIZE_BITGROOM ||
+	    *quantize_mode == NC_QUANTIZE_GRANULARBR ||
+	    *quantize_mode == NC_QUANTIZE_BITROUND)
 	{
 	    /* Only float and double types can have quantization. */
 	    if (var->type_info->hdr.id != NC_FLOAT &&
 		var->type_info->hdr.id != NC_DOUBLE)
 		return NC_EINVAL;
 
-	    /* For bitgroom, number of significant digits is required. */
+	    
+	    /* All quantization codecs require number of significant digits */
 	    if (!nsd)
 		return NC_EINVAL;
 
 	    /* NSD must be in range. */
 	    if (*nsd <= 0)
 		return NC_EINVAL;
-	    if (var->type_info->hdr.id == NC_FLOAT &&
-		*nsd > NC_QUANTIZE_MAX_FLOAT_NSD)
-		return NC_EINVAL;
-	    if (var->type_info->hdr.id == NC_DOUBLE &&
-		*nsd > NC_QUANTIZE_MAX_DOUBLE_NSD)
-		return NC_EINVAL;
 
+	    if (*quantize_mode == NC_QUANTIZE_BITGROOM ||
+		*quantize_mode == NC_QUANTIZE_GRANULARBR)
+	      { 
+		if (var->type_info->hdr.id == NC_FLOAT &&
+		    *nsd > NC_QUANTIZE_MAX_FLOAT_NSD)
+		  return NC_EINVAL;
+		if (var->type_info->hdr.id == NC_DOUBLE &&
+		    *nsd > NC_QUANTIZE_MAX_DOUBLE_NSD)
+		  return NC_EINVAL;
+	      }
+	    else if (*quantize_mode == NC_QUANTIZE_BITROUND)
+	      {
+		if (var->type_info->hdr.id == NC_FLOAT &&
+		    *nsd > NC_QUANTIZE_MAX_FLOAT_NSB)
+		  return NC_EINVAL;
+		if (var->type_info->hdr.id == NC_DOUBLE &&
+		    *nsd > NC_QUANTIZE_MAX_DOUBLE_NSB)
+		  return NC_EINVAL;
+	      }
 	    var->nsd = *nsd;
 	}
 
@@ -812,12 +830,25 @@ NC4_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
  * error.)
  *
  * When quantize is turned on, and the number of significant digits
- * has been specified, then the netCDF library will quantize according
- * to the selected algorithm. BitGroom will apply all zeros or
- * all ones (alternating) to bits which are not needed to specify the
- * value to the number of significant digits. GranularBR will zero
- * more bits than BG, and thus be more compressible and less accurate.
- * Both will change the value of the data, and will make it more compressible.
+ * (NSD) has been specified, then the netCDF library will quantize according
+ * to the selected algorithm. BitGroom interprets NSD as decimal digits
+ * will apply all zeros or all ones (alternating) to bits which are not 
+ * needed to specify the value to the number of significant decimal digits. 
+ * BitGroom retain the same number of bits for all values of a variable. 
+ * BitRound (BR) interprets NSD as binary digits (i.e., bits) and keeps the
+ * the user-specified number of significant bits then rounds the result
+ * to the nearest representable number according to IEEE rounding rules.
+ * BG and BR both retain a uniform number of significant bits for all 
+ * values of a variable. Granular BitRound interprest NSD as decimal
+ * digits. GranularBR determines the number of bits to necessary to 
+ * retain the user-specified number of significant digits individually
+ * for every value of the variable. GranularBR then applies the BR
+ * quantization algorithm on a granular, value-by-value, rather than
+ * uniformly for the entire variable. GranularBR quantizes more bits
+ * than BG, and is thus more compressive and less accurate than BG.
+ * BR knows bits and makes no guarantees about decimal precision.
+ * All quantization algorithms change the values of the data, and make 
+ * it more compressible.
  *
  * Quantizing the data does not reduce the size of the data on disk,
  * but combining quantize with compression will allow for better
@@ -829,10 +860,10 @@ NC4_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
  * size.
  *
  * Variables which use quantize will have added an attribute with name
- * ::NC_QUANTIZE_[ALG_NAME]_ATT_NAME, which will contain the number of
- * significant digits. Users should not delete or change this
- * attribute. This is the only record that quantize has been applied
- * to the data.
+ * ::NC_QUANTIZE_BITGROOM_ATT_NAME, ::NC_QUANTIZE_GRANULARBR_ATT_NAME, 
+ * or ::NC_QUANTIZE_BITROUND_ATT_NAME that contains the number of 
+ * significant digits. Users should not delete or change this attribute. 
+ * This is the only record that quantize has been applied to the data.
  *
  * As with the deflate settings, quantize settings may only be
  * modified before the first call to nc_enddef(). Once nc_enddef() is
@@ -847,10 +878,15 @@ NC4_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
  * @param ncid File ID.
  * @param varid Variable ID. NC_GLOBAL may not be used.
  * @param quantize_mode Quantization mode. May be ::NC_NOQUANTIZE or
- * ::NC_QUANTIZE_BITGROOM or ::NC_QUANTIZE_GRANULARBR.
- * @param nsd Number of significant digits. May be any integer from 1
- * to ::NC_QUANTIZE_MAX_FLOAT_NSD (for variables of type ::NC_FLOAT) or
- * ::NC_QUANTIZE_MAX_DOUBLE_NSD (for variables of type ::NC_DOUBLE).
+ * ::NC_QUANTIZE_BITGROOM, ::NC_QUANTIZE_BITROUND or ::NC_QUANTIZE_GRANULARBR.
+ * @param nsd Number of significant digits (either decimal or binary). 
+ * May be any integer from 1 to ::NC_QUANTIZE_MAX_FLOAT_NSD (for variables 
+ * of type ::NC_FLOAT) or ::NC_QUANTIZE_MAX_DOUBLE_NSD (for variables 
+ * of type ::NC_DOUBLE) for mode ::NC_QUANTIZE_BITGROOM and mode
+ * ::NC_QUANTIZE_GRANULARBR. May be any integer from 1 to 
+ * ::NC_QUANTIZE_MAX_FLOAT_NSB (for variables of type ::NC_FLOAT) or 
+ * ::NC_QUANTIZE_MAX_DOUBLE_NSB (for variables of type ::NC_DOUBLE) 
+ * for mode ::NC_QUANTIZE_BITROUND.
  *
  * @returns ::NC_NOERR No error.
  * @returns ::NC_EBADID Bad ncid.

libhdf5/nc4hdf.c

@@ -1012,7 +1012,9 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
     }
 
     /* If quantization is in use, write an attribute indicating it, a
-     * single integer which is the number of significant digits. */
+     * single integer which is the number of significant digits 
+     * (NSD, for BitGroom and Granular BitRound) or number of significant bits
+     * (NSB, for BitRound). */
     if (var->quantize_mode == NC_QUANTIZE_BITGROOM)
 	if ((retval = nc4_put_att(var->container, var->hdr.id, NC_QUANTIZE_BITGROOM_ATT_NAME, NC_INT, 1,
 				  &var->nsd, NC_INT, 0)))
@@ -1023,6 +1025,11 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
 				  &var->nsd, NC_INT, 0)))
 	    BAIL(retval);
 
+    if (var->quantize_mode == NC_QUANTIZE_BITROUND)
+	if ((retval = nc4_put_att(var->container, var->hdr.id, NC_QUANTIZE_BITROUND_ATT_NAME, NC_INT, 1,
+				  &var->nsd, NC_INT, 0)))
+	    BAIL(retval);
+
     /* Write attributes for this var. */
     if ((retval = write_attlist(var->att, var->hdr.id, grp)))
         BAIL(retval);

libnczarr/zsync.c

@@ -727,6 +727,10 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, int isc
 	    if((stat = NCJinsert(jatts,NC_QUANTIZE_GRANULARBR_ATT_NAME,jint))) goto done;	
 	    jint = NULL;
 	    break;
+	case NC_QUANTIZE_BITROUND:
+	    if((stat = NCJinsert(jatts,NC_QUANTIZE_BITROUND_ATT_NAME,jint))) goto done;	
+	    jint = NULL;
+	    break;
 	default: break;
 	}
     }