Skip to content

Commit 1ee604a

Browse files
WardFWardF
authored
Merge pull request #2017 from DennisHeimbigner/zarrfillv.dmh
NCZarr is outputting fill value as an array instead of a singleton.
2 parents 9426298 + 6f03935 commit 1ee604a

7 files changed

Lines changed: 118 additions & 105 deletions

File tree

‎RELEASE_NOTES.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
77

88
## 4.8.1 - TBD
99

10+
* [Bug Fix] Store NCZarr fillvalue as a singleton instead of a 1-element array. See [Github #2017](https://github.com/Unidata/netcdf-c/issues/2017).
1011
* [Bug Fixes] The netcdf-c library was incorrectly determining the scope of dimension; similar to the type scope problem. See [Github #2012](https://github.com/Unidata/netcdf-c/pull/2012) for more information.
1112
* [Bug Fix] Re-enable DAP2 authorization testing. See [Github #2011](https://github.com/Unidata/netcdf-c/issues/2011).
1213
* [Bug Fix] Fix bug with windows version of mkstemp that causes failure to create more than 26 temp files. See [Github #1998](https://github.com/Unidata/netcdf-c/pull/1998).

‎libnczarr/zcvt.c‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -324,8 +324,12 @@ NCZ_stringconvert(nc_type typeid, size_t len, void* data0, NCjson** jdatap)
324324
/* Create a string valued json object */
325325
if((stat = NCJnewstringn(NCJ_STRING,len,src,&jdata)))
326326
goto done;
327-
} else { /* for all other values, create an array of values */
328-
if((stat = NCJnew(NCJ_ARRAY,&jdata))) goto done;
327+
} else { /* all other cases */
328+
if(len == 0) {stat = NC_EINVAL; goto done;}
329+
if(len > 1) {
330+
if((stat = NCJnew(NCJ_ARRAY,&jdata))) goto done;
331+
} else /* return a singletone */
332+
jdata = NULL;
329333
for(i=0;i<len;i++) {
330334
char* special = NULL;
331335
double d;
@@ -371,7 +375,10 @@ NCZ_stringconvert(nc_type typeid, size_t len, void* data0, NCjson** jdatap)
371375
if(special) {nullfree(str); str = strdup(special);}
372376
jvalue->value = str;
373377
str = NULL;
374-
nclistpush(jdata->contents,jvalue);
378+
if(len == 1)
379+
jdata = jvalue;
380+
else
381+
nclistpush(jdata->contents,jvalue);
375382
jvalue = NULL;
376383
src += typelen;
377384
}

‎libnczarr/zsync.c‎

Lines changed: 42 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ static int ncz_jsonize_atts(NCindex* attlist, NCjson** jattrsp);
1515
static int load_jatts(NCZMAP* map, NC_OBJ* container, NCjson** jattrsp, NClist** atypes);
1616
static int zconvert(nc_type typeid, size_t typelen, void* dst, NCjson* src);
1717
static int computeattrinfo(const char* name, NClist* atypes, NCjson* values,
18-
nc_type* typeidp, size_t* lenp, void** datap);
18+
nc_type* typeidp, size_t* typelenp, size_t* lenp, void** datap);
1919
static int parse_group_content(NCjson* jcontent, NClist* dimdefs, NClist* varnames, NClist* subgrps);
2020
static int parse_group_content_pure(NCZ_FILE_INFO_T* zinfo, NC_GRP_INFO_T* grp, NClist* varnames, NClist* subgrps);
2121
static int define_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp);
@@ -28,7 +28,7 @@ static int locategroup(NC_FILE_INFO_T* file, size_t nsegs, NClist* segments, NC_
2828
static int createdim(NC_FILE_INFO_T* file, const char* name, size64_t dimlen, NC_DIM_INFO_T** dimp);
2929
static int parsedimrefs(NC_FILE_INFO_T*, NClist* dimnames, size64_t* shape, NC_DIM_INFO_T** dims, int create);
3030
static int decodeints(NCjson* jshape, size64_t* shapes);
31-
static int computeattrdata(nc_type* typeidp, NCjson* values, size_t* lenp, void** datap);
31+
static int computeattrdata(nc_type* typeidp, NCjson* values, size_t* typelenp, size_t* lenp, void** datap);
3232
static int inferattrtype(NCjson* values, nc_type* typeidp);
3333
static int mininttype(unsigned long long u64, int negative);
3434
static int computedimrefs(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int purezarr, int xarray, int ndims, NClist* dimnames, size64_t* shapes, NC_DIM_INFO_T** dims);
@@ -244,6 +244,7 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
244244
NCjson* jncvar = NULL;
245245
NCjson* jdimrefs = NULL;
246246
NCjson* jtmp = NULL;
247+
NCjson* jfill = NULL;
247248
size64_t shape[NC_MAX_VAR_DIMS];
248249
NCZ_VAR_INFO_T* zvar = var->format_var_info;
249250

@@ -325,7 +326,6 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
325326
if(!var->no_fill) {
326327
int fillsort;
327328
int atomictype = var->type_info->hdr.id;
328-
NCjson* jfill = NULL;
329329
/* A scalar value providing the default value to use for uninitialized
330330
portions of the array, or ``null`` if no fill_value is to be used. */
331331
/* Use the defaults defined in netdf.h */
@@ -339,9 +339,17 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
339339
if((stat = nc4_get_default_fill_value(atomictype,var->fill_value))) goto done;
340340
}
341341
/* Convert var->fill_value to a string */
342-
if((stat = NCZ_stringconvert(atomictype,1,var->fill_value,&jfill)))
343-
goto done;
344-
if((stat = NCJinsert(jvar,"fill_value",jfill))) goto done;
342+
if((stat = NCZ_stringconvert(atomictype,1,var->fill_value,&jfill))) goto done;
343+
if(jfill->sort == NCJ_ARRAY) { /* stringconvert should prevent this from happening */
344+
assert(NCJlength(jfill) > 0);
345+
if((stat = NCJarrayith(jfill,0,&jtmp))) goto done; /* use the 0th element */
346+
if((stat = NCJclone(jtmp,&jtmp))) goto done; /* clone so we can free it later */
347+
NCJreclaim(jfill);
348+
jfill = jtmp;
349+
jtmp = NULL;
350+
}
351+
if((stat = NCJinsert(jvar,"fill_value",jfill))) goto done;
352+
jfill = NULL;
345353
}
346354

347355
/* order key */
@@ -463,6 +471,7 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
463471
NCJreclaim(jvar);
464472
NCJreclaim(jncvar);
465473
NCJreclaim(jtmp);
474+
NCJreclaim(jfill);
466475
return THROW(stat);
467476
}
468477

@@ -683,6 +692,8 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist)
683692
/**
684693
@internal Convert a list of attributes to corresponding json.
685694
Note that this does not push to the file.
695+
Also note that attributes of length 1 are stored as singletons, not arrays.
696+
This is to be more consistent with pure zarr.
686697
@param attlist - [in] the attributes to dictify
687698
@param jattrsp - [out] the json'ized att list
688699
@return NC_NOERR
@@ -858,11 +869,11 @@ Extract type and data for an attribute
858869
*/
859870
static int
860871
computeattrinfo(const char* name, NClist* atypes, NCjson* values,
861-
nc_type* typeidp, size_t* lenp, void** datap)
872+
nc_type* typeidp, size_t* typelenp, size_t* lenp, void** datap)
862873
{
863874
int stat = NC_NOERR;
864875
int i;
865-
size_t len;
876+
size_t len, typelen;
866877
void* data;
867878
nc_type typeid;
868879

@@ -880,10 +891,11 @@ computeattrinfo(const char* name, NClist* atypes, NCjson* values,
880891
}
881892
if(typeid >= NC_STRING)
882893
{stat = NC_EINTERNAL; goto done;}
883-
if((stat = computeattrdata(&typeid, values, &len, &data))) goto done;
894+
if((stat = computeattrdata(&typeid, values, &typelen, &len, &data))) goto done;
884895

885896
if(typeidp) *typeidp = typeid;
886897
if(lenp) *lenp = len;
898+
if(typelenp) *typelenp = typelen;
887899
if(datap) {*datap = data; data = NULL;}
888900

889901
done:
@@ -895,7 +907,7 @@ computeattrinfo(const char* name, NClist* atypes, NCjson* values,
895907
Extract data for an attribute
896908
*/
897909
static int
898-
computeattrdata(nc_type* typeidp, NCjson* values, size_t* lenp, void** datap)
910+
computeattrdata(nc_type* typeidp, NCjson* values, size_t* typelenp, size_t* lenp, void** datap)
899911
{
900912
int stat = NC_NOERR;
901913
size_t datalen;
@@ -911,7 +923,7 @@ computeattrdata(nc_type* typeidp, NCjson* values, size_t* lenp, void** datap)
911923

912924
/* Collect the length of the attribute; might be a singleton */
913925
switch (values->sort) {
914-
case NCJ_DICT: stat = NC_EINTERNAL; goto done;
926+
case NCJ_DICT: stat = NC_ENCZARR; goto done;
915927
case NCJ_ARRAY:
916928
datalen = nclistlength(values->contents);
917929
break;
@@ -923,21 +935,22 @@ computeattrdata(nc_type* typeidp, NCjson* values, size_t* lenp, void** datap)
923935
break;
924936
}
925937

926-
/* Allocate data space */
927-
if((stat = NC4_inq_atomic_type(typeid, NULL, &typelen)))
928-
goto done;
929-
if(typeid == NC_CHAR)
930-
data = malloc(typelen*(datalen+1));
931-
else
932-
data = malloc(typelen*datalen);
933-
if(data == NULL)
934-
{stat = NC_ENOMEM; goto done;}
935-
936-
/* convert to target type */
937-
if((stat = zconvert(typeid, typelen, data, values)))
938-
goto done;
939-
938+
if(datalen > 0) {
939+
/* Allocate data space */
940+
if((stat = NC4_inq_atomic_type(typeid, NULL, &typelen)))
941+
goto done;
942+
if(typeid == NC_CHAR)
943+
data = malloc(typelen*(datalen+1));
944+
else
945+
data = malloc(typelen*datalen);
946+
if(data == NULL)
947+
{stat = NC_ENOMEM; goto done;}
948+
/* convert to target type */
949+
if((stat = zconvert(typeid, typelen, data, values)))
950+
goto done;
951+
}
940952
if(lenp) *lenp = datalen;
953+
if(typelenp) *typelenp = typelen;
941954
if(datap) {*datap = data; data = NULL;}
942955
if(typeidp) *typeidp = typeid; /* return possibly inferred type */
943956

@@ -982,7 +995,7 @@ inferattrtype(NCjson* value, nc_type* typeidp)
982995
typeid = NC_CHAR;
983996
break;
984997
default:
985-
return NC_EINTERNAL;
998+
return NC_ENCZARR;
986999
}
9871000
if(typeidp) *typeidp = typeid;
9881001
return NC_NOERR;
@@ -1203,7 +1216,7 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container)
12031216
/* Create the attribute */
12041217
/* Collect the attribute's type and value */
12051218
if((stat = computeattrinfo(key->value,atypes,value,
1206-
&typeid,&len,&data)))
1219+
&typeid,NULL,&len,&data)))
12071220
goto done;
12081221
if((stat = ncz_makeattr(container,attlist,key->value,typeid,len,data,&att)))
12091222
goto done;
@@ -1430,9 +1443,10 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
14301443
if(jvalue == NULL)
14311444
var->no_fill = 1;
14321445
else {
1446+
size_t fvlen;
14331447
typeid = var->type_info->hdr.id;
14341448
var->no_fill = 0;
1435-
if((stat = computeattrdata(&typeid, jvalue, NULL, &var->fill_value)))
1449+
if((stat = computeattrdata(&typeid, jvalue, NULL, &fvlen, &var->fill_value)))
14361450
goto done;
14371451
assert(typeid == var->type_info->hdr.id);
14381452
/* Note that we do not create the _FillValue

‎ncdap_test/pathcvt.c‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,11 @@
2727
/*
2828
Synopsis
2929
30-
pathcvt [-u|-w|-m|-c] [-e] PATH
30+
pathcvt [-u|-w|-m|-c] [-e] [-d <driveletter>] PATH
3131
3232
Options
3333
-e add backslash escapes to '\' and ' '
34+
-d <driveletter> use driveletter when needed; defaults to 'c'
3435
Output type options:
3536
-u convert to Unix form of path
3637
-w convert to Windows form of path
@@ -45,7 +46,8 @@ Default is to convert to the format used by the platform.
4546

4647
struct Options {
4748
int target;
48-
int escape;
49+
int escapes;
50+
int drive;
4951
int debug;
5052
} cvtoptions;
5153

@@ -60,11 +62,13 @@ main(int argc, char** argv)
6062
char* inpath;
6163

6264
memset((void*)&cvtoptions,0,sizeof(cvtoptions));
65+
cvtoptions.drive = 'c';
6366

64-
while ((c = getopt(argc, argv, "cD:ehmuw")) != EOF) {
67+
while ((c = getopt(argc, argv, "cD:d:ehmuw")) != EOF) {
6568
switch(c) {
6669
case 'c': cvtoptions.target = NCPD_CYGWIN; break;
67-
case 'e': cvtoptions.escape = 1; break;
70+
case 'd': cvtoptions.drive = optarg[0]; break;
71+
case 'e': cvtoptions.escapes = 1; break;
6872
case 'h': usage(NULL); break;
6973
case 'm': cvtoptions.target = NCPD_MSYS; break;
7074
case 'u': cvtoptions.target = NCPD_NIX; break;
@@ -90,8 +94,8 @@ main(int argc, char** argv)
9094
if(cvtoptions.target == NCPD_UNKNOWN)
9195
cvtpath = NCpathcvt(inpath);
9296
else
93-
cvtpath = NCpathcvt_test(inpath,cvtoptions.target,'c');
94-
if(cvtpath && cvtoptions.escape) {
97+
cvtpath = NCpathcvt_test(inpath,cvtoptions.target,(char)cvtoptions.drive);
98+
if(cvtpath && cvtoptions.escapes) {
9599
char* path = cvtpath; cvtpath = NULL;
96100
cvtpath = escape(path);
97101
free(path);
@@ -116,20 +120,16 @@ escape(const char* path)
116120
const char* p;
117121
char* q;
118122
char* epath = NULL;
123+
const char* escapes = " \\";
119124

120125
epath = (char*)malloc((2*slen) + 1);
121126
if(epath == NULL) usage("out of memtory");
122127
p = path;
123128
q = epath;
124129
for(;*p;p++) {
125-
switch (*p) {
126-
case '\\': case ' ':
130+
if(strchr(escapes,*p) != NULL)
127131
*q++ = '\\';
128-
/* fall thru */
129-
default:
130-
*q++ = *p;
131-
break;
132-
}
132+
*q++ = *p;
133133
}
134134
*q = '\0';
135135
return epath;

‎ncdap_test/ref_pathcvt.txt‎

Lines changed: 36 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,45 +1,36 @@
1-
path: /xxx/a/b
2-
/xxx/a/b
3-
/cygdrive/c/xxx/a/b
4-
/c/xxx/a/b
5-
c:\\xxx\\a\\b
6-
path: d:/x/y
7-
/d/x/y
8-
/cygdrive/d/x/y
9-
/d/x/y
10-
d:\\x\\y
11-
path: /cygdrive/d/x/y
12-
/d/x/y
13-
/cygdrive/d/x/y
14-
/d/x/y
15-
d:\\x\\y
16-
path: /d/x/y
17-
/d/x/y
18-
/cygdrive/d/x/y
19-
/d/x/y
20-
d:\\x\\y
21-
path: /cygdrive/d
22-
/d
23-
/cygdrive/d
24-
/d
25-
d:
26-
path: /d
27-
/d
28-
/cygdrive/d
29-
/d
30-
d:
31-
path: /cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn
32-
/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn
33-
/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn
34-
/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn
35-
d:\\git\\netcdf-c\\dap4_test\\test_anon_dim.2.syn
36-
path: d:\x\y
37-
/d/x/y
38-
/cygdrive/d/x/y
39-
/d/x/y
40-
d:\\x\\y
41-
path: d:\x\y w\z
42-
/d/x/y w/z
43-
/cygdrive/d/x/y w/z
44-
/d/x/y w/z
45-
d:\\x\\y\\ w\\z
1+
path: -u: |/xxx/x/y| => |/xxx/x/y|
2+
path: -c: |/xxx/x/y| => |/cygdrive/c/xxx/x/y|
3+
path: -m: |/xxx/x/y| => |/c/xxx/x/y|
4+
path: -w: |/xxx/x/y| => |c:\\xxx\\x\\y|
5+
path: -u: |d:/x/y| => |/d/x/y|
6+
path: -c: |d:/x/y| => |/cygdrive/d/x/y|
7+
path: -m: |d:/x/y| => |/d/x/y|
8+
path: -w: |d:/x/y| => |d:\\x\\y|
9+
path: -u: |/cygdrive/d/x/y| => |/d/x/y|
10+
path: -c: |/cygdrive/d/x/y| => |/cygdrive/d/x/y|
11+
path: -m: |/cygdrive/d/x/y| => |/d/x/y|
12+
path: -w: |/cygdrive/d/x/y| => |d:\\x\\y|
13+
path: -u: |/d/x/y| => |/d/x/y|
14+
path: -c: |/d/x/y| => |/cygdrive/d/x/y|
15+
path: -m: |/d/x/y| => |/d/x/y|
16+
path: -w: |/d/x/y| => |d:\\x\\y|
17+
path: -u: |/cygdrive/d| => |/d|
18+
path: -c: |/cygdrive/d| => |/cygdrive/d|
19+
path: -m: |/cygdrive/d| => |/d|
20+
path: -w: |/cygdrive/d| => |d:|
21+
path: -u: |/d| => |/d|
22+
path: -c: |/d| => |/cygdrive/d|
23+
path: -m: |/d| => |/d|
24+
path: -w: |/d| => |d:|
25+
path: -u: |/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn| => |/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn|
26+
path: -c: |/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn| => |/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn|
27+
path: -m: |/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn| => |/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn|
28+
path: -w: |/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn| => |d:\\git\\netcdf-c\\dap4_test\\test_anon_dim.2.syn|
29+
path: -u: |d:\x\y| => |/d/x/y|
30+
path: -c: |d:\x\y| => |/cygdrive/d/x/y|
31+
path: -m: |d:\x\y| => |/d/x/y|
32+
path: -w: |d:\x\y| => |d:\\x\\y|
33+
path: -u: |d:\x\y w\z| => |/d/x/y\ w/z|
34+
path: -c: |d:\x\y w\z| => |/cygdrive/d/x/y\ w/z|
35+
path: -m: |d:\x\y w\z| => |/d/x/y\ w/z|
36+
path: -w: |d:\x\y w\z| => |d:\\x\\y\ w\\z|

‎ncdap_test/testauth.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ LOCALRCFILES="$WD/.dodsrc $WD/.daprc $WD/.ncrc $WD/$NETRC $WD/$NETRCIMP"
4141
HOMERCFILES="$HOME/.dodsrc $HOME/.daprc $HOME/.ncrc $HOME/$NETRC $HOME/$NETRCIMP"
4242
NETRCFILE=$WD/$NETRC
4343
DAPRCFILE=$WD/$RC
44-
if test "x$FP_ISMSVC" = x1 ; then
44+
if test "x$FP_ISMSVC" != x ; then
4545
LOCALRCFILES=`${execdir}/pathcvt "$LOCALRCFILES"`
4646
HOMERCFILES=`${execdir}/pathcvt "$HOMERCFILES"`
4747
NETRCFILE=`${execdir}/pathcvt "$NETRCFILE"`

0 commit comments

Comments
 (0)
⚡