Merge pull request #1493 from DennisHeimbigner/newinfer.dmh

Change (again), and simplify, the file model inference algorithm

Author: WardF

debug/nc_test/Make0

@@ -1,57 +1,52 @@
 # Test c output
-T=tst_diskless5
+T=tst_vars3
 
-#H58=8
-H510=10
+#SRC=hdf5plugins/H5Zmisc.c
 
-#ARGS=diskless persist
+#CMD=valgrind --leak-check=full
+CMD=gdb --args
 
-#SRC=
-
-#CMD=env HDF5_DEBUG=trace
-#CMD=export NETCDF_LOG_LEVEL=5 ;gdb --args
-CMD=valgrind --leak-check=full
-#CMD=gdb --args
+#FILTER=H5Zmisc
+#FILTEROBJ=hdf5plugins/${FILTER}.o
 
 #PAR=1
+#SZIP=1
 
-ifdef H58
-H5L=/usr/local
-endif
-ifdef H510
-H5L=/opt
-endif
+#CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include
+CFLAGS = -Wall -g -O0 -I.. -I../include
 
-CFLAGS=-Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include
+LDFLAGS = ../liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -ldl -lcurl -lm -lmfhdf -ldf
 
 ifdef PAR
 CC=mpicc
-#CC=/usr/local/bin/mpicc
-LDFLAGS=../liblib/.libs/libnetcdf.a -L${H5L}/lib -lhdf5_hl -lhdf5 -lz  -ldl -lcurl -lpnetcdf -lmpich -lm
+LDFLAGS += -lmpich
 else
 CC=gcc
-#LDFLAGS=../liblib/.libs/libnetcdf.a  -L${H5L}/lib -lhdf5_hl -lhdf5 -lz -lm -lcurl
-LDFLAGS=../liblib/.libs/libnetcdf.a -L${H5L}/lib -lhdf5_hl -lhdf5 -lz  -ldl -lm -lcurl
 endif
 
-#	cd .. ; ${MAKE} all
+ifdef SZIP
+LDFLAGS += -lsz -laec
+endif
 
-LLP=${H5L}/lib:${LD_LIBRARY_PATH}
+LLP=/usr/local/lib:${LD_LIBRARY_PATH}
 
 all:: cmp
 	export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
 	${CMD} ./t ${ARGS}
 
 cmp::
 	export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
-	${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \
+	${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${FILTEROBJ} ${LDFLAGS}
+
+filter::
+	${CC} ${CFLAGS} -c hdf5plugins/${FILTER}.c ${LDFLAGS}
 
 cpp::
 	${CC} -E ${CFLAGS} ${T}.c > ${T}.txt
 
-#TS = tst_diskless tst_diskless2 tst_diskless3 tst_diskless4 tst_diskless5 tst_diskless6
-TS = tst_diskless5
-several::
+H5=h5testszip
+EXT=testszip.nc
+h5::
 	export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
-	for f in ${TS} ; do ${CC} -o ${TS} ${CFLAGS} ${TS}.c ${SRC} ${LDFLAGS}; done
-
+	${CC} -o h5 ${CFLAGS} ${H5}.c ${SRC} ${LDFLAGS}; \
+	${CMD} ./h5 ${EXT}

docs/byterange.dox

@@ -20,15 +20,15 @@ the remote server supports byte-range access.
 Two examples:
 
 1. An Amazon S3 object containing a netcdf classic file.
-   - location: "http://149.165.169.123:8080/thredds/fileServer/testdata/2004050300_eta_211.nc#bytes" 
+   - location: "http://149.165.169.123:8080/thredds/fileServer/testdata/2004050300_eta_211.nc#mode=bytes" 
 2. A Thredds Server dataset supporting the Thredds HTTPServer protocol.
    and containing a netcdf enhanced file.
-   - location: "http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#bytes"
+   - location: "http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"
 
 Other remote servers may also provide byte-range access in a similar form.
 
 It is important to note that this is not intended as a true
-production capability because, as is known, this kind of access
+production capability because it is believed that this kind of access
 can be quite slow. In addition, the byte-range IO drivers do not
 currently do any sort of optimization or caching.
 
@@ -42,15 +42,15 @@ This capability requires access to *libcurl*, and an error will occur
 if byterange is enabled, but no *libcurl* could not be located.
 In this, it is similar to the DAP2 and DAP4 capabilities.
 
-Note also that the term "http" is often used as a synonym for *byterange*.
+Note also that here, the term "http" is often used as a synonym for *byterange*.
 
 # Run-time Usage {#byterange_url}
 
 In order to use this capability at run-time, with *ncdump* for
 example, it is necessary to provide a URL pointing to the basic
 dataset to be accessed. The URL must be annotated to tell the
 netcdf-c library that byte-range access should be used. This is
-indicated by appending the phrase ````#bytes````
+indicated by appending the phrase ````#mode=bytes````
 to the end of the URL.
 The two examples above show how this will look.
 

docs/internal.dox

@@ -494,39 +494,45 @@ _libdispatch/dinfermodel.c_ via the API in _include/ncmodel.h_.
 The term _model_ is used here to include (at least) the following
 information (see the structure type _NCmodel_ in _include/ncmodel.h_).
 
-1. format -- this is an NC_FORMAT_XXX value defining the file format
-   as seen by the user program.
-2. version -- the specific version of the format.
-3. iosp -- this is and NC_IOSP_XXX value describing internal protocols to use.
-4. impl -- this is an NC_FORMATX_XXX value defining, in effect, the
+1. impl -- this is an NC_FORMATX_XXX value defining, in effect, the
    dispatch table to use.
 
-For example, if the format was NC_FORMAT_CLASSIC, then the client
-will see the netcdf-3 data model, as modified by the version. If the
-version was 5, for example, then that indicates the file format
-is actually NC_FORMAT_64BIT_DATA, which is a variant of the netcdf-3
-format.
-
-The _iosp_ provides information about how the protocol the
-dispatch table will use to access the actual dataset. If the iosp
-is NC_IOSP_S3RAW, then it indicates that the dispatcher, NC_FORMATX_NC3,
-for example, will access the dataset using the Amazon S3 REST API.
-
 The construction of the model is primarily carried out by the function
-_NC_infermodel()_. It is given the following parameters:
+_NC_infermodel()_ (in _libdispatch/dinfermodel.c_).
+It is given the following parameters:
 1. path -- (IN) absolute file path or URL
-2. omodep -- (IN/OUT) the set of mode flags given to _NC_open_ or _NC_create_.
+2. modep -- (IN/OUT) the set of mode flags given to _NC_open_ or _NC_create_.
 3. iscreate -- (IN) distinguish open from create.
 4. useparallel -- (IN) indicate if parallel IO can be used.
 5. params -- (IN/OUT) arbitrary data dependent on the mode and path.
-6. model -- (IN/OUT) place to store inferred model information (e.g. format
-   or version).
-7. newpathp -- (OUT) sometimes, it is necessary to rewrite the path.
+6. model -- (IN/OUT) place to store inferred model.
+7. newpathp -- (OUT) the canonical rewrite of the path argument.
 
 As a rule, these values are used in the this order to infer the model.
 1. file contents -- highest precedence
-2. url (if it is one) -- using the protocol and fragment arguments
+2. url (if it is one) -- using the "mode=" key in the fragment (see below).
 3. mode flags
 4. default format -- lowest precedence
 
+If the path appears to be a URL, then it is parsed.
+Information is extracted from the URL, and specifically,
+the fragment key "mode=" is the critical element.
+The URL will be rewritten to a canonical form with the following
+changes.
+1. The fragment part ("#..." at the end) is parsed and the "mode=" key
+   is extracted and its value is converted to a list of tags.
+2. If the leading protocol is not http/https, then the protocol is added
+   to the mode list. That protocol is then replaced with either http or https.
+3. Certain singleton values inb the fragment are extracted and removed
+   and added to the mode list. Consider, for example, "http://....#dap4".
+   The "dap4" singleton is removed and added to the mode list.
+4. For backward compatibility, the values of "proto=" and "protocol="
+   are removed from the fragment and their value is added to the mode list.
+5. The final mode list is converted to a comma separated string
+   and re-inserted into the fragment.
+6. The final mode list is modified to remove duplicates.
+
+The final result is the canonical form of the URL and is returned in the
+newpathp argument described above.
+
 */

include/nc.h

@@ -8,9 +8,6 @@
 #include "config.h"
 #include "netcdf.h"
 
-/* Forward */
-struct NCmodel;
-
    /* There's an external ncid (ext_ncid) and an internal ncid
     * (int_ncid). The ext_ncid is the ncid returned to the user. If
     * the user has opened or created a netcdf-4 file, then the
@@ -30,7 +27,6 @@ typedef struct NC {
 	void* dispatchdata; /*per-'file' data; points to e.g. NC3_INFO data*/
 	char* path;
 	int   mode; /* as provided to nc_open/nc_create */
-        struct NCmodel*  model; /* as determined by libdispatch/dfile.c */
 } NC;
 
 /*
@@ -79,6 +75,6 @@ extern int iterate_NCList(int i,NC**); /* Walk from 0 ...; ERANGE return => stop
 
 /* Defined in nc.c */
 extern void free_NC(NC*);
-extern int new_NC(const struct NC_Dispatch*, const char*, int, struct NCmodel*, NC**);
+extern int new_NC(const struct NC_Dispatch*, const char*, int, NC**);
 
 #endif /* _NC_H_ */

include/ncmodel.h

@@ -9,39 +9,20 @@
 #ifndef NCINFERMODEL_H
 #define NCINFERMODEL_H
 
-/* Define the io handler to be used to do lowest level
-   access. This is above the libcurl level and below the
-   dispatcher level. This is only used for remote
-   datasets or for implementations where the implementation
-   multiplexes more than one IOSP in a single dispatcher.
-*/
-#define NC_IOSP_FILE	(1)
-#define NC_IOSP_MEMORY	(2)
-#define NC_IOSP_DAP2	(3)
-#define NC_IOSP_DAP4	(4)
-#define NC_IOSP_UDF	(5) /*Placeholder since we do not know IOSP for UDF*/
-#define NC_IOSP_HTTP	(6)
-
 /* Track the information hat will help us
    infer how to access the data defined by
-   path + omode.
+   path + omode + (sometimes) file content.
 */
 typedef struct NCmodel {
-    int format; /* NC_FORMAT_XXX value */
     int impl; /* NC_FORMATX_XXX value */
-    int iosp; /* NC_IOSP_XXX value (above) */
+    int format; /* NC_FORMAT_XXX value; Used to remember extra info; */
 } NCmodel;
 
 /* Keep compiler quiet */
 struct NCURI;
 struct NC_dispatch;
 
-#if 0
-/* return first IOSP or NULL if none */
-EXTERNL int NC_urliosp(struct NCURI* u);
-#endif
-
-/* Infer model format and implementation */
+/* Infer model implementation */
 EXTERNL int NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void* params, NCmodel* model, char** newpathp);
 
 /**
@@ -53,15 +34,4 @@ EXTERNL int NC_infermodel(const char* path, int* omodep, int iscreate, int usepa
  */
 EXTERNL int nc__testurl(const char* path, char** basenamep);
 
-#if 0
-/* allow access url parse and params without exposing nc_url.h */
-EXTERNL int NCDAP_urlparse(const char* s, void** dapurl);
-EXTERNL void NCDAP_urlfree(void* dapurl);
-EXTERNL const char* NCDAP_urllookup(void* dapurl, const char* param);
-
-/* Ping a specific server */
-EXTERNL int NCDAP2_ping(const char*);
-EXTERNL int NCDAP4_ping(const char*);
-#endif
-
 #endif /*NCINFERMODEL_H*/

include/ncrc.h

@@ -61,5 +61,6 @@ extern char* NC_entityescape(const char* s);
 extern int NC_readfile(const char* filename, NCbytes* content);
 extern int NC_writefile(const char* filename, size_t size, void* content);
 extern char* NC_mktmp(const char* base);
-
+extern int NC_getmodelist(const char* url, NClist** modelistp);
+extern int NC_testmode(const char* path, const char* tag);
 #endif /*NCRC_H*/

include/ncuri.h

@@ -6,19 +6,6 @@
 #ifndef NCURI_H
 #define NCURI_H
 
-/* Define error codes */
-#define NCU_OK (0)
-#define NCU_EINVAL        (1) /* Generic, mostly means bad argument */
-#define NCU_EBADURL       (2)
-#define NCU_ENOMEM        (3)
-#define NCU_EPROTO        (4)
-#define NCU_EPATH         (5)
-#define NCU_EUSRPWD       (6)
-#define NCU_EHOST         (7)
-#define NCU_EPORT         (8)
-#define NCU_EPARAMS       (9)
-#define NCU_ENOPARAM      (10)
-#define NCU_ECONSTRAINTS  (11)
 
 /* Define flags to control what is included by ncuribuild*/
 #define NCURIPATH      1
@@ -79,6 +66,9 @@ extern int ncurisetprotocol(NCURI*,const char* newprotocol);
 /* Replace the constraints */
 EXTERNL int ncurisetquery(NCURI*,const char* query);
 
+/* Replace the fragment list */
+extern int ncurisetfragments(NCURI*, const char* fragments);
+
 /* Construct a complete NC URI; caller frees returned string */
 EXTERNL char* ncuribuild(NCURI*,const char* prefix, const char* suffix, int flags);
 

libdap2/ncd2dispatch.c

@@ -322,7 +322,7 @@ NCD2_open(const char* path, int mode, int basepe, size_t *chunksizehintp,
     dapcomm->oc.rawurltext = strdup(path);
 #endif
 
-    if(ncuriparse(dapcomm->oc.rawurltext,&dapcomm->oc.url) != NCU_OK)
+    if(ncuriparse(dapcomm->oc.rawurltext,&dapcomm->oc.url))
 	{ncstat = NC_EURL; goto done;}
 
     if(!constrainable(dapcomm->oc.url))

libdap4/d4file.c

@@ -65,7 +65,7 @@ NCD4_open(const char * path, int mode,
     d4info->controller = (NC*)nc;
 
     /* Parse url and params */
-    if(ncuriparse(nc->path,&d4info->uri) != NCU_OK)
+    if(ncuriparse(nc->path,&d4info->uri))
 	{ret = NC_EDAPURL; goto done;}
 
     /* Load auth info from rc file */

libdap4/d4util.c

@@ -37,7 +37,7 @@ ncd4__testurl(const char* path, char** basenamep)
 {
     NCURI* uri;
     int ok = NC_NOERR;
-    if(ncuriparse(path,&uri) != NCU_OK)
+    if(ncuriparse(path,&uri))
 	ok = NC_EURL;
     else {
 	char* slash = (uri->path == NULL ? NULL : strrchr(uri->path, '/'));