Change (again), and hopefully simplify, the file model inference algorithm.

* For URL paths, the new approach essentially centralizes all information
  in the URL into the "#mode=" fragment key and uses that value
  to determine the dispatcher for (most) URLs.

* The new approach has the following steps:

  1. canonicalize the path if it is a URL.
  2. use the mode= fragment key to determine the dispatcher
  3. if dispatcher still not determined, then use the mode flags
     argument to nc_open/nc_create to determine the dispatcher.
  4. if the path points to something readable, attempt to read the
     magic number at the front, and use that to determine the dispatcher.
     this case may override all previous cases.

* Misc changes.

  1. Update documentation
  2. Moved some unit tests from libdispatch to unit_test directory.
  3. Fixed use of wrong #ifdef macro in test_filter_reg.c
     [I think this may fix an previously reported esupport query].

Author: DennisHeimbigner

debug/nc_test/Make0

@@ -1,57 +1,52 @@
 # Test c output
-T=tst_diskless5
+T=tst_vars3
 
-#H58=8
-H510=10
+#SRC=hdf5plugins/H5Zmisc.c
 
-#ARGS=diskless persist
+#CMD=valgrind --leak-check=full
+CMD=gdb --args
 
-#SRC=
-
-#CMD=env HDF5_DEBUG=trace
-#CMD=export NETCDF_LOG_LEVEL=5 ;gdb --args
-CMD=valgrind --leak-check=full
-#CMD=gdb --args
+#FILTER=H5Zmisc
+#FILTEROBJ=hdf5plugins/${FILTER}.o
 
 #PAR=1
+#SZIP=1
 
-ifdef H58
-H5L=/usr/local
-endif
-ifdef H510
-H5L=/opt
-endif
+#CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include
+CFLAGS = -Wall -g -O0 -I.. -I../include
 
-CFLAGS=-Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include
+LDFLAGS = ../liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -ldl -lcurl -lm -lmfhdf -ldf
 
 ifdef PAR
 CC=mpicc
-#CC=/usr/local/bin/mpicc
-LDFLAGS=../liblib/.libs/libnetcdf.a -L${H5L}/lib -lhdf5_hl -lhdf5 -lz  -ldl -lcurl -lpnetcdf -lmpich -lm
+LDFLAGS += -lmpich
 else
 CC=gcc
-#LDFLAGS=../liblib/.libs/libnetcdf.a  -L${H5L}/lib -lhdf5_hl -lhdf5 -lz -lm -lcurl
-LDFLAGS=../liblib/.libs/libnetcdf.a -L${H5L}/lib -lhdf5_hl -lhdf5 -lz  -ldl -lm -lcurl
 endif
 
-#	cd .. ; ${MAKE} all
+ifdef SZIP
+LDFLAGS += -lsz -laec
+endif
 
-LLP=${H5L}/lib:${LD_LIBRARY_PATH}
+LLP=/usr/local/lib:${LD_LIBRARY_PATH}
 
 all:: cmp
 	export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
 	${CMD} ./t ${ARGS}
 
 cmp::
 	export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
-	${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \
+	${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${FILTEROBJ} ${LDFLAGS}
+
+filter::
+	${CC} ${CFLAGS} -c hdf5plugins/${FILTER}.c ${LDFLAGS}
 
 cpp::
 	${CC} -E ${CFLAGS} ${T}.c > ${T}.txt
 
-#TS = tst_diskless tst_diskless2 tst_diskless3 tst_diskless4 tst_diskless5 tst_diskless6
-TS = tst_diskless5
-several::
+H5=h5testszip
+EXT=testszip.nc
+h5::
 	export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
-	for f in ${TS} ; do ${CC} -o ${TS} ${CFLAGS} ${TS}.c ${SRC} ${LDFLAGS}; done
-
+	${CC} -o h5 ${CFLAGS} ${H5}.c ${SRC} ${LDFLAGS}; \
+	${CMD} ./h5 ${EXT}

docs/byterange.dox

@@ -20,15 +20,15 @@ the remote server supports byte-range access.
 Two examples:
 
 1. An Amazon S3 object containing a netcdf classic file.
-   - location: "http://149.165.169.123:8080/thredds/fileServer/testdata/2004050300_eta_211.nc#bytes" 
+   - location: "http://149.165.169.123:8080/thredds/fileServer/testdata/2004050300_eta_211.nc#mode=bytes" 
 2. A Thredds Server dataset supporting the Thredds HTTPServer protocol.
    and containing a netcdf enhanced file.
-   - location: "http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#bytes"
+   - location: "http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"
 
 Other remote servers may also provide byte-range access in a similar form.
 
 It is important to note that this is not intended as a true
-production capability because, as is known, this kind of access
+production capability because it is believed that this kind of access
 can be quite slow. In addition, the byte-range IO drivers do not
 currently do any sort of optimization or caching.
 
@@ -42,15 +42,15 @@ This capability requires access to *libcurl*, and an error will occur
 if byterange is enabled, but no *libcurl* could not be located.
 In this, it is similar to the DAP2 and DAP4 capabilities.
 
-Note also that the term "http" is often used as a synonym for *byterange*.
+Note also that here, the term "http" is often used as a synonym for *byterange*.
 
 # Run-time Usage {#byterange_url}
 
 In order to use this capability at run-time, with *ncdump* for
 example, it is necessary to provide a URL pointing to the basic
 dataset to be accessed. The URL must be annotated to tell the
 netcdf-c library that byte-range access should be used. This is
-indicated by appending the phrase ````#bytes````
+indicated by appending the phrase ````#mode=bytes````
 to the end of the URL.
 The two examples above show how this will look.
 

docs/internal.dox

@@ -494,39 +494,45 @@ _libdispatch/dinfermodel.c_ via the API in _include/ncmodel.h_.
 The term _model_ is used here to include (at least) the following
 information (see the structure type _NCmodel_ in _include/ncmodel.h_).
 
-1. format -- this is an NC_FORMAT_XXX value defining the file format
-   as seen by the user program.
-2. version -- the specific version of the format.
-3. iosp -- this is and NC_IOSP_XXX value describing internal protocols to use.
-4. impl -- this is an NC_FORMATX_XXX value defining, in effect, the
+1. impl -- this is an NC_FORMATX_XXX value defining, in effect, the
    dispatch table to use.
 
-For example, if the format was NC_FORMAT_CLASSIC, then the client
-will see the netcdf-3 data model, as modified by the version. If the
-version was 5, for example, then that indicates the file format
-is actually NC_FORMAT_64BIT_DATA, which is a variant of the netcdf-3
-format.
-
-The _iosp_ provides information about how the protocol the
-dispatch table will use to access the actual dataset. If the iosp
-is NC_IOSP_S3RAW, then it indicates that the dispatcher, NC_FORMATX_NC3,
-for example, will access the dataset using the Amazon S3 REST API.
-
 The construction of the model is primarily carried out by the function
-_NC_infermodel()_. It is given the following parameters:
+_NC_infermodel()_ (in _libdispatch/dinfermodel.c_).
+It is given the following parameters:
 1. path -- (IN) absolute file path or URL
-2. omodep -- (IN/OUT) the set of mode flags given to _NC_open_ or _NC_create_.
+2. modep -- (IN/OUT) the set of mode flags given to _NC_open_ or _NC_create_.
 3. iscreate -- (IN) distinguish open from create.
 4. useparallel -- (IN) indicate if parallel IO can be used.
 5. params -- (IN/OUT) arbitrary data dependent on the mode and path.
-6. model -- (IN/OUT) place to store inferred model information (e.g. format
-   or version).
-7. newpathp -- (OUT) sometimes, it is necessary to rewrite the path.
+6. model -- (IN/OUT) place to store inferred model.
+7. newpathp -- (OUT) the canonical rewrite of the path argument.
 
 As a rule, these values are used in the this order to infer the model.
 1. file contents -- highest precedence
-2. url (if it is one) -- using the protocol and fragment arguments
+2. url (if it is one) -- using the "mode=" key in the fragment (see below).
 3. mode flags
 4. default format -- lowest precedence
 
+If the path appears to be a URL, then it is parsed.
+Information is extracted from the URL, and specifically,
+the fragment key "mode=" is the critical element.
+The URL will be rewritten to a canonical form with the following
+changes.
+1. The fragment part ("#..." at the end) is parsed and the "mode=" key
+   is extracted and its value is converted to a list of tags.
+2. If the leading protocol is not http/https, then the protocol is added
+   to the mode list. That protocol is then replaced with either http or https.
+3. Certain singleton values inb the fragment are extracted and removed
+   and added to the mode list. Consider, for example, "http://....#dap4".
+   The "dap4" singleton is removed and added to the mode list.
+4. For backward compatibility, the values of "proto=" and "protocol="
+   are removed from the fragment and their value is added to the mode list.
+5. The final mode list is converted to a comma separated string
+   and re-inserted into the fragment.
+6. The final mode list is modified to remove duplicates.
+
+The final result is the canonical form of the URL and is returned in the
+newpathp argument described above.
+
 */

include/nc.h

@@ -8,9 +8,6 @@
 #include "config.h"
 #include "netcdf.h"
 
-/* Forward */
-struct NCmodel;
-
    /* There's an external ncid (ext_ncid) and an internal ncid
     * (int_ncid). The ext_ncid is the ncid returned to the user. If
     * the user has opened or created a netcdf-4 file, then the
@@ -30,7 +27,6 @@ typedef struct NC {
 	void* dispatchdata; /*per-'file' data; points to e.g. NC3_INFO data*/
 	char* path;
 	int   mode; /* as provided to nc_open/nc_create */
-        struct NCmodel*  model; /* as determined by libdispatch/dfile.c */
 } NC;
 
 /*
@@ -79,6 +75,6 @@ extern int iterate_NCList(int i,NC**); /* Walk from 0 ...; ERANGE return => stop
 
 /* Defined in nc.c */
 extern void free_NC(NC*);
-extern int new_NC(const struct NC_Dispatch*, const char*, int, struct NCmodel*, NC**);
+extern int new_NC(const struct NC_Dispatch*, const char*, int, NC**);
 
 #endif /* _NC_H_ */

include/ncmodel.h

@@ -9,39 +9,20 @@
 #ifndef NCINFERMODEL_H
 #define NCINFERMODEL_H
 
-/* Define the io handler to be used to do lowest level
-   access. This is above the libcurl level and below the
-   dispatcher level. This is only used for remote
-   datasets or for implementations where the implementation
-   multiplexes more than one IOSP in a single dispatcher.
-*/
-#define NC_IOSP_FILE	(1)
-#define NC_IOSP_MEMORY	(2)
-#define NC_IOSP_DAP2	(3)
-#define NC_IOSP_DAP4	(4)
-#define NC_IOSP_UDF	(5) /*Placeholder since we do not know IOSP for UDF*/
-#define NC_IOSP_HTTP	(6)
-
 /* Track the information hat will help us
    infer how to access the data defined by
-   path + omode.
+   path + omode + (sometimes) file content.
 */
 typedef struct NCmodel {
-    int format; /* NC_FORMAT_XXX value */
     int impl; /* NC_FORMATX_XXX value */
-    int iosp; /* NC_IOSP_XXX value (above) */
+    int format; /* NC_FORMAT_XXX value; Used to remember extra info; */
 } NCmodel;
 
 /* Keep compiler quiet */
 struct NCURI;
 struct NC_dispatch;
 
-#if 0
-/* return first IOSP or NULL if none */
-EXTERNL int NC_urliosp(struct NCURI* u);
-#endif
-
-/* Infer model format and implementation */
+/* Infer model implementation */
 EXTERNL int NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void* params, NCmodel* model, char** newpathp);
 
 /**
@@ -53,15 +34,4 @@ EXTERNL int NC_infermodel(const char* path, int* omodep, int iscreate, int usepa
  */
 EXTERNL int nc__testurl(const char* path, char** basenamep);
 
-#if 0
-/* allow access url parse and params without exposing nc_url.h */
-EXTERNL int NCDAP_urlparse(const char* s, void** dapurl);
-EXTERNL void NCDAP_urlfree(void* dapurl);
-EXTERNL const char* NCDAP_urllookup(void* dapurl, const char* param);
-
-/* Ping a specific server */
-EXTERNL int NCDAP2_ping(const char*);
-EXTERNL int NCDAP4_ping(const char*);
-#endif
-
 #endif /*NCINFERMODEL_H*/

include/ncrc.h

@@ -61,5 +61,6 @@ extern char* NC_entityescape(const char* s);
 extern int NC_readfile(const char* filename, NCbytes* content);
 extern int NC_writefile(const char* filename, size_t size, void* content);
 extern char* NC_mktmp(const char* base);
-
+extern int NC_getmodelist(const char* url, NClist** modelistp);
+extern int NC_testmode(const char* path, const char* tag);
 #endif /*NCRC_H*/

include/ncuri.h

@@ -6,19 +6,6 @@
 #ifndef NCURI_H
 #define NCURI_H
 
-/* Define error codes */
-#define NCU_OK (0)
-#define NCU_EINVAL        (1) /* Generic, mostly means bad argument */
-#define NCU_EBADURL       (2)
-#define NCU_ENOMEM        (3)
-#define NCU_EPROTO        (4)
-#define NCU_EPATH         (5)
-#define NCU_EUSRPWD       (6)
-#define NCU_EHOST         (7)
-#define NCU_EPORT         (8)
-#define NCU_EPARAMS       (9)
-#define NCU_ENOPARAM      (10)
-#define NCU_ECONSTRAINTS  (11)
 
 /* Define flags to control what is included by ncuribuild*/
 #define NCURIPATH      1
@@ -79,6 +66,9 @@ extern int ncurisetprotocol(NCURI*,const char* newprotocol);
 /* Replace the constraints */
 EXTERNL int ncurisetquery(NCURI*,const char* query);
 
+/* Replace the fragment list */
+extern int ncurisetfragments(NCURI*, const char* fragments);
+
 /* Construct a complete NC URI; caller frees returned string */
 EXTERNL char* ncuribuild(NCURI*,const char* prefix, const char* suffix, int flags);
 

libdap2/ncd2dispatch.c

@@ -322,7 +322,7 @@ NCD2_open(const char* path, int mode, int basepe, size_t *chunksizehintp,
     dapcomm->oc.rawurltext = strdup(path);
 #endif
 
-    if(ncuriparse(dapcomm->oc.rawurltext,&dapcomm->oc.url) != NCU_OK)
+    if(ncuriparse(dapcomm->oc.rawurltext,&dapcomm->oc.url))
 	{ncstat = NC_EURL; goto done;}
 
     if(!constrainable(dapcomm->oc.url))

libdap4/d4file.c

@@ -65,7 +65,7 @@ NCD4_open(const char * path, int mode,
     d4info->controller = (NC*)nc;
 
     /* Parse url and params */
-    if(ncuriparse(nc->path,&d4info->uri) != NCU_OK)
+    if(ncuriparse(nc->path,&d4info->uri))
 	{ret = NC_EDAPURL; goto done;}
 
     /* Load auth info from rc file */

libdap4/d4util.c

@@ -37,7 +37,7 @@ ncd4__testurl(const char* path, char** basenamep)
 {
     NCURI* uri;
     int ok = NC_NOERR;
-    if(ncuriparse(path,&uri) != NCU_OK)
+    if(ncuriparse(path,&uri))
 	ok = NC_EURL;
     else {
 	char* slash = (uri->path == NULL ? NULL : strrchr(uri->path, '/'));