Merge pull request #2021 from brtnfld/DAOS_sync

Check if HDF5 "file" is a DAOS object

Author: WardF

.github/workflows/run_tests_win_cygwin.yml

@@ -41,7 +41,7 @@ jobs:
             libhdf4-devel zipinfo libxml2-devel perl zlib-devel
             libzstd-devel libbz2-devel libaec-devel libzip-devel
             libdeflate-devel gcc-core gcc-g++ libcurl-devel libiconv-devel
-            libssl-devel libcrypt-devel
+            libssl-devel libcrypt-devel attr libattr-devel
 
       - name: (Autotools) Run autoconf and friends
         run: |
@@ -78,10 +78,15 @@ jobs:
               if [ $(find /tmp/pretend-root/$(pwd) -type f | wc -l) -gt 0 ]; then exit 1; fi
           fi
 
-      - name: (Autotools) Build and run tests
+      - name: (Autotools) Build tests
         timeout-minutes: 30
         run: |
-          make check -j8 SHELL=/bin/dash
+          make check -j$(nproc) TESTS="" SHELL=/bin/dash
+
+      - name: (Autotools) Run tests
+        timeout-minutes: 30
+        run: |
+          make check -j$(nproc) SHELL=/bin/dash
   
   build-and-test-cmake:
     name: Cygwin-based CMake tests

CMakeLists.txt

@@ -87,6 +87,11 @@ if(UNAME)
   set(TMP_BUILDNAME "${osname}-${osrel}-${cpu}")
 endif()
 
+find_program(GETFATTR NAMES getfattr)
+if(GETFATTR)
+  set(HAVE_GETFATTR TRUE)
+endif()
+
 # Define some Platforms
 if(osname MATCHES "CYGWIN.*")
   set(ISCYGWIN yes)
@@ -1151,6 +1156,7 @@ CHECK_INCLUDE_file("io.h"      HAVE_IO_H)
 endif(MSVC)
 CHECK_INCLUDE_file("stdlib.h"  HAVE_STDLIB_H)
 CHECK_INCLUDE_file("ctype.h"   HAVE_CTYPE_H)
+CHECK_INCLUDE_file("sys/xattr_h" HAVE_SYS_XATTR_H)
 CHECK_INCLUDE_file("stdarg.h"    HAVE_STDARG_H)
 CHECK_INCLUDE_file("strings.h"   HAVE_STRINGS_H)
 CHECK_INCLUDE_file("signal.h"    HAVE_SIGNAL_H)

config.h.cmake.in

@@ -373,6 +373,12 @@ are set when opening a binary file on Windows. */
 /* Define to 1 if you have the <ctype.h> header file. */
 #cmakedefine HAVE_CTYPE_H 1
 
+/* Define to 1 if you have the getfattr command line utility. */
+#cmakedefine HAVE_GETFATTR 1
+
+/* Define to 1 if you have the <sys/xattr.h> header file. */
+#cmakedefine HAVE_SYS_XATTR_H
+
 /* Define to 1 if you have the <strings.h> header file. */
 #cmakedefine HAVE_STRINGS_H 1
 

configure.ac

@@ -82,6 +82,14 @@ AC_LANG_POP([C])
 if test $have_no_strict_aliasing = no; then
    CFLAGS=$SAVE_CFLAGS
 fi
+
+## 
+# Check to see if we have getfattr
+##
+AC_CHECK_PROGS([HAVE_GETFATTR], [getfattr])
+if test -n "$HAVE_GETFATTR"; then
+   AC_DEFINE_UNQUOTED([HAVE_GETFATTR],[1],[getfattr is available])
+fi
 ##
 # Some files need to exist in build directories
 # that do not correspond to their source directory, or
@@ -1292,7 +1300,7 @@ AC_CHECK_HEADERS([sys/param.h])
 AC_CHECK_HEADERS([libgen.h])
 #AC_CHECK_HEADERS([locale.h])
 AC_HEADER_STDC
-AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h time.h dirent.h stdint.h ctype.h])
+AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h time.h dirent.h stdint.h ctype.h sys/xattr_h])
 
 # Do sys/resource.h separately
 #AC_CHECK_HEADERS([sys/resource.h],[havesysresource=1],[havesysresource=0])

libdispatch/dinfermodel.c

@@ -7,16 +7,24 @@
  * Copyright 2018 University Corporation for Atmospheric
  * Research/Unidata. See COPYRIGHT file for more info.
 */
-
 #include "config.h"
 #include <stddef.h>
 #include <stdlib.h>
+#include <string.h>
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
 #ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
 #endif
+#ifndef _WIN32
+#ifdef USE_HDF5
+#include <hdf5.h>
+#endif /* USE_HDF5 */
+#endif /* _WIN32 */
+#ifdef HAVE_SYS_XATTR_H
+#include <sys/xattr.h>
+#endif
 
 #include "ncdispatch.h"
 #include "ncpathmgr.h"
@@ -229,6 +237,7 @@ static int replacemode(NClist* envv, const char* newval);
 static void infernext(NClist* current, NClist* next);
 static int negateone(const char* mode, NClist* modes);
 static void cleanstringlist(NClist* strs, int caseinsensitive);
+static int isdaoscontainer(const char* path);
 
 /*
 If the path looks like a URL, then parse it, reformat it.
@@ -848,7 +857,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
     const char* modeval = NULL;
     char* abspath = NULL;
     NClist* tmp = NULL;
-    
+
     /* Phase 1:
        1. convert special protocols to http|https
        2. begin collecting fragments
@@ -967,15 +976,22 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
         if((stat = NC_omodeinfer(useparallel,omode,model))) goto done;
     }
 
-    /* Phase 9: Infer from file content, if possible;
-       this has highest precedence, so it may override
-       previous decisions. Note that we do this last
-       because we need previously determined model info
-       to guess if this file is readable.
-    */
-    if(!iscreate && isreadable(uri,model)) {
-	/* Ok, we need to try to read the file */
-	if((stat = check_file_type(path, omode, useparallel, params, model, uri))) goto done;
+    /* Phase 9: Special case for file stored in DAOS container */
+    if(isdaoscontainer(path) == NC_NOERR) {
+        /* This is a DAOS container, so immediately assume it is HDF5. */
+        model->impl = NC_FORMATX_NC_HDF5;
+        model->format = NC_FORMAT_NETCDF4;
+    } else {
+        /* Phase 10: Infer from file content, if possible;
+           this has highest precedence, so it may override
+           previous decisions. Note that we do this last
+           because we need previously determined model info
+           to guess if this file is readable.
+        */
+        if(!iscreate && isreadable(uri,model)) {
+	     /* Ok, we need to try to read the file */
+            if((stat = check_file_type(path, omode, useparallel, params, model, uri))) goto done;
+        }
     }
 
     /* Need a decision */
@@ -1566,6 +1582,100 @@ NC_interpret_magic_number(char* magic, NCmodel* model)
      return check(status);
 }
 
+/* Return NC_NOERR if path is a DAOS container; return NC_EXXX otherwise */
+static int
+isdaoscontainer(const char* path)
+{
+    int stat = NC_ENOTNC; /* default is that this is not a DAOS container */
+#ifndef _WIN32
+#ifdef USE_HDF5
+#if H5_VERSION_GE(1,12,0)
+    htri_t accessible;
+    hid_t fapl_id;
+    int rc;
+    /* Check for a DAOS container */
+    if((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) {stat = NC_EHDFERR; goto done;}
+    H5Pset_fapl_sec2(fapl_id);
+    accessible = H5Fis_accessible(path, fapl_id);
+    H5Pclose(fapl_id); /* Ignore any error */
+    rc = 0;
+    if(accessible > 0) {
+#ifdef HAVE_SYS_XATTR_H
+	ssize_t xlen;
+#ifdef __APPLE__
+	xlen = listxattr(path, NULL, 0, 0);
+#else
+	xlen = listxattr(path, NULL, 0);
+#endif
+        if(xlen > 0) {
+  	    char* xlist = NULL;
+	    char* xvalue = NULL;
+	    char* p;
+	    char* endp;
+	    if((xlist = (char*)calloc(1,(size_t)xlen))==NULL)
+		{stat = NC_ENOMEM; goto done;}
+#ifdef __APPLE__
+	    (void)listxattr(path, xlist, (size_t)xlen, 0); /* Get xattr names */
+#else
+	    (void)listxattr(path, xlist, (size_t)xlen); /* Get xattr names */
+#endif
+	    p = xlist; endp = p + xlen; /* delimit names */
+	    /* walk the list of xattr names */
+	    for(;p < endp;p += (strlen(p)+1)) {
+		/* The popen version looks for the string ".daos";
+                   It would be nice if we know whether that occurred
+		   int the xattr's name or it value.
+		   Oh well, we will do the general search */
+		/* Look for '.daos' in the key */
+		if(strstr(p,".daos") != NULL) {rc = 1; break;} /* success */
+		/* Else get the p'th xattr's value size */
+#ifdef __APPLE__
+		xlen = getxattr(path, p, NULL, 0, 0, 0);
+#else
+		xlen = getxattr(path, p, NULL, 0);
+#endif
+		if((xvalue = (char*)calloc(1,(size_t)xlen))==NULL)
+		    {stat = NC_ENOMEM; goto done;}
+		/* Read the value */
+#ifdef __APPLE__
+		(void)getxattr(path, p, xvalue, (size_t)xlen, 0, 0);
+#else
+		(void)getxattr(path, p, xvalue, (size_t)xlen);
+#endif
+fprintf(stderr,"@@@ %s=|%s|\n",p,xvalue);
+		/* Look for '.daos' in the value */
+		if(strstr(xvalue,".daos") != NULL) {rc = 1; break;} /* success */
+	    }
+        }
+#else /*!HAVE_SYS_XATTR_H*/
+
+#ifdef HAVE_GETFATTR
+	{
+	    FILE *fp;
+	    char cmd[4096];
+	    memset(cmd,0,sizeof(cmd));
+        snprintf(cmd,sizeof(cmd),"getfattr %s | grep -c '.daos'",path);
+        if((fp = popen(cmd, "r")) != NULL) {
+               fscanf(fp, "%d", &rc);
+               pclose(fp);
+	    }
+    }
+#else /*!HAVE_GETFATTR*/
+    /* We just can't test for DAOS container.*/
+    stat = 0;
+#endif /*HAVE_GETFATTR*/
+#endif /*HAVE_SYS_XATTR_H*/
+    }
+    /* Test for DAOS container */
+    stat = (rc == 1 ? NC_NOERR : NC_ENOTNC);
+done:
+#endif
+#endif
+#endif
+    errno = 0; /* reset */
+    return stat;
+}
+
 #ifdef DEBUG
 static void
 printmagic(const char* tag, char* magic, struct MagicFile* f)