diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index 887a64ebbb..463177097b 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -41,7 +41,7 @@ jobs: libhdf4-devel zipinfo libxml2-devel perl zlib-devel libzstd-devel libbz2-devel libaec-devel libzip-devel libdeflate-devel gcc-core gcc-g++ libcurl-devel libiconv-devel - libssl-devel libcrypt-devel + libssl-devel libcrypt-devel attr libattr-devel - name: (Autotools) Run autoconf and friends run: | @@ -78,10 +78,15 @@ jobs: if [ $(find /tmp/pretend-root/$(pwd) -type f | wc -l) -gt 0 ]; then exit 1; fi fi - - name: (Autotools) Build and run tests + - name: (Autotools) Build tests timeout-minutes: 30 run: | - make check -j8 SHELL=/bin/dash + make check -j$(nproc) TESTS="" SHELL=/bin/dash + + - name: (Autotools) Run tests + timeout-minutes: 30 + run: | + make check -j$(nproc) SHELL=/bin/dash build-and-test-cmake: name: Cygwin-based CMake tests diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d68f3219b..8910887894 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,6 +87,11 @@ if(UNAME) set(TMP_BUILDNAME "${osname}-${osrel}-${cpu}") endif() +find_program(GETFATTR NAMES getfattr) +if(GETFATTR) + set(HAVE_GETFATTR TRUE) +endif() + # Define some Platforms if(osname MATCHES "CYGWIN.*") set(ISCYGWIN yes) @@ -1151,6 +1156,7 @@ CHECK_INCLUDE_file("io.h" HAVE_IO_H) endif(MSVC) CHECK_INCLUDE_file("stdlib.h" HAVE_STDLIB_H) CHECK_INCLUDE_file("ctype.h" HAVE_CTYPE_H) +CHECK_INCLUDE_file("sys/xattr_h" HAVE_SYS_XATTR_H) CHECK_INCLUDE_file("stdarg.h" HAVE_STDARG_H) CHECK_INCLUDE_file("strings.h" HAVE_STRINGS_H) CHECK_INCLUDE_file("signal.h" HAVE_SIGNAL_H) diff --git a/config.h.cmake.in b/config.h.cmake.in index a38749a920..fe7dabcdf7 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -373,6 +373,12 @@ are set when opening a binary file on Windows. */ /* Define to 1 if you have the header file. */ #cmakedefine HAVE_CTYPE_H 1 +/* Define to 1 if you have the getfattr command line utility. */ +#cmakedefine HAVE_GETFATTR 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_SYS_XATTR_H + /* Define to 1 if you have the header file. */ #cmakedefine HAVE_STRINGS_H 1 diff --git a/configure.ac b/configure.ac index 61a4563901..d223c19f8a 100644 --- a/configure.ac +++ b/configure.ac @@ -82,6 +82,14 @@ AC_LANG_POP([C]) if test $have_no_strict_aliasing = no; then CFLAGS=$SAVE_CFLAGS fi + +## +# Check to see if we have getfattr +## +AC_CHECK_PROGS([HAVE_GETFATTR], [getfattr]) +if test -n "$HAVE_GETFATTR"; then + AC_DEFINE_UNQUOTED([HAVE_GETFATTR],[1],[getfattr is available]) +fi ## # Some files need to exist in build directories # that do not correspond to their source directory, or @@ -1292,7 +1300,7 @@ AC_CHECK_HEADERS([sys/param.h]) AC_CHECK_HEADERS([libgen.h]) #AC_CHECK_HEADERS([locale.h]) AC_HEADER_STDC -AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h time.h dirent.h stdint.h ctype.h]) +AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h time.h dirent.h stdint.h ctype.h sys/xattr_h]) # Do sys/resource.h separately #AC_CHECK_HEADERS([sys/resource.h],[havesysresource=1],[havesysresource=0]) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 7bb2eaddda..a381ee0082 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -7,16 +7,24 @@ * Copyright 2018 University Corporation for Atmospheric * Research/Unidata. See COPYRIGHT file for more info. */ - #include "config.h" #include #include +#include #ifdef HAVE_UNISTD_H #include #endif #ifdef HAVE_SYS_TYPES_H #include #endif +#ifndef _WIN32 +#ifdef USE_HDF5 +#include +#endif /* USE_HDF5 */ +#endif /* _WIN32 */ +#ifdef HAVE_SYS_XATTR_H +#include +#endif #include "ncdispatch.h" #include "ncpathmgr.h" @@ -229,6 +237,7 @@ static int replacemode(NClist* envv, const char* newval); static void infernext(NClist* current, NClist* next); static int negateone(const char* mode, NClist* modes); static void cleanstringlist(NClist* strs, int caseinsensitive); +static int isdaoscontainer(const char* path); /* If the path looks like a URL, then parse it, reformat it. @@ -848,7 +857,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void const char* modeval = NULL; char* abspath = NULL; NClist* tmp = NULL; - + /* Phase 1: 1. convert special protocols to http|https 2. begin collecting fragments @@ -967,15 +976,22 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void if((stat = NC_omodeinfer(useparallel,omode,model))) goto done; } - /* Phase 9: Infer from file content, if possible; - this has highest precedence, so it may override - previous decisions. Note that we do this last - because we need previously determined model info - to guess if this file is readable. - */ - if(!iscreate && isreadable(uri,model)) { - /* Ok, we need to try to read the file */ - if((stat = check_file_type(path, omode, useparallel, params, model, uri))) goto done; + /* Phase 9: Special case for file stored in DAOS container */ + if(isdaoscontainer(path) == NC_NOERR) { + /* This is a DAOS container, so immediately assume it is HDF5. */ + model->impl = NC_FORMATX_NC_HDF5; + model->format = NC_FORMAT_NETCDF4; + } else { + /* Phase 10: Infer from file content, if possible; + this has highest precedence, so it may override + previous decisions. Note that we do this last + because we need previously determined model info + to guess if this file is readable. + */ + if(!iscreate && isreadable(uri,model)) { + /* Ok, we need to try to read the file */ + if((stat = check_file_type(path, omode, useparallel, params, model, uri))) goto done; + } } /* Need a decision */ @@ -1566,6 +1582,100 @@ NC_interpret_magic_number(char* magic, NCmodel* model) return check(status); } +/* Return NC_NOERR if path is a DAOS container; return NC_EXXX otherwise */ +static int +isdaoscontainer(const char* path) +{ + int stat = NC_ENOTNC; /* default is that this is not a DAOS container */ +#ifndef _WIN32 +#ifdef USE_HDF5 +#if H5_VERSION_GE(1,12,0) + htri_t accessible; + hid_t fapl_id; + int rc; + /* Check for a DAOS container */ + if((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) {stat = NC_EHDFERR; goto done;} + H5Pset_fapl_sec2(fapl_id); + accessible = H5Fis_accessible(path, fapl_id); + H5Pclose(fapl_id); /* Ignore any error */ + rc = 0; + if(accessible > 0) { +#ifdef HAVE_SYS_XATTR_H + ssize_t xlen; +#ifdef __APPLE__ + xlen = listxattr(path, NULL, 0, 0); +#else + xlen = listxattr(path, NULL, 0); +#endif + if(xlen > 0) { + char* xlist = NULL; + char* xvalue = NULL; + char* p; + char* endp; + if((xlist = (char*)calloc(1,(size_t)xlen))==NULL) + {stat = NC_ENOMEM; goto done;} +#ifdef __APPLE__ + (void)listxattr(path, xlist, (size_t)xlen, 0); /* Get xattr names */ +#else + (void)listxattr(path, xlist, (size_t)xlen); /* Get xattr names */ +#endif + p = xlist; endp = p + xlen; /* delimit names */ + /* walk the list of xattr names */ + for(;p < endp;p += (strlen(p)+1)) { + /* The popen version looks for the string ".daos"; + It would be nice if we know whether that occurred + int the xattr's name or it value. + Oh well, we will do the general search */ + /* Look for '.daos' in the key */ + if(strstr(p,".daos") != NULL) {rc = 1; break;} /* success */ + /* Else get the p'th xattr's value size */ +#ifdef __APPLE__ + xlen = getxattr(path, p, NULL, 0, 0, 0); +#else + xlen = getxattr(path, p, NULL, 0); +#endif + if((xvalue = (char*)calloc(1,(size_t)xlen))==NULL) + {stat = NC_ENOMEM; goto done;} + /* Read the value */ +#ifdef __APPLE__ + (void)getxattr(path, p, xvalue, (size_t)xlen, 0, 0); +#else + (void)getxattr(path, p, xvalue, (size_t)xlen); +#endif +fprintf(stderr,"@@@ %s=|%s|\n",p,xvalue); + /* Look for '.daos' in the value */ + if(strstr(xvalue,".daos") != NULL) {rc = 1; break;} /* success */ + } + } +#else /*!HAVE_SYS_XATTR_H*/ + +#ifdef HAVE_GETFATTR + { + FILE *fp; + char cmd[4096]; + memset(cmd,0,sizeof(cmd)); + snprintf(cmd,sizeof(cmd),"getfattr %s | grep -c '.daos'",path); + if((fp = popen(cmd, "r")) != NULL) { + fscanf(fp, "%d", &rc); + pclose(fp); + } + } +#else /*!HAVE_GETFATTR*/ + /* We just can't test for DAOS container.*/ + stat = 0; +#endif /*HAVE_GETFATTR*/ +#endif /*HAVE_SYS_XATTR_H*/ + } + /* Test for DAOS container */ + stat = (rc == 1 ? NC_NOERR : NC_ENOTNC); +done: +#endif +#endif +#endif + errno = 0; /* reset */ + return stat; +} + #ifdef DEBUG static void printmagic(const char* tag, char* magic, struct MagicFile* f)