Introduce more flexible storeChunk() syntax, use to add ADIOS2 memory selection

.pre-commit-config.yaml

@@ -66,7 +66,7 @@ repos:
 # clang-format v13
 #   to run manually, use .github/workflows/clang-format/clang-format.sh
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: v21.1.8
+  rev: v22.1.0
   hooks:
   - id: clang-format
     # By default, the clang-format hook configures:
@@ -88,7 +88,7 @@ repos:
 # Sorts Python imports according to PEP8
 # https://www.python.org/dev/peps/pep-0008/#imports
 - repo: https://github.com/pycqa/isort
-  rev: 7.0.0
+  rev: 8.0.1
   hooks:
   - id: isort
     name: isort (python)

CMakeLists.txt

@@ -400,6 +400,7 @@ set(CORE_SOURCE
         src/Format.cpp
         src/Iteration.cpp
         src/IterationEncoding.cpp
+        src/LoadStoreChunk.cpp
         src/Mesh.cpp
         src/ParticlePatches.cpp
         src/ParticleSpecies.cpp
@@ -411,6 +412,7 @@ set(CORE_SOURCE
         src/version.cpp
         src/auxiliary/Date.cpp
         src/auxiliary/Filesystem.cpp
+        src/auxiliary/Future.cpp
         src/auxiliary/JSON.cpp
         src/auxiliary/JSONMatcher.cpp
         src/auxiliary/Memory.cpp
@@ -826,6 +828,7 @@ if(openPMD_BUILD_TESTING)
         elseif(${test_name} STREQUAL "Core")
             list(APPEND ${out_list}
                 test/Files_Core/automatic_variable_encoding.cpp
+                test/Files_Core/read_nonexistent_attribute.cpp
             )
         endif()
     endmacro()

include/openPMD/Dataset.hpp

@@ -34,6 +34,18 @@ namespace openPMD
 using Extent = std::vector<std::uint64_t>;
 using Offset = std::vector<std::uint64_t>;
 
+/** Selection of a region of memory for storing chunks.
+ *
+ * Used to specify a non-contiguous memory region when storing
+ * data chunks. This allows writing data that is not contiguous
+ * in memory.
+ */
+struct MemorySelection
+{
+    Offset offset;
+    Extent extent;
+};
+
 class Dataset
 {
     friend class RecordComponent;

include/openPMD/Datatype.hpp

@@ -294,7 +294,8 @@ template <typename T>
 inline constexpr Datatype determineDatatype(T &&val)
 {
     (void)val; // don't need this, it only has a name for Doxygen
-    using T_stripped = std::remove_cv_t<std::remove_reference_t<T>>;
+    using T_stripped =
+        std::remove_extent_t<std::remove_cv_t<std::remove_reference_t<T>>>;
     if constexpr (auxiliary::IsPointer_v<T_stripped>)
     {
         return determineDatatype<auxiliary::IsPointer_t<T_stripped>>();
@@ -419,6 +420,13 @@ inline size_t toBits(Datatype d)
     return toBytes(d) * CHAR_BIT;
 }
 
+/** Check if a Datatype is a signed type
+ *
+ * @param d Datatype to test
+ * @return true if signed type (integer, floating point, complex), else false
+ */
+constexpr bool isSigned(Datatype d);
+
 /** Compare if a Datatype is a vector type
  *
  * @param d Datatype to test
@@ -595,14 +603,26 @@ inline std::tuple<bool, bool> isInteger()
  */
 template <typename T_FP>
 inline bool isSameFloatingPoint(Datatype d)
+{
+    return isSameFloatingPoint(d, determineDatatype<T_FP>());
+}
+
+/** Compare if two Datatypes are equivalent floating point types
+ *
+ * @param d1 First Datatype to compare
+ * @param d2 Second Datatype to compare
+ * @return true if both types are floating point and have same bitness, else
+ * false
+ */
+inline bool isSameFloatingPoint(Datatype d1, Datatype d2)
 {
     // template
-    bool tt_is_fp = isFloatingPoint<T_FP>();
+    bool tt_is_fp = isFloatingPoint(d1);
 
     // Datatype
-    bool dt_is_fp = isFloatingPoint(d);
+    bool dt_is_fp = isFloatingPoint(d2);
 
-    if (tt_is_fp && dt_is_fp && toBits(d) == toBits(determineDatatype<T_FP>()))
+    if (tt_is_fp && dt_is_fp && toBits(d1) == toBits(d2))
         return true;
     else
         return false;
@@ -617,15 +637,26 @@ inline bool isSameFloatingPoint(Datatype d)
  */
 template <typename T_CFP>
 inline bool isSameComplexFloatingPoint(Datatype d)
+{
+    return isSameComplexFloatingPoint(d, determineDatatype<T_CFP>());
+}
+
+/** Compare if two Datatypes are equivalent complex floating point types
+ *
+ * @param d1 First Datatype to compare
+ * @param d2 Second Datatype to compare
+ * @return true if both types are complex floating point and have same bitness,
+ * else false
+ */
+inline bool isSameComplexFloatingPoint(Datatype d1, Datatype d2)
 {
     // template
-    bool tt_is_cfp = isComplexFloatingPoint<T_CFP>();
+    bool tt_is_cfp = isComplexFloatingPoint(d1);
 
     // Datatype
-    bool dt_is_cfp = isComplexFloatingPoint(d);
+    bool dt_is_cfp = isComplexFloatingPoint(d2);
 
-    if (tt_is_cfp && dt_is_cfp &&
-        toBits(d) == toBits(determineDatatype<T_CFP>()))
+    if (tt_is_cfp && dt_is_cfp && toBits(d1) == toBits(d2))
         return true;
     else
         return false;
@@ -640,17 +671,29 @@ inline bool isSameComplexFloatingPoint(Datatype d)
  */
 template <typename T_Int>
 inline bool isSameInteger(Datatype d)
+{
+    return isSameInteger(d, determineDatatype<T_Int>());
+}
+
+/** Compare if two Datatypes are equivalent integer types
+ *
+ * @param d1 First Datatype to compare
+ * @param d2 Second Datatype to compare
+ * @return true if both types are integers, same signedness and same bitness,
+ * else false
+ */
+inline bool isSameInteger(Datatype d1, Datatype d2)
 {
     // template
     bool tt_is_int, tt_is_sig;
-    std::tie(tt_is_int, tt_is_sig) = isInteger<T_Int>();
+    std::tie(tt_is_int, tt_is_sig) = isInteger(d1);
 
     // Datatype
     bool dt_is_int, dt_is_sig;
-    std::tie(dt_is_int, dt_is_sig) = isInteger(d);
+    std::tie(dt_is_int, dt_is_sig) = isInteger(d2);
 
     if (tt_is_int && dt_is_int && tt_is_sig == dt_is_sig &&
-        toBits(d) == toBits(determineDatatype<T_Int>()))
+        toBits(d1) == toBits(d2))
         return true;
     else
         return false;
@@ -691,46 +734,26 @@ constexpr bool isChar(Datatype d)
 template <typename T_Char>
 constexpr bool isSameChar(Datatype d);
 
+/** Compare if two Datatypes are equivalent char types
+ *
+ * @param d1 First Datatype to compare
+ * @param d2 Second Datatype to compare
+ * @return true if both types are chars with same signedness and size, else
+ * false
+ */
+constexpr bool isSameChar(Datatype d1, Datatype d2);
+
 /** Comparison for two Datatypes
  *
  * Besides returning true for the same types, identical implementations on
  * some platforms, e.g. if long and long long are the same or double and
  * long double will also return true.
+ *
+ * @param d First Datatype to compare
+ * @param e Second Datatype to compare
+ * @return true if the datatypes are equivalent
  */
-inline bool isSame(openPMD::Datatype const d, openPMD::Datatype const e)
-{
-    // exact same type
-    if (static_cast<int>(d) == static_cast<int>(e))
-        return true;
-
-    bool d_is_vec = isVector(d);
-    bool e_is_vec = isVector(e);
-
-    // same int
-    bool d_is_int, d_is_sig;
-    std::tie(d_is_int, d_is_sig) = isInteger(d);
-    bool e_is_int, e_is_sig;
-    std::tie(e_is_int, e_is_sig) = isInteger(e);
-    if (d_is_int && e_is_int && d_is_vec == e_is_vec && d_is_sig == e_is_sig &&
-        toBits(d) == toBits(e))
-        return true;
-
-    // same float
-    bool d_is_fp = isFloatingPoint(d);
-    bool e_is_fp = isFloatingPoint(e);
-
-    if (d_is_fp && e_is_fp && d_is_vec == e_is_vec && toBits(d) == toBits(e))
-        return true;
-
-    // same complex floating point
-    bool d_is_cfp = isComplexFloatingPoint(d);
-    bool e_is_cfp = isComplexFloatingPoint(e);
-
-    if (d_is_cfp && e_is_cfp && d_is_vec == e_is_vec && toBits(d) == toBits(e))
-        return true;
-
-    return false;
-}
+constexpr bool isSame(openPMD::Datatype d, openPMD::Datatype e);
 
 /**
  * @brief basicDatatype Strip openPMD Datatype of std::vector, std::array et.
@@ -740,15 +763,34 @@ inline bool isSame(openPMD::Datatype const d, openPMD::Datatype const e)
  */
 Datatype basicDatatype(Datatype dt);
 
+/** Convert a scalar Datatype to its vector variant
+ *
+ * @param dt Scalar Datatype to convert
+ * @return Vector Datatype (e.g., INT becomes VEC_INT)
+ */
 Datatype toVectorType(Datatype dt);
 
+/** Convert a Datatype to its string representation
+ *
+ * @param dt Datatype to convert
+ * @return String representation of the Datatype
+ */
 std::string datatypeToString(Datatype dt);
 
+/** Convert a string to a Datatype
+ *
+ * @param s String representation of a Datatype
+ * @return The corresponding Datatype
+ */
 Datatype stringToDatatype(const std::string &s);
 
-void warnWrongDtype(std::string const &key, Datatype store, Datatype request);
-
-std::ostream &operator<<(std::ostream &, openPMD::Datatype const &);
+/** Stream operator for Datatype
+ *
+ * @param os Output stream
+ * @param dt Datatype to output
+ * @return Reference to the stream
+ */
+std::ostream &operator<<(std::ostream &os, openPMD::Datatype const &dt);
 
 template <typename T>
 constexpr auto datatypeIndex() -> size_t