Move flushing from storeChunk to resetDataset

Author: franzpoeschel

include/openPMD/RecordComponent.tpp

@@ -85,35 +85,22 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer)
 {
     verifyChunk<T>(o, e);
 
-    /*
-     * The openPMD backend might not yet know about this dataset.
-     * Flush the openPMD hierarchy to the backend without flushing any actual
-     * data yet.
-     */
-    seriesFlush_impl</* flush_entire_series = */ false>(
-        {FlushLevel::SkeletonOnly});
-
     size_t size = 1;
     for (auto ext : e)
     {
         size *= ext;
     }
+
     /*
      * Flushing the skeleton does not create datasets,
      * so we might need to do it now.
      */
-    if (!written())
+    auto &rc = get();
+    if (!rc.m_dataset.has_value())
     {
-        auto &rc = get();
-        if (!rc.m_dataset.has_value())
-        {
-            throw error::WrongAPIUsage(
-                "[RecordComponent] Must specify dataset type and extent before "
-                "using storeChunk() (see RecordComponent::resetDataset()).");
-        }
-        Parameter<Operation::CREATE_DATASET> dCreate(rc.m_dataset.value());
-        dCreate.name = Attributable::get().m_writable.ownKeyWithinParent;
-        IOHandler()->enqueue(IOTask(this, dCreate));
+        throw error::WrongAPIUsage(
+            "[RecordComponent] Must specify dataset type and extent before "
+            "using storeChunk() (see RecordComponent::resetDataset()).");
     }
 
     Parameter<Operation::GET_BUFFER_VIEW> getBufferView;

include/openPMD/backend/Attributable.hpp

@@ -421,7 +421,7 @@ OPENPMD_protected
     /** @} */
 
     template <bool flush_entire_series>
-    void seriesFlush_impl(internal::FlushParams const &);
+    void seriesFlush_impl(internal::FlushParams const &, bool flush_io_handler);
 
     void flushAttributes(internal::FlushParams const &);
 

include/openPMD/backend/Writable.hpp

@@ -126,14 +126,15 @@ class Writable final
      * it.
      */
     template <bool flush_entire_series>
-    void seriesFlush(std::string backendConfig = "{}");
+    void
+    seriesFlush(std::string backendConfig = "{}", bool flush_io_handler = true);
 
     // clang-format off
 OPENPMD_private
     // clang-format on
 
     template <bool flush_entire_series>
-    void seriesFlush(internal::FlushParams const &);
+    void seriesFlush(internal::FlushParams const &, bool flush_io_handler);
     /*
      * These members need to be shared pointers since distinct instances of
      * Writable may share them.

src/RecordComponent.cpp

@@ -22,6 +22,7 @@
 #include "openPMD/Dataset.hpp"
 #include "openPMD/DatatypeHelpers.hpp"
 #include "openPMD/Error.hpp"
+#include "openPMD/IO/AbstractIOHandler.hpp"
 #include "openPMD/IO/Format.hpp"
 #include "openPMD/Series.hpp"
 #include "openPMD/auxiliary/Environment.hpp"
@@ -237,10 +238,15 @@ RecordComponent &RecordComponent::resetDataset(Dataset d)
     auto &rc = get();
     auto cleanup = defer([&rc, this]() {
         if (rc.m_dataset.has_value() &&
-            rc.m_dataset->dtype != Datatype::UNDEFINED)
+            rc.m_dataset->dtype != Datatype::UNDEFINED &&
+            IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing)
         {
+            // TODO: try getting flush_io_handler = false to run
             seriesFlush_impl</* flush_entire_series = */ false>(
-                {FlushLevel::SkeletonOnly});
+                {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true);
+            Parameter<Operation::CREATE_DATASET> dCreate(rc.m_dataset.value());
+            dCreate.name = Attributable::get().m_writable.ownKeyWithinParent;
+            IOHandler()->enqueue(IOTask(this, dCreate));
         }
     });
     if (written())

src/backend/Attributable.cpp

@@ -271,14 +271,15 @@ OpenpmdStandard Attributable::openPMDStandard() const
 }
 
 template <bool flush_entire_series>
-void Attributable::seriesFlush_impl(internal::FlushParams const &flushParams)
+void Attributable::seriesFlush_impl(
+    internal::FlushParams const &flushParams, bool flush_io_handler)
 {
-    writable().seriesFlush<flush_entire_series>(flushParams);
+    writable().seriesFlush<flush_entire_series>(flushParams, flush_io_handler);
 }
-template void
-Attributable::seriesFlush_impl<true>(internal::FlushParams const &flushParams);
-template void
-Attributable::seriesFlush_impl<false>(internal::FlushParams const &flushParams);
+template void Attributable::seriesFlush_impl<true>(
+    internal::FlushParams const &flushParams, bool flush_io_handler);
+template void Attributable::seriesFlush_impl<false>(
+    internal::FlushParams const &flushParams, bool flush_io_handler);
 
 void Attributable::flushAttributes(internal::FlushParams const &flushParams)
 {

src/backend/BaseRecord.cpp

@@ -19,6 +19,7 @@
  * If not, see <http://www.gnu.org/licenses/>.
  */
 #include "openPMD/backend/BaseRecord.hpp"
+#include "openPMD/IO/AbstractIOHandler.hpp"
 #include "openPMD/backend/MeshRecordComponent.hpp"
 #include "openPMD/backend/PatchRecordComponent.hpp"
 

src/backend/Writable.cpp

@@ -52,16 +52,20 @@ Writable::~Writable()
 }
 
 template <bool flush_entire_series>
-void Writable::seriesFlush(std::string backendConfig)
+void Writable::seriesFlush(std::string backendConfig, bool flush_io_handler)
 {
     seriesFlush<flush_entire_series>(
-        internal::FlushParams{FlushLevel::UserFlush, std::move(backendConfig)});
+        internal::FlushParams{FlushLevel::UserFlush, std::move(backendConfig)},
+        flush_io_handler);
 }
-template void Writable::seriesFlush<true>(std::string backendConfig);
-template void Writable::seriesFlush<false>(std::string backendConfig);
+template void
+Writable::seriesFlush<true>(std::string backendConfig, bool flush_io_handler);
+template void
+Writable::seriesFlush<false>(std::string backendConfig, bool flush_io_handler);
 
 template <bool flush_entire_series>
-void Writable::seriesFlush(internal::FlushParams const &flushParams)
+void Writable::seriesFlush(
+    internal::FlushParams const &flushParams, bool flush_io_handler)
 {
     Attributable impl;
     impl.setData({attributable, [](auto const *) {}});
@@ -103,10 +107,10 @@ void Writable::seriesFlush(internal::FlushParams const &flushParams)
             return {series.iterations.begin(), series.iterations.end()};
         }
     }();
-    series.flush_impl(begin, end, flushParams);
+    series.flush_impl(begin, end, flushParams, flush_io_handler);
 }
-template void
-Writable::seriesFlush<true>(internal::FlushParams const &flushParams);
-template void
-Writable::seriesFlush<false>(internal::FlushParams const &flushParams);
+template void Writable::seriesFlush<true>(
+    internal::FlushParams const &flushParams, bool flush_io_handler);
+template void Writable::seriesFlush<false>(
+    internal::FlushParams const &flushParams, bool flush_io_handler);
 } // namespace openPMD