Parallel HDF5: Try fixing strict collective requirements of HDF5 >= 2.0

CMakeLists.txt

@@ -426,6 +426,11 @@ set(CORE_SOURCE
         src/backend/PatchRecord.cpp
         src/backend/PatchRecordComponent.cpp
         src/backend/Writable.cpp
+        src/backend/scientific_defaults/ScientificDefaults.cpp
+        src/backend/scientific_defaults/ScientificDefaults_auxiliary.cpp
+        src/backend/scientific_defaults/ProcessParsedAttribute.cpp
+        src/backend/scientific_defaults/AttributeReader.cpp
+        src/backend/scientific_defaults/ConfigAttribute.cpp
         src/auxiliary/OneDimensionalBlockSlicer.cpp
         src/helper/list_series.cpp
         src/snapshots/ContainerImpls.cpp

include/openPMD/IO/AbstractIOHandler.hpp

@@ -26,6 +26,7 @@
 #include "openPMD/IterationEncoding.hpp"
 #include "openPMD/config.hpp"
 #include "openPMD/version.hpp"
+#include <ostream>
 
 #if openPMD_HAVE_MPI
 #include <mpi.h>
@@ -81,6 +82,66 @@ enum class FlushLevel
     CreateOrOpenFiles
 };
 
+std::ostream &operator<<(std::ostream &, FlushLevel);
+
+namespace flush_level
+{
+    inline constexpr auto global_flushpoint(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+            return true;
+        case FlushLevel::InternalFlush:
+        case FlushLevel::SkeletonOnly:
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+    // same as global_flushpoint for now, but we will soon introduce
+    // immediate_flush
+    inline constexpr auto write_datasets(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+            return true;
+        case FlushLevel::InternalFlush:
+        case FlushLevel::SkeletonOnly:
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+    inline constexpr auto write_attributes(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+        case FlushLevel::InternalFlush:
+            return true;
+        case FlushLevel::SkeletonOnly:
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+    inline constexpr auto flush_hierarchy(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+        case FlushLevel::InternalFlush:
+        case FlushLevel::SkeletonOnly:
+            return true;
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+} // namespace flush_level
+
 enum class OpenpmdStandard
 {
     v_1_0_0,
@@ -121,6 +182,7 @@ namespace internal
      * To be used for reading
      */
     FlushParams const defaultFlushParams{};
+    FlushParams const publicFlush{FlushLevel::UserFlush};
 
     struct ParsedFlushParams;
 

include/openPMD/IO/AbstractIOHandlerImpl.hpp

@@ -39,7 +39,7 @@ class AbstractIOHandlerImpl
 
     virtual ~AbstractIOHandlerImpl() = default;
 
-    std::future<void> flush();
+    std::future<void> flush(FlushLevel);
 
     /**
      * Close the file corresponding with the writable and release file handles.

include/openPMD/IO/AbstractIOHandlerImplCommon.hpp

@@ -28,10 +28,10 @@
 #include "openPMD/auxiliary/StringManip.hpp"
 #include "openPMD/backend/Writable.hpp"
 
+#include <set>
 #include <stdexcept>
 #include <string>
 #include <unordered_map>
-#include <unordered_set>
 
 namespace openPMD
 {
@@ -50,7 +50,10 @@ class AbstractIOHandlerImplCommon : public AbstractIOHandlerImpl
      * without the OS path
      */
     std::unordered_map<Writable *, InvalidatableFile> m_files;
-    std::unordered_set<InvalidatableFile> m_dirty;
+    // MUST be an ordered set in order to consistently flush on different
+    // parallel processes (same logic cant apply to m_files since Writable*
+    // pointers are not predictable)
+    std::set<InvalidatableFile> m_dirty;
 
     enum PossiblyExisting
     {

include/openPMD/IO/IOTask.hpp

@@ -35,6 +35,7 @@
 #include <cstddef>
 #include <memory>
 #include <optional>
+#include <ostream>
 #include <string>
 #include <utility>
 #include <variant>
@@ -89,6 +90,8 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){
 }; // note: if you change the enum members here, please update
    // docs/source/dev/design.rst
 
+std::ostream &operator<<(std::ostream &os, Operation op);
+
 namespace internal
 {
     /*

include/openPMD/IO/InvalidatableFile.hpp

@@ -82,4 +82,14 @@ struct hash<openPMD::InvalidatableFile>
 
     result_type operator()(argument_type const &s) const noexcept;
 };
+
+template <>
+struct less<openPMD::InvalidatableFile>
+{
+    using first_argument_type = openPMD::InvalidatableFile;
+    using second_argument_type = first_argument_type;
+    using result_type = typename std::less<std::string>::result_type;
+    result_type
+    operator()(first_argument_type const &, second_argument_type const &) const;
+};
 } // namespace std

include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

@@ -241,7 +241,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
 
     void touch(Writable *, Parameter<Operation::TOUCH> const &) override;
 
-    std::future<void> flush();
+    std::future<void> flush(internal::ParsedFlushParams &params);
 
 private:
 #if openPMD_HAVE_MPI

include/openPMD/Iteration.hpp

@@ -27,6 +27,7 @@
 #include "openPMD/auxiliary/Variant.hpp"
 #include "openPMD/backend/Attributable.hpp"
 #include "openPMD/backend/Container.hpp"
+#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp"
 
 #include <cstdint>
 #include <deque>
@@ -151,7 +152,9 @@ namespace internal
  * @see
  * https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#required-attributes-for-the-basepath
  */
-class Iteration : public Attributable
+class Iteration
+    : public Attributable
+    , internal::ScientificDefaults
 {
     template <typename T, typename T_key, typename T_container>
     friend class Container;
@@ -164,6 +167,7 @@ class Iteration : public Attributable
     friend class StatefulSnapshotsContainer;
     template <typename>
     friend struct traits::GenerationPolicy;
+    friend class internal::ScientificDefaults;
 
 public:
     Iteration(Iteration const &) = default;
@@ -279,6 +283,19 @@ class Iteration : public Attributable
     [[deprecated("This attribute is no longer set by the openPMD-api.")]] bool
     closedByWriter() const;
 
+    template <typename Visitor>
+    void visitHierarchy(Visitor &&v)
+    {
+        v(*this);
+        meshes.visitHierarchy(v);
+        particles.visitHierarchy(v);
+    }
+
+    // TODO: make this available on every class that inherits from
+    // scientificdefaults for this, somehow make visitHierarchy a non-template,
+    // so we can use it as a virtual function of attributable
+    void populateDefaultMetadata();
+
     Container<Mesh> meshes{};
     Container<ParticleSpecies> particles{}; // particleSpecies?
 
@@ -442,14 +459,17 @@ class Iteration : public Attributable
      *
      * @param w The Writable representing the parent.
      */
-    virtual void linkHierarchy(Writable &w);
+    void linkHierarchy(Writable &w) override;
 
     /**
      * @brief Access an iteration in read mode that has potentially not been
      *      parsed yet.
      *
      */
     void runDeferredParseAccess();
+
+protected:
+    void scientificDefaults_impl(bool write, OpenpmdStandard) override;
 }; // Iteration
 
 namespace traits

include/openPMD/Mesh.hpp

@@ -23,7 +23,9 @@
 #include "openPMD/UnitDimension.hpp"
 #include "openPMD/backend/Attributable.hpp"
 #include "openPMD/backend/BaseRecord.hpp"
+#include "openPMD/backend/Container.hpp"
 #include "openPMD/backend/MeshRecordComponent.hpp"
+#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp"
 
 #include <ostream>
 #include <string>
@@ -41,6 +43,7 @@ class Mesh : public BaseRecord<MeshRecordComponent>
 {
     friend class Container<Mesh>;
     friend class Iteration;
+    friend class internal::ScientificDefaults;
 
 public:
     Mesh(Mesh const &) = default;
@@ -328,6 +331,10 @@ class Mesh : public BaseRecord<MeshRecordComponent>
     void
     flush_impl(std::string const &, internal::FlushParams const &) override;
     void read();
+    auto retrieveDimensionality() const -> uint64_t;
+
+protected:
+    void scientificDefaults_impl(bool write, OpenpmdStandard) override;
 }; // Mesh
 
 template <typename T>

include/openPMD/ParticleSpecies.hpp

@@ -24,23 +24,34 @@
 #include "openPMD/Record.hpp"
 #include "openPMD/backend/Attributable.hpp"
 #include "openPMD/backend/Container.hpp"
+#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp"
 
 #include <string>
 
 namespace openPMD
 {
 
-class ParticleSpecies : public Container<Record>
+class ParticleSpecies
+    : public Container<Record>
+    , internal::ScientificDefaults
 {
     friend class Container<ParticleSpecies>;
     friend class Container<Record>;
     friend class Iteration;
     template <typename T>
     friend T &internal::makeOwning(T &self, Series);
+    friend class internal::ScientificDefaults;
 
 public:
     ParticlePatches particlePatches;
 
+    template <typename Visitor>
+    void visitHierarchy(Visitor &&v)
+    {
+        Container<Record>::visitHierarchy(v);
+        particlePatches.visitHierarchy(v);
+    }
+
 private:
     ParticleSpecies();
 
@@ -53,6 +64,9 @@ class ParticleSpecies : public Container<Record>
     {
         return m_containerData;
     }
+
+protected:
+    void scientificDefaults_impl(bool write, OpenpmdStandard) override;
 };
 
 namespace traits

include/openPMD/Record.hpp

@@ -23,6 +23,7 @@
 #include "openPMD/RecordComponent.hpp"
 #include "openPMD/UnitDimension.hpp"
 #include "openPMD/backend/BaseRecord.hpp"
+#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp"
 
 #include <string>
 #include <type_traits>
@@ -34,6 +35,7 @@ class Record : public BaseRecord<RecordComponent>
     friend class Container<Record>;
     friend class Iteration;
     friend class ParticleSpecies;
+    friend class internal::ScientificDefaults;
 
 public:
     Record(Record const &) = default;
@@ -55,6 +57,9 @@ class Record : public BaseRecord<RecordComponent>
     flush_impl(std::string const &, internal::FlushParams const &) override;
 
     [[nodiscard]] internal::HomogenizeExtents read();
+
+protected:
+    void scientificDefaults_impl(bool write, OpenpmdStandard) override;
 }; // Record
 
 template <typename T>

include/openPMD/RecordComponent.hpp

@@ -27,6 +27,7 @@
 #include "openPMD/auxiliary/UniquePtr.hpp"
 #include "openPMD/backend/Attributable.hpp"
 #include "openPMD/backend/BaseRecordComponent.hpp"
+#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp"
 
 // comment to prevent this include from being moved by clang-format
 #include "openPMD/DatatypeMacros.hpp"
@@ -110,7 +111,9 @@ namespace internal
 template <typename>
 class BaseRecord;
 
-class RecordComponent : public BaseRecordComponent
+class RecordComponent
+    : public BaseRecordComponent
+    , protected internal::ScientificDefaults
 {
     template <typename T, typename T_key, typename T_container>
     friend class Container;
@@ -128,6 +131,7 @@ class RecordComponent : public BaseRecordComponent
     friend class MeshRecordComponent;
     template <typename T>
     friend T &internal::makeOwning(T &self, Series);
+    friend class internal::ScientificDefaults;
 
 public:
     enum class Allocation
@@ -482,11 +486,17 @@ class RecordComponent : public BaseRecordComponent
     auto visit(Args &&...args) -> decltype(Visitor::template call<char>(
         std::declval<RecordComponent &>(), std::forward<Args>(args)...));
 
+    template <typename Visitor>
+    void visitHierarchy(Visitor &&v)
+    {
+        v(*this);
+    }
+
     static constexpr char const *const SCALAR = "\vScalar";
 
 protected:
     void flush(std::string const &, internal::FlushParams const &);
-    void read(bool require_unit_si);
+    void read();
 
 private:
     /**
@@ -534,12 +544,15 @@ OPENPMD_protected
         BaseRecordComponent::setData(m_recordComponentData);
     }
 
-    void readBase(bool require_unit_si);
+    void readBase();
 
     template <typename T>
     void verifyChunk(Offset const &, Extent const &) const;
 
     void verifyChunk(Datatype, Offset const &, Extent const &) const;
+
+protected:
+    void scientificDefaults_impl(bool write, OpenpmdStandard) override;
 }; // RecordComponent
 
 namespace internal