diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ada05698a..194bbd8881 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -426,6 +426,11 @@ set(CORE_SOURCE src/backend/PatchRecord.cpp src/backend/PatchRecordComponent.cpp src/backend/Writable.cpp + src/backend/scientific_defaults/ScientificDefaults.cpp + src/backend/scientific_defaults/ScientificDefaults_auxiliary.cpp + src/backend/scientific_defaults/ProcessParsedAttribute.cpp + src/backend/scientific_defaults/AttributeReader.cpp + src/backend/scientific_defaults/ConfigAttribute.cpp src/auxiliary/OneDimensionalBlockSlicer.cpp src/helper/list_series.cpp src/snapshots/ContainerImpls.cpp diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 9b7735b5ba..b54b661c78 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -26,6 +26,7 @@ #include "openPMD/IterationEncoding.hpp" #include "openPMD/config.hpp" #include "openPMD/version.hpp" +#include #if openPMD_HAVE_MPI #include @@ -81,6 +82,66 @@ enum class FlushLevel CreateOrOpenFiles }; +std::ostream &operator<<(std::ostream &, FlushLevel); + +namespace flush_level +{ + inline constexpr auto global_flushpoint(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + // same as global_flushpoint for now, but we will soon introduce + // immediate_flush + inline constexpr auto write_datasets(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto write_attributes(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + return true; + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto flush_hierarchy(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + return true; + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } +} // namespace flush_level + enum class OpenpmdStandard { v_1_0_0, @@ -121,6 +182,7 @@ namespace internal * To be used for reading */ FlushParams const defaultFlushParams{}; + FlushParams const publicFlush{FlushLevel::UserFlush}; struct ParsedFlushParams; diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index d45ce1bdcc..fdb8af3599 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -39,7 +39,7 @@ class AbstractIOHandlerImpl virtual ~AbstractIOHandlerImpl() = default; - std::future flush(); + std::future flush(FlushLevel); /** * Close the file corresponding with the writable and release file handles. diff --git a/include/openPMD/IO/AbstractIOHandlerImplCommon.hpp b/include/openPMD/IO/AbstractIOHandlerImplCommon.hpp index 7261b4bf71..4c995be817 100644 --- a/include/openPMD/IO/AbstractIOHandlerImplCommon.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImplCommon.hpp @@ -28,10 +28,10 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/backend/Writable.hpp" +#include #include #include #include -#include namespace openPMD { @@ -50,7 +50,10 @@ class AbstractIOHandlerImplCommon : public AbstractIOHandlerImpl * without the OS path */ std::unordered_map m_files; - std::unordered_set m_dirty; + // MUST be an ordered set in order to consistently flush on different + // parallel processes (same logic cant apply to m_files since Writable* + // pointers are not predictable) + std::set m_dirty; enum PossiblyExisting { diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 25e0d6ad54..a8efab571e 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -89,6 +90,8 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){ }; // note: if you change the enum members here, please update // docs/source/dev/design.rst +std::ostream &operator<<(std::ostream &os, Operation op); + namespace internal { /* diff --git a/include/openPMD/IO/InvalidatableFile.hpp b/include/openPMD/IO/InvalidatableFile.hpp index 6bdc24cbe6..6ac0051cbe 100644 --- a/include/openPMD/IO/InvalidatableFile.hpp +++ b/include/openPMD/IO/InvalidatableFile.hpp @@ -82,4 +82,14 @@ struct hash result_type operator()(argument_type const &s) const noexcept; }; + +template <> +struct less +{ + using first_argument_type = openPMD::InvalidatableFile; + using second_argument_type = first_argument_type; + using result_type = typename std::less::result_type; + result_type + operator()(first_argument_type const &, second_argument_type const &) const; +}; } // namespace std diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 6df0c60ced..3e0758aee2 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -241,7 +241,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl void touch(Writable *, Parameter const &) override; - std::future flush(); + std::future flush(internal::ParsedFlushParams ¶ms); private: #if openPMD_HAVE_MPI diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index e2a4e6b1a5..38f33ff2f6 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -27,6 +27,7 @@ #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include #include @@ -151,7 +152,9 @@ namespace internal * @see * https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#required-attributes-for-the-basepath */ -class Iteration : public Attributable +class Iteration + : public Attributable + , internal::ScientificDefaults { template friend class Container; @@ -164,6 +167,7 @@ class Iteration : public Attributable friend class StatefulSnapshotsContainer; template friend struct traits::GenerationPolicy; + friend class internal::ScientificDefaults; public: Iteration(Iteration const &) = default; @@ -279,6 +283,19 @@ class Iteration : public Attributable [[deprecated("This attribute is no longer set by the openPMD-api.")]] bool closedByWriter() const; + template + void visitHierarchy(Visitor &&v) + { + v(*this); + meshes.visitHierarchy(v); + particles.visitHierarchy(v); + } + + // TODO: make this available on every class that inherits from + // scientificdefaults for this, somehow make visitHierarchy a non-template, + // so we can use it as a virtual function of attributable + void populateDefaultMetadata(); + Container meshes{}; Container particles{}; // particleSpecies? @@ -442,7 +459,7 @@ class Iteration : public Attributable * * @param w The Writable representing the parent. */ - virtual void linkHierarchy(Writable &w); + void linkHierarchy(Writable &w) override; /** * @brief Access an iteration in read mode that has potentially not been @@ -450,6 +467,9 @@ class Iteration : public Attributable * */ void runDeferredParseAccess(); + +protected: + void scientificDefaults_impl(bool write, OpenpmdStandard) override; }; // Iteration namespace traits diff --git a/include/openPMD/Mesh.hpp b/include/openPMD/Mesh.hpp index d0bf81ddef..dacd4cf2eb 100644 --- a/include/openPMD/Mesh.hpp +++ b/include/openPMD/Mesh.hpp @@ -23,7 +23,9 @@ #include "openPMD/UnitDimension.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/Container.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include #include @@ -41,6 +43,7 @@ class Mesh : public BaseRecord { friend class Container; friend class Iteration; + friend class internal::ScientificDefaults; public: Mesh(Mesh const &) = default; @@ -328,6 +331,10 @@ class Mesh : public BaseRecord void flush_impl(std::string const &, internal::FlushParams const &) override; void read(); + auto retrieveDimensionality() const -> uint64_t; + +protected: + void scientificDefaults_impl(bool write, OpenpmdStandard) override; }; // Mesh template diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index 7309afddef..c654821c1e 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -24,23 +24,34 @@ #include "openPMD/Record.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include namespace openPMD { -class ParticleSpecies : public Container +class ParticleSpecies + : public Container + , internal::ScientificDefaults { friend class Container; friend class Container; friend class Iteration; template friend T &internal::makeOwning(T &self, Series); + friend class internal::ScientificDefaults; public: ParticlePatches particlePatches; + template + void visitHierarchy(Visitor &&v) + { + Container::visitHierarchy(v); + particlePatches.visitHierarchy(v); + } + private: ParticleSpecies(); @@ -53,6 +64,9 @@ class ParticleSpecies : public Container { return m_containerData; } + +protected: + void scientificDefaults_impl(bool write, OpenpmdStandard) override; }; namespace traits diff --git a/include/openPMD/Record.hpp b/include/openPMD/Record.hpp index 2bb070365f..bcb812dbdb 100644 --- a/include/openPMD/Record.hpp +++ b/include/openPMD/Record.hpp @@ -23,6 +23,7 @@ #include "openPMD/RecordComponent.hpp" #include "openPMD/UnitDimension.hpp" #include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include #include @@ -34,6 +35,7 @@ class Record : public BaseRecord friend class Container; friend class Iteration; friend class ParticleSpecies; + friend class internal::ScientificDefaults; public: Record(Record const &) = default; @@ -55,6 +57,9 @@ class Record : public BaseRecord flush_impl(std::string const &, internal::FlushParams const &) override; [[nodiscard]] internal::HomogenizeExtents read(); + +protected: + void scientificDefaults_impl(bool write, OpenpmdStandard) override; }; // Record template diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index d06b4213f4..53cd947995 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -27,6 +27,7 @@ #include "openPMD/auxiliary/UniquePtr.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" // comment to prevent this include from being moved by clang-format #include "openPMD/DatatypeMacros.hpp" @@ -110,7 +111,9 @@ namespace internal template class BaseRecord; -class RecordComponent : public BaseRecordComponent +class RecordComponent + : public BaseRecordComponent + , protected internal::ScientificDefaults { template friend class Container; @@ -128,6 +131,7 @@ class RecordComponent : public BaseRecordComponent friend class MeshRecordComponent; template friend T &internal::makeOwning(T &self, Series); + friend class internal::ScientificDefaults; public: enum class Allocation @@ -482,11 +486,17 @@ class RecordComponent : public BaseRecordComponent auto visit(Args &&...args) -> decltype(Visitor::template call( std::declval(), std::forward(args)...)); + template + void visitHierarchy(Visitor &&v) + { + v(*this); + } + static constexpr char const *const SCALAR = "\vScalar"; protected: void flush(std::string const &, internal::FlushParams const &); - void read(bool require_unit_si); + void read(); private: /** @@ -534,12 +544,15 @@ OPENPMD_protected BaseRecordComponent::setData(m_recordComponentData); } - void readBase(bool require_unit_si); + void readBase(); template void verifyChunk(Offset const &, Extent const &) const; void verifyChunk(Datatype, Offset const &, Extent const &) const; + +protected: + void scientificDefaults_impl(bool write, OpenpmdStandard) override; }; // RecordComponent namespace internal diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index b796ab1a93..523f4f1e41 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -90,6 +90,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) { size *= ext; } + /* * Flushing the skeleton does not create datasets, * so we might need to do it now. @@ -129,7 +130,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) * actual data yet. */ seriesFlush_impl( - {FlushLevel::SkeletonOnly}); + {FlushLevel::SkeletonOnly}, /*flush_io_handler=*/false); Parameter dCreate(rc.m_dataset.value()); dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; IOHandler()->enqueue(IOTask(this, dCreate)); diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 603e540c2b..18c6b78e26 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -778,6 +778,13 @@ class Series : public Attributable */ void close(); + template + void visitHierarchy(Visitor &&v) + { + v(*this); + get().iterations.visitHierarchy(v); + } + /** * This overrides Attributable::iterationFlush() which will fail on Series. */ @@ -898,9 +905,7 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushMeshesPath(); - void flushParticlesPath(); - void flushRankTable(); + void flushRankTable(FlushLevel); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. */ @@ -984,7 +989,7 @@ OPENPMD_private * * @param doFlush If true, flush the IO handler. */ - void flushStep(bool doFlush); + void flushStep(bool doFlush, FlushLevel l); /* * setIterationEncoding() should only be called by users of our public API, diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index faf2b29472..e373873a4f 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -124,6 +124,22 @@ namespace detail { constexpr static bool value = true; }; + + template + struct ScalarType + { + using type = T; + }; + template + struct ScalarType> + { + using type = T; + }; + template + struct ScalarType> + { + using type = T; + }; } // namespace detail template @@ -141,6 +157,12 @@ using IsPointer_t = typename detail::IsPointer::type; template inline constexpr bool IsChar_v = detail::IsChar::value; +template +using ScalarType_t = typename detail::ScalarType::type; + +template +using VectorType_t = std::vector>; + /** Emulate in the C++ concept ContiguousContainer * * Users can implement this trait for a type to signal it can be used as @@ -203,6 +225,34 @@ namespace detail // little trick to avoid trailing commas in the macro expansions below template using variant_tail_t = std::variant; + + template