diff --git a/CMakeLists.txt b/CMakeLists.txt index 5fa5b1a0ca..587a2455a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -400,6 +400,7 @@ set(CORE_SOURCE src/Format.cpp src/Iteration.cpp src/IterationEncoding.cpp + src/LoadStoreChunk.cpp src/Mesh.cpp src/ParticlePatches.cpp src/ParticleSpecies.cpp @@ -411,6 +412,7 @@ set(CORE_SOURCE src/version.cpp src/auxiliary/Date.cpp src/auxiliary/Filesystem.cpp + src/auxiliary/Future.cpp src/auxiliary/JSON.cpp src/auxiliary/JSONMatcher.cpp src/auxiliary/Memory.cpp @@ -426,6 +428,7 @@ set(CORE_SOURCE src/backend/PatchRecord.cpp src/backend/PatchRecordComponent.cpp src/backend/Writable.cpp + src/backend/ScientificDefaults.cpp src/auxiliary/OneDimensionalBlockSlicer.cpp src/helper/list_series.cpp src/snapshots/ContainerImpls.cpp diff --git a/include/openPMD/Dataset.hpp b/include/openPMD/Dataset.hpp index e1f0058885..27195abb4d 100644 --- a/include/openPMD/Dataset.hpp +++ b/include/openPMD/Dataset.hpp @@ -34,6 +34,12 @@ namespace openPMD using Extent = std::vector; using Offset = std::vector; +struct MemorySelection +{ + Offset offset; + Extent extent; +}; + class Dataset { friend class RecordComponent; diff --git a/include/openPMD/Datatype.hpp b/include/openPMD/Datatype.hpp index a11d3db75f..579c7c665d 100644 --- a/include/openPMD/Datatype.hpp +++ b/include/openPMD/Datatype.hpp @@ -294,7 +294,8 @@ template inline constexpr Datatype determineDatatype(T &&val) { (void)val; // don't need this, it only has a name for Doxygen - using T_stripped = std::remove_cv_t>; + using T_stripped = + std::remove_extent_t>>; if constexpr (auxiliary::IsPointer_v) { return determineDatatype>(); diff --git a/include/openPMD/IO/ADIOS/ADIOS2File.hpp b/include/openPMD/IO/ADIOS/ADIOS2File.hpp index d34cc8ebe5..0b7aa6f314 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2File.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2File.hpp @@ -20,6 +20,7 @@ */ #pragma once +#include "openPMD/Dataset.hpp" #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" #include "openPMD/IO/ADIOS/ADIOS2PreloadVariables.hpp" @@ -112,6 +113,7 @@ struct BufferedUniquePtrPut std::string name; Offset offset; Extent extent; + std::optional memorySelection; UniquePtrWithLambda data; Datatype dtype = Datatype::UNDEFINED; diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 4316f5181f..e20c256a67 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -21,6 +21,7 @@ */ #pragma once +#include "openPMD/Dataset.hpp" #include "openPMD/Error.hpp" #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" @@ -30,6 +31,7 @@ #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp" +#include "openPMD/IO/AbstractIOHandler_internal.hpp" #include "openPMD/IO/FlushParametersInternal.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IO/InvalidatableFile.hpp" @@ -509,6 +511,7 @@ class ADIOS2IOHandlerImpl adios2::Variable verifyDataset( Offset const &offset, Extent const &extent, + std::optional const &memorySelection, adios2::IO &IO, adios2::Engine &engine, std::string const &varName, @@ -622,6 +625,18 @@ class ADIOS2IOHandlerImpl var.SetSelection( {adios2::Dims(offset.begin(), offset.end()), adios2::Dims(extent.begin(), extent.end())}); + + if (memorySelection.has_value()) + { + var.SetMemorySelection( + {adios2::Dims( + memorySelection->offset.begin(), + memorySelection->offset.end()), + adios2::Dims( + memorySelection->extent.begin(), + memorySelection->extent.end())}); + } + return var; } @@ -629,6 +644,7 @@ class ADIOS2IOHandlerImpl { bool noGroupBased = false; bool blosc2bp5 = false; + bool memorySelection = false; } printedWarningsAlready; }; // ADIOS2IOHandlerImpl @@ -942,7 +958,7 @@ class ADIOS2IOHandler : public AbstractIOHandler try { auto params = internal::defaultParsedFlushParams; - this->flush(params); + this->flush_impl(params); } catch (std::exception const &ex) { @@ -962,9 +978,7 @@ class ADIOS2IOHandler : public AbstractIOHandler #if openPMD_HAVE_MPI ADIOS2IOHandler( - std::optional> initialize_from, - std::string path, - Access, + internal::AbstractIOHandlerInitFrom &&initialize_from, MPI_Comm, json::TracingJSON options, std::string engineType, @@ -973,9 +987,7 @@ class ADIOS2IOHandler : public AbstractIOHandler #endif ADIOS2IOHandler( - std::optional> initialize_from, - std::string path, - Access, + internal::AbstractIOHandlerInitFrom &&initialize_from, json::TracingJSON options, std::string engineType, std::string specifiedExtension); @@ -990,6 +1002,6 @@ class ADIOS2IOHandler : public AbstractIOHandler return true; } - std::future flush(internal::ParsedFlushParams &) override; + std::future flush_impl(internal::ParsedFlushParams &) override; }; // ADIOS2IOHandler } // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/macros.hpp b/include/openPMD/IO/ADIOS/macros.hpp index 8e57d9191d..47b38912cc 100644 --- a/include/openPMD/IO/ADIOS/macros.hpp +++ b/include/openPMD/IO/ADIOS/macros.hpp @@ -41,6 +41,29 @@ #define openPMD_HAVE_ADIOS2_BP5 0 #endif +namespace openPMD +{ +namespace detail +{ + template + struct CanTheMemorySelectionBeReset + { + static constexpr bool value = false; + }; + + template + struct CanTheMemorySelectionBeReset< + Variable, + decltype(std::declval().SetMemorySelection())> + { + static constexpr bool value = true; + }; +} // namespace detail + +constexpr bool CanTheMemorySelectionBeReset = + detail::CanTheMemorySelectionBeReset>::value; +} // namespace openPMD + #else #define openPMD_HAS_ADIOS_2_8 0 diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 9b7735b5ba..ae4078ec94 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -59,6 +59,11 @@ enum class FlushLevel * This mode defines a flush point (see docs/source/usage/workflow.rst.rst). */ UserFlush, + /** + * Flush triggered by storeChunk in immediate flush mode. + * Must not perform operations enqueued in m_chunks. + */ + ImmediateFlush, /** * Default mode, used when flushes are triggered internally, e.g. during * parsing to read attributes. Does not trigger a flush point. @@ -81,6 +86,66 @@ enum class FlushLevel CreateOrOpenFiles }; +namespace flush_level +{ + inline constexpr auto global_flushpoint(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::ImmediateFlush: + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto write_datasets(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::ImmediateFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto write_attributes(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::ImmediateFlush: + case FlushLevel::InternalFlush: + return true; + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto flush_hierarchy(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::ImmediateFlush: + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + return true; + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } +} // namespace flush_level + enum class OpenpmdStandard { v_1_0_0, @@ -187,12 +252,64 @@ namespace internal return res; } } + + /************************************************************************** + * Since the AbstractIOHandler is linked to every object of the + * frontend, it stores a number of members that are needed by methods + * traversing the object hierarchy. Those members are found in this struct + * from which AbstractIOHandler derives. + **************************************************************************/ + struct GlobalParameters + { + GlobalParameters(Access at); + + std::string directory; + /* + * Originally, the reason for distinguishing these two was that during + * parsing in reading access modes, the access type would be temporarily + * const_cast'ed to an access type that would support modifying + * the openPMD object model. Then, it would be const_cast'ed back to + * READ_ONLY, to disable further modifications. + * Due to this approach's tendency to cause subtle bugs, and due to its + * difficult debugging properties, this was replaced by the SeriesStatus + * enum, defined in this file. + * The distinction of backendAccess and frontendAccess stays relevant, + * since the frontend can use it in order to pretend to the backend that + * another access type is being used. This is used by the file-based + * append mode, which is entirely implemented by the frontend, which + * internally uses the backend in CREATE mode. + */ + Access m_backendAccess; + Access m_frontendAccess; + + /** + * This is to avoid that the destructor tries flushing again if an error + * happened. Otherwise, this would lead to confusing error messages. + * Initialized as false, set to true after successful construction. + * If flushing results in an error, set this back to false. + * The destructor will only attempt flushing again if this is true. + */ + bool m_lastFlushSuccessful = false; + internal::SeriesStatus m_seriesStatus = internal::SeriesStatus::Default; + IterationEncoding m_encoding = IterationEncoding::groupBased; + OpenpmdStandard m_standard = + auxiliary::parseStandard(getStandardDefault()); + bool m_verify_homogeneous_extents = true; + // If true, then flush directly upon storeChunk + bool m_flush_immediately = false; + + protected: + explicit GlobalParameters(); + }; } // namespace internal namespace detail { class ADIOS2File; -} + struct InitFrom_Tag + {}; + constexpr InitFrom_Tag InitFrom_Tag_v; +} // namespace detail /** Interface for communicating between logical and physically persistent data. * @@ -202,7 +319,7 @@ namespace detail * scenarios it is therefore necessary to manually execute all operations * by calling AbstractIOHandler::flush(). */ -class AbstractIOHandler +class AbstractIOHandler : public internal::GlobalParameters { friend class Series; friend class ADIOS2IOHandlerImpl; @@ -221,23 +338,16 @@ class AbstractIOHandler public: #if openPMD_HAVE_MPI - template + template AbstractIOHandler( - std::optional> initialize_from, - std::string path, - Access at, - TracingJSON &&jsonConfig, - MPI_Comm); + InitFrom &&initialize_from, TracingJSON &&jsonConfig, MPI_Comm); #endif - template - AbstractIOHandler( - std::optional> initialize_from, - std::string path, - Access at, - TracingJSON &&jsonConfig); + template + AbstractIOHandler(InitFrom &&initialize_from, TracingJSON &&jsonConfig); - AbstractIOHandler(std::optional>); + template + AbstractIOHandler(detail::InitFrom_Tag, InitFrom &&); virtual ~AbstractIOHandler(); @@ -265,56 +375,23 @@ class AbstractIOHandler * backends that decide to implement this operation asynchronously. */ std::future flush(internal::FlushParams const &); + std::queue m_work; + std::shared_ptr m_flushCounter = + std::make_shared(0); /** Process operations in queue according to FIFO. * * @return Future indicating the completion state of the operation for * backends that decide to implement this operation asynchronously. */ - virtual std::future flush(internal::ParsedFlushParams &) = 0; + std::future flush(internal::ParsedFlushParams &); /** The currently used backend */ virtual std::string backendName() const = 0; virtual bool fullSupportForVariableBasedEncoding() const; - std::string directory; - /* - * Originally, the reason for distinguishing these two was that during - * parsing in reading access modes, the access type would be temporarily - * const_cast'ed to an access type that would support modifying - * the openPMD object model. Then, it would be const_cast'ed back to - * READ_ONLY, to disable further modifications. - * Due to this approach's tendency to cause subtle bugs, and due to its - * difficult debugging properties, this was replaced by the SeriesStatus - * enum, defined in this file. - * The distinction of backendAccess and frontendAccess stays relevant, since - * the frontend can use it in order to pretend to the backend that another - * access type is being used. This is used by the file-based append mode, - * which is entirely implemented by the frontend, which internally uses - * the backend in CREATE mode. - */ - Access m_backendAccess; - Access m_frontendAccess; - std::queue m_work; - - /************************************************************************** - * Since the AbstractIOHandler is linked to every object of the frontend, * - * it stores a number of members that are needed by methods traversing * - * the object hierarchy. Those members are found below. * - **************************************************************************/ - - /** - * This is to avoid that the destructor tries flushing again if an error - * happened. Otherwise, this would lead to confusing error messages. - * Initialized as false, set to true after successful construction. - * If flushing results in an error, set this back to false. - * The destructor will only attempt flushing again if this is true. - */ - bool m_lastFlushSuccessful = false; - internal::SeriesStatus m_seriesStatus = internal::SeriesStatus::Default; - IterationEncoding m_encoding = IterationEncoding::groupBased; - OpenpmdStandard m_standard = auxiliary::parseStandard(getStandardDefault()); - bool m_verify_homogeneous_extents = true; +protected: + virtual std::future flush_impl(internal::ParsedFlushParams &) = 0; }; // AbstractIOHandler } // namespace openPMD diff --git a/include/openPMD/IO/AbstractIOHandlerHelper.hpp b/include/openPMD/IO/AbstractIOHandlerHelper.hpp index d86911ad35..c50f2c82d1 100644 --- a/include/openPMD/IO/AbstractIOHandlerHelper.hpp +++ b/include/openPMD/IO/AbstractIOHandlerHelper.hpp @@ -33,10 +33,6 @@ namespace openPMD * @param initialize_from Optionally initialize the IOHandler from a previous * interim IOHandler which to replace with the handler now * being initialized. - * @param path Path to root folder for all operations associated with - the desired handler. - * @param access Access mode describing desired operations and - permissions of the desired handler. * @param format Format describing the IO backend of the desired handler. * @param originalExtension The filename extension as it was originally * specified by the user. @@ -48,11 +44,9 @@ namespace openPMD including nlohmann::json in a .hpp file. * @return Smart pointer to created IOHandler. */ -template +template std::unique_ptr createIOHandler( - std::optional> initialize_from, - std::string path, - Access access, + Init_t &&initialize_from, Format format, std::string originalExtension, MPI_Comm comm, @@ -65,10 +59,6 @@ std::unique_ptr createIOHandler( * @param initialize_from Optionally initialize the IOHandler from a previous * interim IOHandler which to replace with the handler now * being initialized. - * @param path Path to root folder for all operations associated with - * the desired handler. - * @param access Access describing desired operations and permissions - * of the desired handler. * @param format Format describing the IO backend of the desired handler. * @param originalExtension The filename extension as it was originally * specified by the user. @@ -79,21 +69,16 @@ std::unique_ptr createIOHandler( including nlohmann::json in a .hpp file. * @return Smart pointer to created IOHandler. */ -template +template std::unique_ptr createIOHandler( - std::optional> initialize_from, - std::string path, - Access access, + Init_t &&initialize_from, Format format, std::string originalExtension, JSON options, std::string const &pathAsItWasSpecifiedInTheConstructor); // version without configuration to use in AuxiliaryTest +template std::unique_ptr createIOHandler( - std::optional> initialize_from, - std::string path, - Access access, - Format format, - std::string originalExtension); + Init_t &&initialize_from, Format format, std::string originalExtension); } // namespace openPMD diff --git a/include/openPMD/IO/AbstractIOHandler_internal.hpp b/include/openPMD/IO/AbstractIOHandler_internal.hpp new file mode 100644 index 0000000000..7caa2f3b6f --- /dev/null +++ b/include/openPMD/IO/AbstractIOHandler_internal.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include "openPMD/IO/AbstractIOHandler.hpp" + +#include + +namespace openPMD::internal +{ +struct AbstractIOHandlerInitFrom + : std::variant +{ + using Left = GlobalParameters; + using Right = AbstractIOHandler *; + using parent_t = std::variant; + using parent_t::parent_t; + inline auto as_parent() -> parent_t & + { + return *this; + } + [[nodiscard]] inline auto as_parent() const -> parent_t const & + { + return *this; + } + auto asGlobalParameters() -> GlobalParameters &; + [[nodiscard]] auto asGlobalParameters() const -> GlobalParameters const &; +}; +} // namespace openPMD::internal diff --git a/include/openPMD/IO/DummyIOHandler.hpp b/include/openPMD/IO/DummyIOHandler.hpp index 8abcf20990..dfc91215e5 100644 --- a/include/openPMD/IO/DummyIOHandler.hpp +++ b/include/openPMD/IO/DummyIOHandler.hpp @@ -21,6 +21,7 @@ #pragma once #include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/AbstractIOHandler_internal.hpp" #include "openPMD/IO/Access.hpp" #include "openPMD/IO/IOTask.hpp" @@ -34,7 +35,7 @@ namespace openPMD class DummyIOHandler : public AbstractIOHandler { public: - DummyIOHandler(std::string, Access); + DummyIOHandler(internal::AbstractIOHandlerInitFrom &&); ~DummyIOHandler() override = default; /** No-op consistent with the IOHandler interface to enable library use @@ -44,7 +45,7 @@ class DummyIOHandler : public AbstractIOHandler /** No-op consistent with the IOHandler interface to enable library use * without IO. */ - std::future flush(internal::ParsedFlushParams &) override; + std::future flush_impl(internal::ParsedFlushParams &) override; std::string backendName() const override; }; // DummyIOHandler } // namespace openPMD diff --git a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp index 07b3978b87..ba8daaa048 100644 --- a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp @@ -21,6 +21,7 @@ #pragma once #include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/AbstractIOHandler_internal.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" #include @@ -35,9 +36,7 @@ class HDF5IOHandler : public AbstractIOHandler { public: HDF5IOHandler( - std::optional> initialize_from, - std::string path, - Access, + internal::AbstractIOHandlerInitFrom &&initialize_from, json::TracingJSON config); ~HDF5IOHandler() override; @@ -46,7 +45,7 @@ class HDF5IOHandler : public AbstractIOHandler return "HDF5"; } - std::future flush(internal::ParsedFlushParams &) override; + std::future flush_impl(internal::ParsedFlushParams &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp index abeb196b11..d8a69d479e 100644 --- a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp @@ -21,6 +21,7 @@ #pragma once #include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/AbstractIOHandler_internal.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/config.hpp" @@ -37,16 +38,12 @@ class ParallelHDF5IOHandler : public AbstractIOHandler public: #if openPMD_HAVE_MPI ParallelHDF5IOHandler( - std::optional> initialize_from, - std::string path, - Access, + internal::AbstractIOHandlerInitFrom &&initialize_from, MPI_Comm, json::TracingJSON config); #else ParallelHDF5IOHandler( - std::optional> initialize_from, - std::string const &path, - Access, + internal::AbstractIOHandlerInitFrom &&initialize_from, json::TracingJSON config); #endif ~ParallelHDF5IOHandler() override; @@ -56,7 +53,7 @@ class ParallelHDF5IOHandler : public AbstractIOHandler return "MPI_HDF5"; } - std::future flush(internal::ParsedFlushParams &) override; + std::future flush_impl(internal::ParsedFlushParams &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 1ee7248b32..f25fc389f8 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -495,6 +495,7 @@ struct OPENPMDAPI_EXPORT Extent extent = {}; Offset offset = {}; + std::optional memorySelection = std::nullopt; Datatype dtype = Datatype::UNDEFINED; auxiliary::WriteBuffer data; }; diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 07e797d4b3..c0821a17fb 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -22,6 +22,7 @@ #pragma once #include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/AbstractIOHandler_internal.hpp" #include "openPMD/IO/JSON/JSONIOHandlerImpl.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" @@ -35,17 +36,13 @@ class JSONIOHandler : public AbstractIOHandler { public: JSONIOHandler( - std::optional> initialize_from, - std::string path, - Access at, + internal::AbstractIOHandlerInitFrom &&initialize_from, openPMD::json::TracingJSON config, JSONIOHandlerImpl::FileFormat, std::string originalExtension); #if openPMD_HAVE_MPI JSONIOHandler( - std::optional> initialize_from, - std::string path, - Access at, + internal::AbstractIOHandlerInitFrom &&initialize_from, MPI_Comm, openPMD::json::TracingJSON config, JSONIOHandlerImpl::FileFormat, @@ -59,7 +56,7 @@ class JSONIOHandler : public AbstractIOHandler return "JSON"; } - std::future flush(internal::ParsedFlushParams &) override; + std::future flush_impl(internal::ParsedFlushParams &) override; private: JSONIOHandlerImpl m_impl; diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index c66199c46f..4aed643472 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -27,6 +27,7 @@ #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include "openPMD/backend/ScientificDefaults.hpp" #include #include @@ -142,7 +143,9 @@ namespace internal * @see * https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#required-attributes-for-the-basepath */ -class Iteration : public Attributable +class Iteration + : public Attributable + , internal::ScientificDefaults { template friend class Container; @@ -153,6 +156,7 @@ class Iteration : public Attributable friend class Writable; friend class StatefulIterator; friend class StatefulSnapshotsContainer; + friend class internal::ScientificDefaults; public: Iteration(Iteration const &) = default; diff --git a/include/openPMD/LoadStoreChunk.hpp b/include/openPMD/LoadStoreChunk.hpp new file mode 100644 index 0000000000..e579c54ae2 --- /dev/null +++ b/include/openPMD/LoadStoreChunk.hpp @@ -0,0 +1,305 @@ +#pragma once + +#include "openPMD/Dataset.hpp" +#include "openPMD/auxiliary/Future.hpp" +#include "openPMD/auxiliary/ShareRawInternal.hpp" +#include "openPMD/auxiliary/UniquePtr.hpp" + +// comment to prevent this include from being moved by clang-format +#include "openPMD/DatatypeMacros.hpp" + +#include +#include +#include +#include + +namespace openPMD +{ +class RecordComponent; +template +class ConfigureStoreChunkFromBuffer; +template +class ConfigureLoadStoreFromBuffer; +template +class DynamicMemoryView; +class Attributable; + +namespace internal +{ + struct LoadStoreConfig + { + Offset offset; + Extent extent; + }; + struct LoadStoreConfigWithBuffer + { + Offset offset; + Extent extent; + std::optional memorySelection; + }; + +} // namespace internal + +namespace auxiliary::detail +{ +#define OPENPMD_ENUMERATE_TYPES(type) , std::shared_ptr + using shared_ptr_dataset_types = auxiliary::detail::variant_tail_t< + auxiliary::detail::bottom OPENPMD_FOREACH_DATASET_DATATYPE( + OPENPMD_ENUMERATE_TYPES)>; +#undef OPENPMD_ENUMERATE_TYPES +} // namespace auxiliary::detail + +namespace compose +{ + template + class ConfigureLoadStore; + template + class ConfigureLoadStoreFromBuffer; +} // namespace compose + +enum class EnqueuePolicy +{ + Defer, + Immediate +}; + +namespace core +{ + /* + * Actual data members of `ConfigureLoadStore<>` and methods that don't + * depend on the ChildClass template parameter. By extracting the members to + * this struct, we can pass them around between different instances of the + * class template. Numbers of method instantiations can be reduced. + */ + class ConfigureLoadStore + { + template + friend class compose::ConfigureLoadStore; + template + friend class compose::ConfigureLoadStoreFromBuffer; + + protected: + ConfigureLoadStore(RecordComponent &); + RecordComponent &m_rc; + + std::optional m_offset; + std::optional m_extent; + + [[nodiscard]] auto dim() const -> uint8_t; + auto storeChunkConfig() -> internal::LoadStoreConfig; + + auto deferFlush(Attributable &); + + public: + auto getOffset() -> Offset const &; + auto getExtent() -> Extent const &; + /* + * If the type is non-const, then the return type should be + * ConfigureLoadStoreFromBuffer<>, ... + */ + template + struct shared_ptr_return_type_impl + { + using type = ConfigureLoadStoreFromBuffer< + std::shared_ptr>>; + }; + /* + * ..., but if it is a const type, Load operations make no sense, so the + * return type should be ConfigureStoreChunkFromBuffer<>. + */ + template + struct shared_ptr_return_type_impl + { + using type = ConfigureStoreChunkFromBuffer< + std::shared_ptr const>>; + }; + + template + using shared_ptr_return_type = + typename shared_ptr_return_type_impl>::type; + + /* + * As loading into unique pointer types makes no sense, the case is + * simpler for unique pointers. Just remove the array extents here. + */ + template + using unique_ptr_return_type = ConfigureStoreChunkFromBuffer< + UniquePtrWithLambda>>; + + // @todo rvalue references..? + template + auto withSharedPtr(std::shared_ptr) -> shared_ptr_return_type; + template + auto withUniquePtr(UniquePtrWithLambda) -> unique_ptr_return_type; + template + auto withUniquePtr(std::unique_ptr) + -> unique_ptr_return_type; + template + auto withRawPtr(T *data) -> shared_ptr_return_type; + template + auto withContiguousContainer(T_ContiguousContainer &data) + -> std::enable_if_t< + auxiliary::IsContiguousContainer_v, + shared_ptr_return_type< + typename T_ContiguousContainer::value_type>>; + + template + [[nodiscard]] auto enqueueStore() -> DynamicMemoryView; + // definition for this one is in RecordComponent.tpp since it needs the + // definition of class RecordComponent. + template + [[nodiscard]] auto enqueueStore(F &&createBuffer) + -> DynamicMemoryView; + + template + [[nodiscard]] auto enqueueLoad() + -> auxiliary::DeferredComputation>; + + template + [[nodiscard]] auto load(EnqueuePolicy) -> std::shared_ptr; + + [[nodiscard]] auto enqueueLoadVariant() + -> auxiliary::DeferredComputation< + auxiliary::detail::shared_ptr_dataset_types>; + + [[nodiscard]] auto loadVariant(EnqueuePolicy) + -> auxiliary::detail::shared_ptr_dataset_types; + }; + + template + class ConfigureStoreChunkFromBuffer : public ConfigureLoadStore + { + public: + Ptr_Type m_buffer; + std::optional m_mem_select; + + ConfigureStoreChunkFromBuffer(Ptr_Type buffer, ConfigureLoadStore &&); + + auto storeChunkConfig() -> internal::LoadStoreConfigWithBuffer; + + auto enqueueStore() -> auxiliary::DeferredComputation; + + auto store(EnqueuePolicy) -> void; + + /** This intentionally shadows the parent class's enqueueLoad methods in + * order to show a compile error when using enqueueLoad() on an object + * of this class. The parent method can still be accessed through + * as_parent() if needed. + */ + template + auto enqueueLoad() + { + static_assert( + auxiliary::dependent_false_v, + "Cannot load chunk data into a buffer that is const or a " + "unique_ptr."); + } + + template + auto load(EnqueuePolicy) + { + static_assert( + auxiliary::dependent_false_v, + "Cannot load chunk data into a buffer that is const or a " + "unique_ptr."); + } + }; + + template + class ConfigureLoadStoreFromBuffer + : public ConfigureStoreChunkFromBuffer + { + public: + using ConfigureStoreChunkFromBuffer< + Ptr_Type>::ConfigureStoreChunkFromBuffer; + + auto enqueueLoad() -> auxiliary::DeferredComputation; + + auto load(EnqueuePolicy) -> void; + }; +} // namespace core + +namespace compose +{ + /** Basic configuration for a Load/Store operation. + * + * @tparam ChildClass CRT pattern. + * The purpose is that in child classes `return *this` should return + * an instance of the child class, not of ConfigureLoadStore. + * Instantiate with void when using without subclass. + */ + template + class ConfigureLoadStore + { + ConfigureLoadStore() = default; + + public: + auto offset(Offset) -> ChildClass &; + auto extent(Extent) -> ChildClass &; + friend ChildClass; + }; + + /** Configuration for a Store operation with a buffer type. + * + * This class does intentionally not support Load operations since there are + * pointer types (const pointers, unique pointers) where Load operations + * make no sense. See the \ref ConfigureLoadStoreFromBuffer class template + * for both Load/Store operations. + * + * @tparam Ptr_Type The type of pointer used internally. + * @tparam ChildClass CRT pattern. + * The purpose is that in child classes `return *this` should return + * an instance of the child class, not of + * ConfigureStoreChunkFromBuffer. Instantiate with void when using without + * subclass. + */ + template + class ConfigureStoreChunkFromBuffer + { + public: + auto memorySelection(MemorySelection) -> ChildClass &; + }; +} // namespace compose + +class ConfigureLoadStore + : public core::ConfigureLoadStore + , public compose::ConfigureLoadStore +{ + friend class RecordComponent; + friend class core::ConfigureLoadStore; + + ConfigureLoadStore(RecordComponent &rc); + ConfigureLoadStore(core::ConfigureLoadStore &&); +}; + +template +class ConfigureStoreChunkFromBuffer + : public core::ConfigureStoreChunkFromBuffer + , public compose::ConfigureLoadStore< + ConfigureStoreChunkFromBuffer> + , public compose::ConfigureStoreChunkFromBuffer< + ConfigureStoreChunkFromBuffer> +{ + friend class core::ConfigureLoadStore; + + using core::ConfigureStoreChunkFromBuffer< + Ptr_Type>::ConfigureStoreChunkFromBuffer; +}; + +template +class ConfigureLoadStoreFromBuffer + : public core::ConfigureLoadStoreFromBuffer + , public compose::ConfigureLoadStore> + , public compose::ConfigureStoreChunkFromBuffer< + ConfigureLoadStoreFromBuffer> +{ + friend class ConfigureLoadStoreCore; + + using core::ConfigureLoadStoreFromBuffer< + Ptr_Type>::ConfigureLoadStoreFromBuffer; +}; +} // namespace openPMD + +#include "openPMD/UndefDatatypeMacros.hpp" +// comment to prevent these includes from being moved by clang-format +#include "openPMD/LoadStoreChunk.tpp" diff --git a/include/openPMD/LoadStoreChunk.tpp b/include/openPMD/LoadStoreChunk.tpp new file mode 100644 index 0000000000..c3c104cebd --- /dev/null +++ b/include/openPMD/LoadStoreChunk.tpp @@ -0,0 +1,64 @@ +#pragma once + +#include "openPMD/LoadStoreChunk.hpp" + +namespace openPMD::core +{ +template +auto ConfigureLoadStore::withSharedPtr(std::shared_ptr data) + -> shared_ptr_return_type +{ + if (!data) + { + throw std::runtime_error( + "Unallocated pointer passed during chunk store."); + } + return shared_ptr_return_type( + std::static_pointer_cast>(std::move(data)), + {std::move(*this)}); +} +template +auto ConfigureLoadStore::withUniquePtr(UniquePtrWithLambda data) + -> unique_ptr_return_type + +{ + if (!data) + { + throw std::runtime_error( + "Unallocated pointer passed during chunk store."); + } + return unique_ptr_return_type( + std::move(data).template static_cast_>(), + {std::move(*this)}); +} +template +auto ConfigureLoadStore::withRawPtr(T *data) -> shared_ptr_return_type +{ + if (!data) + { + throw std::runtime_error( + "Unallocated pointer passed during chunk store."); + } + return shared_ptr_return_type( + auxiliary::shareRaw(data), {std::move(*this)}); +} + +template +auto ConfigureLoadStore::withUniquePtr(std::unique_ptr data) + -> unique_ptr_return_type +{ + return withUniquePtr(UniquePtrWithLambda(std::move(data))); +} +template +auto ConfigureLoadStore::withContiguousContainer(T_ContiguousContainer &data) + -> std::enable_if_t< + auxiliary::IsContiguousContainer_v, + shared_ptr_return_type> +{ + if (!m_extent.has_value() && dim() == 1) + { + m_extent = Extent{data.size()}; + } + return withRawPtr(data.data()); +} +} // namespace openPMD::core diff --git a/include/openPMD/Mesh.hpp b/include/openPMD/Mesh.hpp index d0bf81ddef..4ff5d3749d 100644 --- a/include/openPMD/Mesh.hpp +++ b/include/openPMD/Mesh.hpp @@ -23,7 +23,9 @@ #include "openPMD/UnitDimension.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/Container.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/ScientificDefaults.hpp" #include #include @@ -37,10 +39,14 @@ namespace openPMD * @see * https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#mesh-based-records */ -class Mesh : public BaseRecord +class Mesh + : public BaseRecord + , internal::ScientificDefaults { friend class Container; friend class Iteration; + template + friend class internal::ScientificDefaults; public: Mesh(Mesh const &) = default; @@ -328,8 +334,14 @@ class Mesh : public BaseRecord void flush_impl(std::string const &, internal::FlushParams const &) override; void read(); + auto retrieveDimensionality() const -> uint64_t; }; // Mesh +static_assert(internal::IsContainer_v); +static_assert(std::is_same_v< + Container, + internal::AsContainer_t>); + template inline std::vector Mesh::gridSpacing() const { diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index 4f309c0f2a..075d5bb8cd 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -24,19 +24,24 @@ #include "openPMD/Record.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include "openPMD/backend/ScientificDefaults.hpp" #include namespace openPMD { -class ParticleSpecies : public Container +class ParticleSpecies + : public Container + , internal::ScientificDefaults { friend class Container; friend class Container; friend class Iteration; template friend T &internal::makeOwning(T &self, Series); + template + friend class internal::ScientificDefaults; public: ParticlePatches particlePatches; diff --git a/include/openPMD/Record.hpp b/include/openPMD/Record.hpp index 2bb070365f..618a246c83 100644 --- a/include/openPMD/Record.hpp +++ b/include/openPMD/Record.hpp @@ -23,17 +23,22 @@ #include "openPMD/RecordComponent.hpp" #include "openPMD/UnitDimension.hpp" #include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/ScientificDefaults.hpp" #include #include namespace openPMD { -class Record : public BaseRecord +class Record + : public BaseRecord + , internal::ScientificDefaults { friend class Container; friend class Iteration; friend class ParticleSpecies; + template + friend class internal::ScientificDefaults; public: Record(Record const &) = default; @@ -57,6 +62,8 @@ class Record : public BaseRecord [[nodiscard]] internal::HomogenizeExtents read(); }; // Record +static_assert(internal::HasScientificDefaults_v); + template inline T Record::timeOffset() const { diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index d06b4213f4..e8ab57def7 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -22,14 +22,13 @@ #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" +#include "openPMD/LoadStoreChunk.hpp" #include "openPMD/auxiliary/ShareRaw.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/auxiliary/UniquePtr.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" - -// comment to prevent this include from being moved by clang-format -#include "openPMD/DatatypeMacros.hpp" +#include "openPMD/backend/ScientificDefaults.hpp" #include #include @@ -75,7 +74,8 @@ namespace internal */ std::queue m_chunks; - void push_chunk(IOTask &&task); + void push_chunk( + IOTask &&task, std::optional immediate_flush = std::nullopt); /** * Stores the value for constant record components. * Ignored otherwise. @@ -107,10 +107,23 @@ namespace internal class BaseRecordData; } // namespace internal +namespace core +{ + class ConfigureLoadStore; + template + class ConfigureLoadStoreFromBuffer; + template + class ConfigureStoreChunkFromBuffer; + struct VisitorEnqueueLoadVariant; + struct VisitorLoadVariant; +} // namespace core + template class BaseRecord; -class RecordComponent : public BaseRecordComponent +class RecordComponent + : public BaseRecordComponent + , internal::ScientificDefaults { template friend class Container; @@ -128,6 +141,15 @@ class RecordComponent : public BaseRecordComponent friend class MeshRecordComponent; template friend T &internal::makeOwning(T &self, Series); + friend class core::ConfigureLoadStore; + template + friend class core::ConfigureLoadStoreFromBuffer; + template + friend class core::ConfigureStoreChunkFromBuffer; + friend struct core::VisitorEnqueueLoadVariant; + friend struct core::VisitorLoadVariant; + template + friend class internal::ScientificDefaults; public: enum class Allocation @@ -214,6 +236,8 @@ class RecordComponent : public BaseRecordComponent */ bool empty() const; + ConfigureLoadStore prepareLoadStore(); + /** Load and allocate a chunk of data * * Set offset to {0u} and extent to {-1u} for full selection. @@ -224,11 +248,8 @@ class RecordComponent : public BaseRecordComponent template std::shared_ptr loadChunk(Offset = {0u}, Extent = {-1u}); -#define OPENPMD_ENUMERATE_TYPES(type) , std::shared_ptr - using shared_ptr_dataset_types = auxiliary::detail::variant_tail_t< - auxiliary::detail::bottom OPENPMD_FOREACH_DATASET_DATATYPE( - OPENPMD_ENUMERATE_TYPES)>; -#undef OPENPMD_ENUMERATE_TYPES + using shared_ptr_dataset_types = + auxiliary::detail::shared_ptr_dataset_types; /** std::variant-based version of allocating loadChunk(Offset, Extent) * @@ -266,25 +287,6 @@ class RecordComponent : public BaseRecordComponent template void loadChunk(std::shared_ptr data, Offset offset, Extent extent); - /** Load a chunk of data into pre-allocated memory, array version. - * - * @param data Preallocated, contiguous buffer, large enough to load the - * the requested data into it. - * The shared pointer must own and manage the buffer. - * Optimizations might be implemented based on this - * assumption (e.g. skipping the operation if the backend - * is the unique owner). - * The array-based overload helps avoid having to manually - * specify the delete[] destructor (C++17 feature). - * @param offset Offset within the dataset. Set to {0u} for full selection. - * @param extent Extent within the dataset, counted from the offset. - * Set to {-1u} for full selection. - * If offset is non-zero and extent is {-1u} the leftover - * extent in the record component will be selected. - */ - template - void loadChunk(std::shared_ptr data, Offset offset, Extent extent); - /** Load a chunk of data into pre-allocated memory, raw pointer version. * * @param data Preallocated, contiguous buffer, large enough to load the @@ -324,18 +326,6 @@ class RecordComponent : public BaseRecordComponent template void storeChunk(std::shared_ptr data, Offset offset, Extent extent); - /** Store a chunk of data from a chunk of memory, array version. - * - * @param data Preallocated, contiguous buffer, large enough to read the - * the specified data from it. - * The array-based overload helps avoid having to manually - * specify the delete[] destructor (C++17 feature). - * @param offset Offset within the dataset. - * @param extent Extent within the dataset, counted from the offset. - */ - template - void storeChunk(std::shared_ptr data, Offset offset, Extent extent); - /** Store a chunk of data from a chunk of memory, unique pointer version. * * @param data Preallocated, contiguous buffer, large enough to read the @@ -486,7 +476,7 @@ class RecordComponent : public BaseRecordComponent protected: void flush(std::string const &, internal::FlushParams const &); - void read(bool require_unit_si); + void read(); private: /** @@ -497,8 +487,23 @@ class RecordComponent : public BaseRecordComponent */ RecordComponent &makeEmpty(Dataset d); - void storeChunk( - auxiliary::WriteBuffer buffer, Datatype datatype, Offset o, Extent e); + void storeChunk_impl( + auxiliary::WriteBuffer buffer, + Datatype datatype, + internal::LoadStoreConfigWithBuffer, + std::optional flush_immediately = std::nullopt); + + template + DynamicMemoryView storeChunkSpan_impl(internal::LoadStoreConfig); + template + DynamicMemoryView storeChunkSpanCreateBuffer_impl( + internal::LoadStoreConfig, F &&createBuffer); + + template + void + loadChunk_impl(std::shared_ptr, internal::LoadStoreConfigWithBuffer); + template + std::shared_ptr loadChunkAllocate_impl(internal::LoadStoreConfig); // clang-format off OPENPMD_protected @@ -534,7 +539,7 @@ OPENPMD_protected BaseRecordComponent::setData(m_recordComponentData); } - void readBase(bool require_unit_si); + void readBase(); template void verifyChunk(Offset const &, Extent const &) const; @@ -564,6 +569,4 @@ namespace internal } // namespace openPMD -#include "openPMD/UndefDatatypeMacros.hpp" -// comment to prevent these includes from being moved by clang-format #include "RecordComponent.tpp" diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 9d1d8332b4..da64c9e898 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -23,6 +23,7 @@ #include "openPMD/Datatype.hpp" #include "openPMD/Error.hpp" +#include "openPMD/LoadStoreChunk.hpp" #include "openPMD/RecordComponent.hpp" #include "openPMD/Span.hpp" #include "openPMD/auxiliary/Memory.hpp" @@ -32,6 +33,7 @@ #include "openPMD/backend/Attributable.hpp" #include +#include #include namespace openPMD @@ -41,8 +43,11 @@ template inline void RecordComponent::storeChunk(std::unique_ptr data, Offset o, Extent e) { - storeChunk( - UniquePtrWithLambda(std::move(data)), std::move(o), std::move(e)); + prepareLoadStore() + .offset(std::move(o)) + .extent(std::move(e)) + .withUniquePtr(std::move(data)) + .store(EnqueuePolicy::Defer); } template @@ -50,39 +55,73 @@ inline typename std::enable_if_t< auxiliary::IsContiguousContainer_v> RecordComponent::storeChunk(T_ContiguousContainer &data, Offset o, Extent e) { - uint8_t dim = getDimensionality(); + auto storeChunkConfig = prepareLoadStore(); - // default arguments - // offset = {0u}: expand to right dim {0u, 0u, ...} - Offset offset = o; - if (o.size() == 1u && o.at(0) == 0u) + auto joined_dim = joinedDimension(); + if (!joined_dim.has_value() && (o.size() != 1 || o.at(0) != 0u)) { - if (joinedDimension().has_value()) - { - offset.clear(); - } - else if (dim > 1u) - { - offset = Offset(dim, 0u); - } + storeChunkConfig.offset(std::move(o)); + } + if (e.size() != 1 || e.at(0) != -1u) + { + storeChunkConfig.extent(std::move(e)); } - // extent = {-1u}: take full size - Extent extent(dim, 1u); - // avoid outsmarting the user: - // - stdlib data container implement 1D -> 1D chunk to write - if (e.size() == 1u && e.at(0) == -1u && dim == 1u) - extent.at(0) = data.size(); - else - extent = e; - - storeChunk(auxiliary::shareRaw(data.data()), offset, extent); + std::move(storeChunkConfig) + .withContiguousContainer(data) + .store(EnqueuePolicy::Defer); } template inline DynamicMemoryView RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) { + return prepareLoadStore() + .offset(std::move(o)) + .extent(std::move(e)) + .enqueueStore(std::forward(createBuffer)); +} + +namespace detail +{ + template + struct pointer_as_void; + + template + struct pointer_as_void> + { + static auto call(std::shared_ptr const &ptr) + -> std::shared_ptr + { + return std::static_pointer_cast(ptr); + } + }; + + template + struct pointer_as_void> + { + static auto call(std::unique_ptr &&ptr) -> UniquePtrWithLambda + { + return UniquePtrWithLambda(std::move(ptr)); + } + }; + template + struct pointer_as_void> + { + static auto call(std::unique_ptr &&ptr) + -> UniquePtrWithLambda + { + return UniquePtrWithLambda(std::move(ptr)) + .template static_cast_(); + } + }; +} // namespace detail + +template +inline DynamicMemoryView RecordComponent::storeChunkSpanCreateBuffer_impl( + internal::LoadStoreConfig cfg, F &&createBuffer) +{ + auto [o, e] = std::move(cfg); verifyChunk(o, e); /* @@ -112,10 +151,9 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) "using storeChunk() (see RecordComponent::resetDataset())."); } Parameter dCreate(rc.m_dataset.value()); - dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; + dCreate.name = rc->m_writable.ownKeyWithinParent; IOHandler()->enqueue(IOTask(this, dCreate)); } - Parameter getBufferView; getBufferView.offset = o; getBufferView.extent = e; @@ -128,11 +166,16 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) // note that data might have either // type shared_ptr or shared_ptr auto data = std::forward(createBuffer)(size); + using ptr_type = decltype(data); out.ptr = static_cast(data.get()); - if (size > 0) - { - storeChunk(std::move(data), std::move(o), std::move(e)); - } + internal::LoadStoreConfigWithBuffer ls_cfg{ + std::move(o), std::move(e), std::nullopt}; + storeChunk_impl( + auxiliary::WriteBuffer( + detail::pointer_as_void::call(std::move(data))), + getBufferView.dtype, + std::move(ls_cfg), + /*flush_immediately=*/false); } setDirtyRecursive(true); return DynamicMemoryView{std::move(getBufferView), size, *this}; @@ -169,4 +212,13 @@ inline auto RecordComponent::visit(Args &&...args) return switchDatasetType>( getDatatype(), *this, std::forward(args)...); } + +// definitions for LoadStoreChunk.hpp +template +auto core::ConfigureLoadStore::enqueueStore(F &&createBuffer) + -> DynamicMemoryView +{ + return m_rc.storeChunkSpanCreateBuffer_impl( + storeChunkConfig(), std::forward(createBuffer)); +} } // namespace openPMD diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 603e540c2b..c889406dc5 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -789,6 +789,8 @@ class Series : public Attributable "Cannot call this on an instance of Series."); } + [[nodiscard]] bool flushImmediately() const; + // clang-format off OPENPMD_private // clang-format on @@ -835,7 +837,7 @@ OPENPMD_private Attributable::setData(m_series); } - std::unique_ptr parseInput(std::string); + std::unique_ptr parseInput(std::string) const; /** * @brief Parse non-backend-specific configuration in JSON config. * @@ -846,7 +848,8 @@ OPENPMD_private * to include the JSON lib here */ template - void parseJsonOptions(TracingJSON &options, ParsedInput &); + void parseJsonOptions( + TracingJSON &options, ParsedInput &, internal::GlobalParameters &); bool hasExpansionPattern(std::string filenameWithExtension); bool reparseExpansionPattern(std::string filenameWithExtension); template @@ -856,10 +859,11 @@ OPENPMD_private std::string const &options, MPI_Communicator &&...); template - std::tuple, TracingJSON> initIOHandler( + std::tuple, TracingJSON> + prepareIOHandlerArguments( + internal::GlobalParameters &, std::string const &filepath, std::string const &options, - Access at, bool resolve_generic_extension, MPI_Communicator &&...); void initSeries( diff --git a/include/openPMD/auxiliary/Future.hpp b/include/openPMD/auxiliary/Future.hpp new file mode 100644 index 0000000000..8a4b2a516c --- /dev/null +++ b/include/openPMD/auxiliary/Future.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include "openPMD/auxiliary/TypeTraits.hpp" + +#include + +namespace openPMD::auxiliary +{ +template +class DeferredComputation +{ + using task_type = std::function; + task_type m_task; + bool m_valid = false; + +public: + DeferredComputation(task_type); + ~DeferredComputation(); + + auto get() -> T; + + [[nodiscard]] auto valid() const noexcept -> bool; +}; +} // namespace openPMD::auxiliary diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index faf2b29472..e373873a4f 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -124,6 +124,22 @@ namespace detail { constexpr static bool value = true; }; + + template + struct ScalarType + { + using type = T; + }; + template + struct ScalarType> + { + using type = T; + }; + template + struct ScalarType> + { + using type = T; + }; } // namespace detail template @@ -141,6 +157,12 @@ using IsPointer_t = typename detail::IsPointer::type; template inline constexpr bool IsChar_v = detail::IsChar::value; +template +using ScalarType_t = typename detail::ScalarType::type; + +template +using VectorType_t = std::vector>; + /** Emulate in the C++ concept ContiguousContainer * * Users can implement this trait for a type to signal it can be used as @@ -203,6 +225,34 @@ namespace detail // little trick to avoid trailing commas in the macro expansions below template using variant_tail_t = std::variant; + + template