Port the Metapack module into this project

CMakeLists.txt

@@ -51,6 +51,7 @@ option(SOURCEMETA_CORE_CSS "Build the Sourcemeta Core CSS library" ON)
 option(SOURCEMETA_CORE_MARKDOWN "Build the Sourcemeta Core Markdown library" ON)
 option(SOURCEMETA_CORE_DIFF "Build the Sourcemeta Core Diff library" ON)
 option(SOURCEMETA_CORE_TERMINAL "Build the Sourcemeta Core Terminal library" ON)
+option(SOURCEMETA_CORE_EXTRA_METAPACK "Build the Sourcemeta Core Metapack library" ON)
 option(SOURCEMETA_CORE_TESTS "Build the Sourcemeta Core tests" OFF)
 option(SOURCEMETA_CORE_TESTS_CI "Build the Sourcemeta Core CI tests" OFF)
 option(SOURCEMETA_CORE_BENCHMARK "Build the Sourcemeta Core benchmarks" OFF)
@@ -319,6 +320,10 @@ if(SOURCEMETA_CORE_TERMINAL)
   add_subdirectory(src/core/terminal)
 endif()
 
+if(SOURCEMETA_CORE_EXTRA_METAPACK)
+  add_subdirectory(src/extra/metapack)
+endif()
+
 if(SOURCEMETA_CORE_DOCS)
   sourcemeta_target_doxygen(CONFIG "${PROJECT_SOURCE_DIR}/doxygen/Doxyfile.in"
     OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/website")
@@ -518,6 +523,10 @@ if(SOURCEMETA_CORE_TESTS)
     add_subdirectory(test/terminal)
   endif()
 
+  if(SOURCEMETA_CORE_EXTRA_METAPACK)
+    add_subdirectory(test/metapack)
+  endif()
+
   if(PROJECT_IS_TOP_LEVEL)
     # Otherwise we need the child project to link
     # against the sanitizers too.

benchmark/CMakeLists.txt

@@ -56,6 +56,10 @@ if(SOURCEMETA_CORE_CRYPTO)
   list(APPEND BENCHMARK_SOURCES crypto.cc)
 endif()
 
+if(SOURCEMETA_CORE_EXTRA_METAPACK)
+  list(APPEND BENCHMARK_SOURCES metapack.cc)
+endif()
+
 if(BENCHMARK_SOURCES)
   sourcemeta_benchmark(NAMESPACE sourcemeta PROJECT core
     SOURCES ${BENCHMARK_SOURCES})
@@ -137,6 +141,12 @@ if(BENCHMARK_SOURCES)
       PRIVATE sourcemeta::core::crypto sourcemeta::core::io)
   endif()
 
+  if(SOURCEMETA_CORE_EXTRA_METAPACK)
+    target_link_libraries(sourcemeta_core_benchmarks
+      PRIVATE sourcemeta::core::metapack sourcemeta::core::json
+              sourcemeta::core::io)
+  endif()
+
   # A single run prints the human readable report and writes the JSON file, so
   # that recording results for later comparison costs nothing extra
   add_custom_target(benchmark

benchmark/metapack.cc

@@ -0,0 +1,56 @@
+#include <sourcemeta/core/benchmark.h>
+
+#include <sourcemeta/core/io.h>
+#include <sourcemeta/core/json.h>
+#include <sourcemeta/core/metapack.h>
+
+#include <filesystem> // std::filesystem
+
+static auto benchmark_document() -> sourcemeta::core::JSON {
+  return sourcemeta::core::read_json(std::filesystem::path{CURRENT_DIRECTORY} /
+                                     "files" / "2019_09_omc_json_v2.json");
+}
+
+BENCHMARK(Metapack_Parse_OMC_JSON_V2) {
+  const sourcemeta::core::TemporaryDirectory directory{
+      std::filesystem::temp_directory_path(), ".metapack-"};
+  const auto path{directory.path() / "benchmark.metapack"};
+  sourcemeta::core::metapack_write_json(
+      path, benchmark_document(), "application/schema+json",
+      sourcemeta::core::MetapackEncoding::GZIP);
+  const sourcemeta::core::FileView view{path};
+
+  for (auto iteration : state) {
+    auto result{sourcemeta::core::metapack_parse(view)};
+    sourcemeta::core::benchmark_do_not_optimize(result);
+  }
+}
+
+BENCHMARK(Metapack_Read_OMC_JSON_V2) {
+  const sourcemeta::core::TemporaryDirectory directory{
+      std::filesystem::temp_directory_path(), ".metapack-"};
+  const auto path{directory.path() / "benchmark.metapack"};
+  sourcemeta::core::metapack_write_json(
+      path, benchmark_document(), "application/schema+json",
+      sourcemeta::core::MetapackEncoding::GZIP);
+  const sourcemeta::core::FileView view{path};
+
+  for (auto iteration : state) {
+    auto result{sourcemeta::core::metapack_read_json(view)};
+    sourcemeta::core::benchmark_do_not_optimize(result);
+  }
+}
+
+BENCHMARK(Metapack_Write_OMC_JSON_V2) {
+  const sourcemeta::core::TemporaryDirectory directory{
+      std::filesystem::temp_directory_path(), ".metapack-"};
+  const auto path{directory.path() / "benchmark.metapack"};
+  const auto document{benchmark_document()};
+
+  for (auto iteration : state) {
+    sourcemeta::core::metapack_write_json(
+        path, document, "application/schema+json",
+        sourcemeta::core::MetapackEncoding::GZIP);
+    sourcemeta::core::benchmark_do_not_optimize(path);
+  }
+}

config.cmake.in

@@ -45,6 +45,7 @@ if(NOT SOURCEMETA_CORE_COMPONENTS)
   list(APPEND SOURCEMETA_CORE_COMPONENTS markdown)
   list(APPEND SOURCEMETA_CORE_COMPONENTS diff)
   list(APPEND SOURCEMETA_CORE_COMPONENTS terminal)
+  list(APPEND SOURCEMETA_CORE_COMPONENTS metapack)
   list(APPEND SOURCEMETA_CORE_COMPONENTS error)
   list(APPEND SOURCEMETA_CORE_COMPONENTS options)
   list(APPEND SOURCEMETA_CORE_COMPONENTS text)
@@ -384,6 +385,19 @@ foreach(component ${SOURCEMETA_CORE_COMPONENTS})
     include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_diff.cmake")
   elseif(component STREQUAL "terminal")
     include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_terminal.cmake")
+  elseif(component STREQUAL "metapack")
+    if("@SOURCEMETA_CORE_CRYPTO_BACKEND@" STREQUAL "openssl")
+      find_dependency(OpenSSL 3.0)
+    endif()
+    include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_preprocessor.cmake")
+    include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_numeric.cmake")
+    include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake")
+    include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_crypto.cmake")
+    include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_gzip.cmake")
+    include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_io.cmake")
+    include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_unicode.cmake")
+    include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_json.cmake")
+    include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_metapack.cmake")
   elseif(component STREQUAL "error")
     include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_error.cmake")
   elseif(component STREQUAL "options")

src/extra/metapack/CMakeLists.txt

@@ -0,0 +1,13 @@
+sourcemeta_library(NAMESPACE sourcemeta PROJECT core NAME metapack
+  PRIVATE_HEADERS error.h format.h read.h write.h
+  SOURCES metapack_read.cc metapack_write.cc)
+
+target_link_libraries(sourcemeta_core_metapack PUBLIC sourcemeta::core::io)
+target_link_libraries(sourcemeta_core_metapack PUBLIC sourcemeta::core::json)
+target_link_libraries(sourcemeta_core_metapack PRIVATE sourcemeta::core::crypto)
+target_link_libraries(sourcemeta_core_metapack PRIVATE sourcemeta::core::gzip)
+target_link_libraries(sourcemeta_core_metapack PRIVATE sourcemeta::core::text)
+
+if(SOURCEMETA_CORE_INSTALL)
+  sourcemeta_library_install(NAMESPACE sourcemeta PROJECT core NAME metapack)
+endif()

src/extra/metapack/include/sourcemeta/core/metapack.h

@@ -0,0 +1,49 @@
+#ifndef SOURCEMETA_CORE_METAPACK_H_
+#define SOURCEMETA_CORE_METAPACK_H_
+
+#ifndef SOURCEMETA_CORE_METAPACK_EXPORT
+#include <sourcemeta/core/metapack_export.h>
+#endif
+
+// NOLINTBEGIN(misc-include-cleaner)
+#include <sourcemeta/core/metapack_error.h>
+#include <sourcemeta/core/metapack_format.h>
+#include <sourcemeta/core/metapack_read.h>
+#include <sourcemeta/core/metapack_write.h>
+// NOLINTEND(misc-include-cleaner)
+
+/// @defgroup metapack Metapack
+/// @brief A file format that puts a memory-mappable metadata header in front
+/// of arbitrary contents
+///
+/// A filesystem answers a few questions about a file without anyone opening
+/// it, which is what a stat call is for: how large it is, when it last
+/// changed, what kind of file it is. Everything else a file has to say about
+/// itself has nowhere to live. Keeping it beside the file, in a sidecar, a
+/// database, or a naming convention, costs a second lookup and lets the two
+/// drift apart. Keeping it inside the contents means decoding the contents to
+/// reach it.
+///
+/// A Metapack file carries it in front of the contents instead, as a packed
+/// little-endian structure whose fields sit at offsets fixed at compile time.
+/// A reader memory maps the file, points the header structure at the start of
+/// the mapping, and has every field without a read, a copy, or an allocation.
+/// What follows is handed back as a view into that same mapping, so the
+/// contents cost nothing until something reaches for them.
+///
+/// What the header holds, field by field, is the
+/// @ref sourcemeta::core::MetapackHeader
+/// structure itself. After it come the media type, then an extension area,
+/// then the contents, each located from the length recorded before it.
+///
+/// The extension area carries whatever else a given kind of file needs to say
+/// about itself, as a packed structure that the writer defines and that a
+/// reader interprets in place, exactly like the header itself.
+///
+/// This functionality is included as follows:
+///
+/// ```cpp
+/// #include <sourcemeta/core/metapack.h>
+/// ```
+
+#endif

src/extra/metapack/include/sourcemeta/core/metapack_error.h

@@ -0,0 +1,47 @@
+#ifndef SOURCEMETA_CORE_METAPACK_ERROR_H_
+#define SOURCEMETA_CORE_METAPACK_ERROR_H_
+
+#ifndef SOURCEMETA_CORE_METAPACK_EXPORT
+#include <sourcemeta/core/metapack_export.h>
+#endif
+
+#include <exception>   // std::exception
+#include <string>      // std::string
+#include <string_view> // std::string_view
+
+namespace sourcemeta::core {
+
+// Exporting symbols that depends on the standard C++ library is considered
+// safe
+// https://learn.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-2-c4275?view=msvc-170&redirectedfrom=MSDN
+#if defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable : 4251 4275)
+#endif
+
+/// @ingroup metapack
+/// An error that represents contents that cannot be written as a file of this
+/// format
+class SOURCEMETA_CORE_METAPACK_EXPORT MetapackError : public std::exception {
+public:
+  /// Construct an error with the given message
+  MetapackError(const char *message) : message_{message} {}
+  MetapackError(std::string message) = delete;
+  MetapackError(std::string &&message) = delete;
+  MetapackError(std::string_view message) = delete;
+
+  [[nodiscard]] auto what() const noexcept -> const char * override {
+    return this->message_;
+  }
+
+private:
+  const char *message_;
+};
+
+#if defined(_MSC_VER)
+#pragma warning(pop)
+#endif
+
+} // namespace sourcemeta::core
+
+#endif

src/extra/metapack/include/sourcemeta/core/metapack_format.h

@@ -0,0 +1,80 @@
+#ifndef SOURCEMETA_CORE_METAPACK_FORMAT_H_
+#define SOURCEMETA_CORE_METAPACK_FORMAT_H_
+
+#ifndef SOURCEMETA_CORE_METAPACK_EXPORT
+#include <sourcemeta/core/metapack_export.h>
+#endif
+
+#include <array>   // std::array
+#include <bit>     // std::endian
+#include <cstddef> // std::size_t
+#include <cstdint> // std::int64_t, std::uint8_t, std::uint16_t, std::uint32_t, std::uint64_t
+#include <limits> // std::numeric_limits
+
+namespace sourcemeta::core {
+
+// Every multi-byte field is read where it lies rather than byte-swapped, so
+// the byte order the format is written in is the only one it can be read in
+static_assert(std::endian::native == std::endian::little,
+              "The metapack format is little-endian and this host is not");
+
+/// @ingroup metapack
+/// How the contents of a file are stored
+enum class MetapackEncoding : std::uint8_t {
+  /// The contents are stored as they are
+  Identity = 0,
+  /// The contents are stored compressed as per RFC 1952
+  GZIP = 1
+};
+
+#pragma pack(push, 1)
+/// @ingroup metapack
+/// The metadata that a file of this format begins with
+struct MetapackHeader {
+  /// The magic number that identifies the format
+  std::uint32_t magic{0};
+  /// The revision of the format the file was written to
+  std::uint16_t format_version{0};
+  /// The encoding the contents are stored under
+  MetapackEncoding encoding{MetapackEncoding::Identity};
+  /// Reserved for a future revision of the format
+  std::uint8_t reserved{0};
+  /// Nanoseconds since the Unix epoch at which the file was written
+  std::int64_t last_modified{0};
+  /// How many bytes the contents take once decoded
+  std::uint64_t content_bytes{0};
+  /// The SHA-256 digest of the decoded contents, which names this revision of
+  /// them for a cache to compare against rather than being checked on read
+  std::array<std::uint8_t, 32> checksum{};
+  /// How many bytes the media type that follows the header takes
+  std::uint16_t media_type_length{0};
+};
+#pragma pack(pop)
+
+static_assert(alignof(MetapackHeader) == 1,
+              "The header is read where it lies, so its layout must be packed");
+static_assert(sizeof(MetapackHeader) == 58,
+              "The header layout is part of the format, so moving, resizing or "
+              "adding a field means bumping the format revision");
+
+/// @ingroup metapack
+/// How many characters it takes to render a checksum
+inline constexpr std::size_t METAPACK_CHECKSUM_HEX_LENGTH{64};
+
+static_assert(METAPACK_CHECKSUM_HEX_LENGTH ==
+                  2 * sizeof(MetapackHeader::checksum),
+              "Rendering a checksum takes two characters per byte");
+
+/// @ingroup metapack
+/// The longest media type a file of this format can declare
+inline constexpr std::uint16_t METAPACK_MAXIMUM_MEDIA_TYPE_BYTES{
+    std::numeric_limits<std::uint16_t>::max()};
+
+/// @ingroup metapack
+/// The largest extension area a file of this format can carry
+inline constexpr std::uint32_t METAPACK_MAXIMUM_EXTENSION_BYTES{
+    std::numeric_limits<std::uint32_t>::max()};
+
+} // namespace sourcemeta::core
+
+#endif