diff --git a/CMakeLists.txt b/CMakeLists.txt index bad2eecb5..751bd498e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,7 @@ option(SOURCEMETA_CORE_CSS "Build the Sourcemeta Core CSS library" ON) option(SOURCEMETA_CORE_MARKDOWN "Build the Sourcemeta Core Markdown library" ON) option(SOURCEMETA_CORE_DIFF "Build the Sourcemeta Core Diff library" ON) option(SOURCEMETA_CORE_TERMINAL "Build the Sourcemeta Core Terminal library" ON) +option(SOURCEMETA_CORE_EXTRA_METAPACK "Build the Sourcemeta Core Metapack library" ON) option(SOURCEMETA_CORE_TESTS "Build the Sourcemeta Core tests" OFF) option(SOURCEMETA_CORE_TESTS_CI "Build the Sourcemeta Core CI tests" OFF) option(SOURCEMETA_CORE_BENCHMARK "Build the Sourcemeta Core benchmarks" OFF) @@ -319,6 +320,10 @@ if(SOURCEMETA_CORE_TERMINAL) add_subdirectory(src/core/terminal) endif() +if(SOURCEMETA_CORE_EXTRA_METAPACK) + add_subdirectory(src/extra/metapack) +endif() + if(SOURCEMETA_CORE_DOCS) sourcemeta_target_doxygen(CONFIG "${PROJECT_SOURCE_DIR}/doxygen/Doxyfile.in" OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/website") @@ -518,6 +523,10 @@ if(SOURCEMETA_CORE_TESTS) add_subdirectory(test/terminal) endif() + if(SOURCEMETA_CORE_EXTRA_METAPACK) + add_subdirectory(test/metapack) + endif() + if(PROJECT_IS_TOP_LEVEL) # Otherwise we need the child project to link # against the sanitizers too. diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index b03ca00be..b4aedefec 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -56,6 +56,10 @@ if(SOURCEMETA_CORE_CRYPTO) list(APPEND BENCHMARK_SOURCES crypto.cc) endif() +if(SOURCEMETA_CORE_EXTRA_METAPACK) + list(APPEND BENCHMARK_SOURCES metapack.cc) +endif() + if(BENCHMARK_SOURCES) sourcemeta_benchmark(NAMESPACE sourcemeta PROJECT core SOURCES ${BENCHMARK_SOURCES}) @@ -137,6 +141,12 @@ if(BENCHMARK_SOURCES) PRIVATE sourcemeta::core::crypto sourcemeta::core::io) endif() + if(SOURCEMETA_CORE_EXTRA_METAPACK) + target_link_libraries(sourcemeta_core_benchmarks + PRIVATE sourcemeta::core::metapack sourcemeta::core::json + sourcemeta::core::io) + endif() + # A single run prints the human readable report and writes the JSON file, so # that recording results for later comparison costs nothing extra add_custom_target(benchmark diff --git a/benchmark/metapack.cc b/benchmark/metapack.cc new file mode 100644 index 000000000..fc2dafcd7 --- /dev/null +++ b/benchmark/metapack.cc @@ -0,0 +1,56 @@ +#include + +#include +#include +#include + +#include // std::filesystem + +static auto benchmark_document() -> sourcemeta::core::JSON { + return sourcemeta::core::read_json(std::filesystem::path{CURRENT_DIRECTORY} / + "files" / "2019_09_omc_json_v2.json"); +} + +BENCHMARK(Metapack_Parse_OMC_JSON_V2) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "benchmark.metapack"}; + sourcemeta::core::metapack_write_json( + path, benchmark_document(), "application/schema+json", + sourcemeta::core::MetapackEncoding::GZIP); + const sourcemeta::core::FileView view{path}; + + for (auto iteration : state) { + auto result{sourcemeta::core::metapack_parse(view)}; + sourcemeta::core::benchmark_do_not_optimize(result); + } +} + +BENCHMARK(Metapack_Read_OMC_JSON_V2) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "benchmark.metapack"}; + sourcemeta::core::metapack_write_json( + path, benchmark_document(), "application/schema+json", + sourcemeta::core::MetapackEncoding::GZIP); + const sourcemeta::core::FileView view{path}; + + for (auto iteration : state) { + auto result{sourcemeta::core::metapack_read_json(view)}; + sourcemeta::core::benchmark_do_not_optimize(result); + } +} + +BENCHMARK(Metapack_Write_OMC_JSON_V2) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "benchmark.metapack"}; + const auto document{benchmark_document()}; + + for (auto iteration : state) { + sourcemeta::core::metapack_write_json( + path, document, "application/schema+json", + sourcemeta::core::MetapackEncoding::GZIP); + sourcemeta::core::benchmark_do_not_optimize(path); + } +} diff --git a/config.cmake.in b/config.cmake.in index 7b62dd62a..2fef0eb6b 100644 --- a/config.cmake.in +++ b/config.cmake.in @@ -45,6 +45,7 @@ if(NOT SOURCEMETA_CORE_COMPONENTS) list(APPEND SOURCEMETA_CORE_COMPONENTS markdown) list(APPEND SOURCEMETA_CORE_COMPONENTS diff) list(APPEND SOURCEMETA_CORE_COMPONENTS terminal) + list(APPEND SOURCEMETA_CORE_COMPONENTS metapack) list(APPEND SOURCEMETA_CORE_COMPONENTS error) list(APPEND SOURCEMETA_CORE_COMPONENTS options) list(APPEND SOURCEMETA_CORE_COMPONENTS text) @@ -384,6 +385,19 @@ foreach(component ${SOURCEMETA_CORE_COMPONENTS}) include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_diff.cmake") elseif(component STREQUAL "terminal") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_terminal.cmake") + elseif(component STREQUAL "metapack") + if("@SOURCEMETA_CORE_CRYPTO_BACKEND@" STREQUAL "openssl") + find_dependency(OpenSSL 3.0) + endif() + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_preprocessor.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_numeric.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_crypto.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_gzip.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_io.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_unicode.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_json.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_metapack.cmake") elseif(component STREQUAL "error") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_error.cmake") elseif(component STREQUAL "options") diff --git a/src/extra/metapack/CMakeLists.txt b/src/extra/metapack/CMakeLists.txt new file mode 100644 index 000000000..5e6cfe580 --- /dev/null +++ b/src/extra/metapack/CMakeLists.txt @@ -0,0 +1,13 @@ +sourcemeta_library(NAMESPACE sourcemeta PROJECT core NAME metapack + PRIVATE_HEADERS error.h format.h read.h write.h + SOURCES metapack_read.cc metapack_write.cc) + +target_link_libraries(sourcemeta_core_metapack PUBLIC sourcemeta::core::io) +target_link_libraries(sourcemeta_core_metapack PUBLIC sourcemeta::core::json) +target_link_libraries(sourcemeta_core_metapack PRIVATE sourcemeta::core::crypto) +target_link_libraries(sourcemeta_core_metapack PRIVATE sourcemeta::core::gzip) +target_link_libraries(sourcemeta_core_metapack PRIVATE sourcemeta::core::text) + +if(SOURCEMETA_CORE_INSTALL) + sourcemeta_library_install(NAMESPACE sourcemeta PROJECT core NAME metapack) +endif() diff --git a/src/extra/metapack/include/sourcemeta/core/metapack.h b/src/extra/metapack/include/sourcemeta/core/metapack.h new file mode 100644 index 000000000..aee9b1760 --- /dev/null +++ b/src/extra/metapack/include/sourcemeta/core/metapack.h @@ -0,0 +1,49 @@ +#ifndef SOURCEMETA_CORE_METAPACK_H_ +#define SOURCEMETA_CORE_METAPACK_H_ + +#ifndef SOURCEMETA_CORE_METAPACK_EXPORT +#include +#endif + +// NOLINTBEGIN(misc-include-cleaner) +#include +#include +#include +#include +// NOLINTEND(misc-include-cleaner) + +/// @defgroup metapack Metapack +/// @brief A file format that puts a memory-mappable metadata header in front +/// of arbitrary contents +/// +/// A filesystem answers a few questions about a file without anyone opening +/// it, which is what a stat call is for: how large it is, when it last +/// changed, what kind of file it is. Everything else a file has to say about +/// itself has nowhere to live. Keeping it beside the file, in a sidecar, a +/// database, or a naming convention, costs a second lookup and lets the two +/// drift apart. Keeping it inside the contents means decoding the contents to +/// reach it. +/// +/// A Metapack file carries it in front of the contents instead, as a packed +/// little-endian structure whose fields sit at offsets fixed at compile time. +/// A reader memory maps the file, points the header structure at the start of +/// the mapping, and has every field without a read, a copy, or an allocation. +/// What follows is handed back as a view into that same mapping, so the +/// contents cost nothing until something reaches for them. +/// +/// What the header holds, field by field, is the +/// @ref sourcemeta::core::MetapackHeader +/// structure itself. After it come the media type, then an extension area, +/// then the contents, each located from the length recorded before it. +/// +/// The extension area carries whatever else a given kind of file needs to say +/// about itself, as a packed structure that the writer defines and that a +/// reader interprets in place, exactly like the header itself. +/// +/// This functionality is included as follows: +/// +/// ```cpp +/// #include +/// ``` + +#endif diff --git a/src/extra/metapack/include/sourcemeta/core/metapack_error.h b/src/extra/metapack/include/sourcemeta/core/metapack_error.h new file mode 100644 index 000000000..55c947db4 --- /dev/null +++ b/src/extra/metapack/include/sourcemeta/core/metapack_error.h @@ -0,0 +1,47 @@ +#ifndef SOURCEMETA_CORE_METAPACK_ERROR_H_ +#define SOURCEMETA_CORE_METAPACK_ERROR_H_ + +#ifndef SOURCEMETA_CORE_METAPACK_EXPORT +#include +#endif + +#include // std::exception +#include // std::string +#include // std::string_view + +namespace sourcemeta::core { + +// Exporting symbols that depends on the standard C++ library is considered +// safe +// https://learn.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-2-c4275?view=msvc-170&redirectedfrom=MSDN +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4251 4275) +#endif + +/// @ingroup metapack +/// An error that represents contents that cannot be written as a file of this +/// format +class SOURCEMETA_CORE_METAPACK_EXPORT MetapackError : public std::exception { +public: + /// Construct an error with the given message + MetapackError(const char *message) : message_{message} {} + MetapackError(std::string message) = delete; + MetapackError(std::string &&message) = delete; + MetapackError(std::string_view message) = delete; + + [[nodiscard]] auto what() const noexcept -> const char * override { + return this->message_; + } + +private: + const char *message_; +}; + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +} // namespace sourcemeta::core + +#endif diff --git a/src/extra/metapack/include/sourcemeta/core/metapack_format.h b/src/extra/metapack/include/sourcemeta/core/metapack_format.h new file mode 100644 index 000000000..595b1a59c --- /dev/null +++ b/src/extra/metapack/include/sourcemeta/core/metapack_format.h @@ -0,0 +1,80 @@ +#ifndef SOURCEMETA_CORE_METAPACK_FORMAT_H_ +#define SOURCEMETA_CORE_METAPACK_FORMAT_H_ + +#ifndef SOURCEMETA_CORE_METAPACK_EXPORT +#include +#endif + +#include // std::array +#include // std::endian +#include // std::size_t +#include // std::int64_t, std::uint8_t, std::uint16_t, std::uint32_t, std::uint64_t +#include // std::numeric_limits + +namespace sourcemeta::core { + +// Every multi-byte field is read where it lies rather than byte-swapped, so +// the byte order the format is written in is the only one it can be read in +static_assert(std::endian::native == std::endian::little, + "The metapack format is little-endian and this host is not"); + +/// @ingroup metapack +/// How the contents of a file are stored +enum class MetapackEncoding : std::uint8_t { + /// The contents are stored as they are + Identity = 0, + /// The contents are stored compressed as per RFC 1952 + GZIP = 1 +}; + +#pragma pack(push, 1) +/// @ingroup metapack +/// The metadata that a file of this format begins with +struct MetapackHeader { + /// The magic number that identifies the format + std::uint32_t magic{0}; + /// The revision of the format the file was written to + std::uint16_t format_version{0}; + /// The encoding the contents are stored under + MetapackEncoding encoding{MetapackEncoding::Identity}; + /// Reserved for a future revision of the format + std::uint8_t reserved{0}; + /// Nanoseconds since the Unix epoch at which the file was written + std::int64_t last_modified{0}; + /// How many bytes the contents take once decoded + std::uint64_t content_bytes{0}; + /// The SHA-256 digest of the decoded contents, which names this revision of + /// them for a cache to compare against rather than being checked on read + std::array checksum{}; + /// How many bytes the media type that follows the header takes + std::uint16_t media_type_length{0}; +}; +#pragma pack(pop) + +static_assert(alignof(MetapackHeader) == 1, + "The header is read where it lies, so its layout must be packed"); +static_assert(sizeof(MetapackHeader) == 58, + "The header layout is part of the format, so moving, resizing or " + "adding a field means bumping the format revision"); + +/// @ingroup metapack +/// How many characters it takes to render a checksum +inline constexpr std::size_t METAPACK_CHECKSUM_HEX_LENGTH{64}; + +static_assert(METAPACK_CHECKSUM_HEX_LENGTH == + 2 * sizeof(MetapackHeader::checksum), + "Rendering a checksum takes two characters per byte"); + +/// @ingroup metapack +/// The longest media type a file of this format can declare +inline constexpr std::uint16_t METAPACK_MAXIMUM_MEDIA_TYPE_BYTES{ + std::numeric_limits::max()}; + +/// @ingroup metapack +/// The largest extension area a file of this format can carry +inline constexpr std::uint32_t METAPACK_MAXIMUM_EXTENSION_BYTES{ + std::numeric_limits::max()}; + +} // namespace sourcemeta::core + +#endif diff --git a/src/extra/metapack/include/sourcemeta/core/metapack_read.h b/src/extra/metapack/include/sourcemeta/core/metapack_read.h new file mode 100644 index 000000000..16da26fb3 --- /dev/null +++ b/src/extra/metapack/include/sourcemeta/core/metapack_read.h @@ -0,0 +1,194 @@ +#ifndef SOURCEMETA_CORE_METAPACK_READ_H_ +#define SOURCEMETA_CORE_METAPACK_READ_H_ + +#ifndef SOURCEMETA_CORE_METAPACK_EXPORT +#include +#endif + +// NOLINTBEGIN(misc-include-cleaner) +#include +// NOLINTEND(misc-include-cleaner) + +#include +#include + +#include // std::array +#include // std::chrono::system_clock +#include // std::uint8_t, std::uint64_t +#include // std::filesystem::path +#include // std::optional +#include // std::span +#include // std::string_view +#include // std::is_trivially_copyable_v + +namespace sourcemeta::core { + +/// @ingroup metapack +/// How far stored contents may inflate before a reader refuses them, so that a +/// small file cannot ask for an unbounded amount of memory +inline constexpr std::uint64_t METAPACK_MAXIMUM_DECOMPRESSION_RATIO{1024}; + +/// @ingroup metapack +/// Every part of a file, located in a single pass. The views point into the +/// memory-mapped file, so they last exactly as long as it stays mapped +struct MetapackView { + /// The metadata the file begins with + const MetapackHeader *header{nullptr}; + /// The media type to serve the contents as + std::string_view media_type; + /// The extension area, as the bytes the writer put there + std::span extension; + /// The contents, as stored + std::span payload; +}; + +/// @ingroup metapack +/// Locate every part of a memory-mapped file. For example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// +/// const sourcemeta::core::FileView view{"/tmp/document.metapack"}; +/// const auto parsed{sourcemeta::core::metapack_parse(view)}; +/// assert(parsed.has_value()); +/// assert(parsed.value().media_type == "application/json"); +/// ``` +SOURCEMETA_CORE_METAPACK_EXPORT +auto metapack_parse(const FileView &view) -> std::optional; + +/// @ingroup metapack +/// Read the contents of a file at the given path as a JSON document. For +/// example: +/// +/// ```cpp +/// #include +/// #include +/// +/// const auto document{ +/// sourcemeta::core::metapack_read_json("/tmp/document.metapack")}; +/// assert(document.has_value()); +/// assert(document.value().is_object()); +/// ``` +SOURCEMETA_CORE_METAPACK_EXPORT +auto metapack_read_json(const std::filesystem::path &path, + const std::uint64_t maximum_decompression_ratio = + METAPACK_MAXIMUM_DECOMPRESSION_RATIO) + -> std::optional; + +/// @ingroup metapack +/// Read the contents of a memory-mapped file as a JSON document. For example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// +/// const sourcemeta::core::FileView view{"/tmp/document.metapack"}; +/// const auto document{sourcemeta::core::metapack_read_json(view)}; +/// assert(document.has_value()); +/// ``` +SOURCEMETA_CORE_METAPACK_EXPORT +auto metapack_read_json(const FileView &view, + const std::uint64_t maximum_decompression_ratio = + METAPACK_MAXIMUM_DECOMPRESSION_RATIO) + -> std::optional; + +/// @ingroup metapack +/// Read the contents of an already located file as a JSON document. For +/// example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// +/// const sourcemeta::core::FileView view{"/tmp/document.metapack"}; +/// const auto parsed{sourcemeta::core::metapack_parse(view).value()}; +/// const auto document{sourcemeta::core::metapack_read_json(parsed)}; +/// assert(document.has_value()); +/// ``` +SOURCEMETA_CORE_METAPACK_EXPORT +auto metapack_read_json(const MetapackView &parsed, + const std::uint64_t maximum_decompression_ratio = + METAPACK_MAXIMUM_DECOMPRESSION_RATIO) + -> std::optional; + +/// @ingroup metapack +/// Render the checksum a header records as lowercase hexadecimal. For example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// +/// const sourcemeta::core::FileView view{"/tmp/document.metapack"}; +/// const auto parsed{sourcemeta::core::metapack_parse(view).value()}; +/// const auto checksum{sourcemeta::core::metapack_checksum_hex( +/// *parsed.header)}; +/// std::cout.write(checksum.data(), checksum.size()); +/// ``` +SOURCEMETA_CORE_METAPACK_EXPORT +auto metapack_checksum_hex(const MetapackHeader &header) + -> std::array; + +/// @ingroup metapack +/// The point in time a header records. For example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// #include +/// +/// const sourcemeta::core::FileView view{"/tmp/document.metapack"}; +/// const auto parsed{sourcemeta::core::metapack_parse(view).value()}; +/// assert(sourcemeta::core::metapack_last_modified(*parsed.header) <= +/// std::chrono::system_clock::now()); +/// ``` +SOURCEMETA_CORE_METAPACK_EXPORT +auto metapack_last_modified(const MetapackHeader &header) + -> std::chrono::system_clock::time_point; + +/// @ingroup metapack +/// What an extension area holds, as the layout the writer of the file put +/// there, or nothing when it is too small to hold one. For example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// +/// #pragma pack(push, 1) +/// struct Revision { +/// std::uint32_t number; +/// }; +/// #pragma pack(pop) +/// +/// const sourcemeta::core::FileView view{"/tmp/document.metapack"}; +/// const auto parsed{sourcemeta::core::metapack_parse(view).value()}; +/// const auto *revision{ +/// sourcemeta::core::metapack_extension_as(parsed.extension)}; +/// assert(revision != nullptr); +/// ``` +template +auto metapack_extension_as(const std::span extension) + -> const T * { + static_assert(std::is_trivially_copyable_v, + "An extension layout is read where it lies, so it has to be " + "trivially copyable"); + static_assert(alignof(T) == 1, + "An extension begins at an offset the media type decides, so " + "its layout must be packed to be read where it lies"); + if (extension.size() < sizeof(T)) { + return nullptr; + } + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return reinterpret_cast(extension.data()); +} + +} // namespace sourcemeta::core + +#endif diff --git a/src/extra/metapack/include/sourcemeta/core/metapack_write.h b/src/extra/metapack/include/sourcemeta/core/metapack_write.h new file mode 100644 index 000000000..1b4f857fe --- /dev/null +++ b/src/extra/metapack/include/sourcemeta/core/metapack_write.h @@ -0,0 +1,125 @@ +#ifndef SOURCEMETA_CORE_METAPACK_WRITE_H_ +#define SOURCEMETA_CORE_METAPACK_WRITE_H_ + +#ifndef SOURCEMETA_CORE_METAPACK_EXPORT +#include +#endif + +// NOLINTBEGIN(misc-include-cleaner) +#include +#include +// NOLINTEND(misc-include-cleaner) + +#include + +#include // std::uint8_t +#include // std::filesystem::path +#include // std::span +#include // std::string_view + +namespace sourcemeta::core { + +/// @ingroup metapack +/// Write a JSON document as the contents of a file. A media type or +/// extension too large for the format to record throws. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// const auto document{sourcemeta::core::parse_json("{ \"foo\": 1 }")}; +/// sourcemeta::core::metapack_write_json( +/// "/tmp/document.metapack", document, "application/json", +/// sourcemeta::core::MetapackEncoding::GZIP); +/// ``` +SOURCEMETA_CORE_METAPACK_EXPORT +auto metapack_write_json(const std::filesystem::path &destination, + const JSON &document, + const std::string_view media_type, + const MetapackEncoding encoding, + const std::span extension = {}) + -> void; + +/// @ingroup metapack +/// Write a JSON document as the contents of a file, indented for a human to +/// read. A media type or extension too large for the format to record +/// throws. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// const auto document{sourcemeta::core::parse_json("{ \"foo\": 1 }")}; +/// sourcemeta::core::metapack_write_json_pretty( +/// "/tmp/document.metapack", document, "application/json", +/// sourcemeta::core::MetapackEncoding::Identity); +/// ``` +SOURCEMETA_CORE_METAPACK_EXPORT +auto metapack_write_json_pretty( + const std::filesystem::path &destination, const JSON &document, + const std::string_view media_type, const MetapackEncoding encoding, + const std::span extension = {}) -> void; + +/// @ingroup metapack +/// Write text as the contents of a file, terminated by a line feed. A media +/// type or extension too large for the format to record throws. For +/// example: +/// +/// ```cpp +/// #include +/// +/// sourcemeta::core::metapack_write_text( +/// "/tmp/greeting.metapack", "Hello World", "text/plain", +/// sourcemeta::core::MetapackEncoding::Identity); +/// ``` +SOURCEMETA_CORE_METAPACK_EXPORT +auto metapack_write_text(const std::filesystem::path &destination, + const std::string_view contents, + const std::string_view media_type, + const MetapackEncoding encoding, + const std::span extension = {}) + -> void; + +/// @ingroup metapack +/// Write bytes as the contents of a file. A media type or extension too +/// large for the format to record throws. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// constexpr std::array bytes{{0x41, 0x42, 0x43}}; +/// sourcemeta::core::metapack_write_bytes( +/// "/tmp/blob.metapack", bytes, "application/octet-stream", +/// sourcemeta::core::MetapackEncoding::Identity); +/// ``` +SOURCEMETA_CORE_METAPACK_EXPORT +auto metapack_write_bytes(const std::filesystem::path &destination, + const std::span contents, + const std::string_view media_type, + const MetapackEncoding encoding, + const std::span extension = {}) + -> void; + +/// @ingroup metapack +/// Write another file as the contents of a file. A media type or extension +/// too large for the format to record throws. For example: +/// +/// ```cpp +/// #include +/// +/// sourcemeta::core::metapack_write_file( +/// "/tmp/page.metapack", "/tmp/page.html", "text/html", +/// sourcemeta::core::MetapackEncoding::GZIP); +/// ``` +SOURCEMETA_CORE_METAPACK_EXPORT +auto metapack_write_file(const std::filesystem::path &destination, + const std::filesystem::path &source, + const std::string_view media_type, + const MetapackEncoding encoding, + const std::span extension = {}) + -> void; + +} // namespace sourcemeta::core + +#endif diff --git a/src/extra/metapack/metapack_read.cc b/src/extra/metapack/metapack_read.cc new file mode 100644 index 000000000..ab80ce2ef --- /dev/null +++ b/src/extra/metapack/metapack_read.cc @@ -0,0 +1,181 @@ +#include + +#include "signature.h" + +#include +#include +#include +#include + +#include // std::array +#include // assert +#include // std::chrono +#include // std::size_t +#include // std::uint8_t, std::uint32_t, std::uint64_t +#include // std::memcpy +#include // std::filesystem::path +#include // std::optional, std::nullopt +#include // std::string +#include // std::string_view +#include // std::in_range + +namespace sourcemeta::core { + +namespace { + +// The contents as they read once the stored compression is undone +auto decompress(const MetapackView &parsed, + const std::uint64_t maximum_decompression_ratio) + -> std::optional { + assert(!parsed.payload.empty()); + // Integer division floors, so what lands exactly on the limit is only + // within it when nothing is left over + const auto declared_ratio{parsed.header->content_bytes / + parsed.payload.size()}; + if (declared_ratio > maximum_decompression_ratio || + (declared_ratio == maximum_decompression_ratio && + parsed.header->content_bytes % parsed.payload.size() != 0)) { + return std::nullopt; + } + + if (!std::in_range(parsed.header->content_bytes)) { + return std::nullopt; + } + + const auto content_bytes{ + static_cast(parsed.header->content_bytes)}; + + try { + // The declared size bounds the work as well as sizing the buffer, so + // that contents which inflate past it are abandoned as they decode + // rather than after a buffer has grown to hold all of them + auto decompressed{gunzip(parsed.payload.data(), parsed.payload.size(), + content_bytes, content_bytes)}; + // Contents that inflate to less than the declared size are corrupt, or a + // decompression bomb whose small declared size slipped past the guards + if (decompressed.size() != content_bytes) { + return std::nullopt; + } + + return decompressed; + } catch (const GZIPError &) { + return std::nullopt; + } +} + +} // namespace + +auto metapack_parse(const FileView &view) -> std::optional { + if (view.size() < sizeof(MetapackHeader) + sizeof(std::uint32_t)) { + return std::nullopt; + } + + const auto *header{view.as()}; + if (header->magic != METAPACK_MAGIC || + header->format_version != METAPACK_VERSION) { + return std::nullopt; + } + + // An encoding that this revision of the format does not define leaves a + // reader no way to know what the contents are, so the file is refused + // rather than read as if the contents had been stored as they are + if (header->encoding != MetapackEncoding::Identity && + header->encoding != MetapackEncoding::GZIP) { + return std::nullopt; + } + + const auto extension_size_offset{sizeof(MetapackHeader) + + header->media_type_length}; + if (extension_size_offset + sizeof(std::uint32_t) > view.size()) { + return std::nullopt; + } + + // The media type decides where this lands, which is rarely an offset that a + // four byte length may be read from in place + std::uint32_t extension_size{0}; + std::memcpy(&extension_size, view.as(extension_size_offset), + sizeof(extension_size)); + + const auto extension_offset{extension_size_offset + sizeof(std::uint32_t)}; + if (extension_size > view.size() - extension_offset) { + return std::nullopt; + } + + const auto payload_offset{extension_offset + extension_size}; + const auto bytes{view.bytes()}; + return MetapackView{ + .header = header, + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + .media_type = std::string_view{reinterpret_cast( + bytes.data() + sizeof(MetapackHeader)), + header->media_type_length}, + .extension = bytes.subspan(extension_offset, extension_size), + .payload = bytes.subspan(payload_offset)}; +} + +auto metapack_read_json(const MetapackView &parsed, + const std::uint64_t maximum_decompression_ratio) + -> std::optional { + if (parsed.header == nullptr || parsed.payload.empty()) { + return std::nullopt; + } + + if (parsed.header->encoding == MetapackEncoding::GZIP) { + const auto decompressed{decompress(parsed, maximum_decompression_ratio)}; + if (!decompressed.has_value()) { + return std::nullopt; + } + + return try_parse_json(decompressed.value()); + } + + // Contents stored as they are run to the end of the file, so a declared + // size that disagrees means the file was truncated or appended to + if (parsed.header->content_bytes != parsed.payload.size()) { + return std::nullopt; + } + + return try_parse_json(std::string_view{ + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + reinterpret_cast(parsed.payload.data()), + static_cast(parsed.header->content_bytes)}); +} + +auto metapack_read_json(const FileView &view, + const std::uint64_t maximum_decompression_ratio) + -> std::optional { + const auto parsed{metapack_parse(view)}; + if (!parsed.has_value()) { + return std::nullopt; + } + + return metapack_read_json(parsed.value(), maximum_decompression_ratio); +} + +auto metapack_read_json(const std::filesystem::path &path, + const std::uint64_t maximum_decompression_ratio) + -> std::optional { + try { + const FileView view{path}; + return metapack_read_json(view, maximum_decompression_ratio); + } catch (const FileViewError &) { + return std::nullopt; + } +} + +auto metapack_checksum_hex(const MetapackHeader &header) + -> std::array { + std::array result; + bytes_to_hex(header.checksum, result); + return result; +} + +auto metapack_last_modified(const MetapackHeader &header) + -> std::chrono::system_clock::time_point { + const std::chrono::nanoseconds nanoseconds{header.last_modified}; + return std::chrono::system_clock::time_point{ + std::chrono::duration_cast( + nanoseconds)}; +} + +} // namespace sourcemeta::core diff --git a/src/extra/metapack/metapack_write.cc b/src/extra/metapack/metapack_write.cc new file mode 100644 index 000000000..7c728a07f --- /dev/null +++ b/src/extra/metapack/metapack_write.cc @@ -0,0 +1,161 @@ +#include + +#include "signature.h" + +#include +#include +#include +#include + +#include // std::chrono +#include // std::uint16_t, std::uint32_t +#include // std::filesystem::path +#include // std::streamsize +#include // std::ostream +#include // std::span +#include // std::ostringstream +#include // std::string +#include // std::string_view +#include // std::in_range, std::move + +namespace sourcemeta::core { + +namespace { + +auto write_header(std::ostream &output, const std::string_view media_type, + const MetapackEncoding encoding, + const std::span extension, + const std::string_view contents) -> void { + MetapackHeader header; + header.magic = METAPACK_MAGIC; + header.format_version = METAPACK_VERSION; + header.encoding = encoding; + + const auto now{std::chrono::system_clock::now()}; + header.last_modified = std::chrono::duration_cast( + now.time_since_epoch()) + .count(); + header.content_bytes = contents.size(); + header.checksum = sha256_digest(contents); + header.media_type_length = static_cast(media_type.size()); + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + output.write(reinterpret_cast(&header), sizeof(MetapackHeader)); + output.write(media_type.data(), + static_cast(media_type.size())); + + const auto extension_size{static_cast(extension.size())}; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + output.write(reinterpret_cast(&extension_size), + sizeof(extension_size)); + if (!extension.empty()) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + output.write(reinterpret_cast(extension.data()), + static_cast(extension.size())); + } +} + +auto write_metapack(const std::filesystem::path &destination, + const std::string_view media_type, + const MetapackEncoding encoding, + const std::span extension, + const std::string_view contents) -> void { + // A length the header cannot record would be truncated into one it can, + // which silently moves every section that follows it + if (!std::in_range(media_type.size())) { + throw MetapackError{"The media type is too long for this format"}; + } + + if (!std::in_range(extension.size())) { + throw MetapackError{"The extension is too large for this format"}; + } + + // An encoding that this revision of the format does not define would be + // written into a file that no reader of it can accept + if (encoding != MetapackEncoding::Identity && + encoding != MetapackEncoding::GZIP) { + throw MetapackError{"Unknown encoding"}; + } + + // Only contents that are stored compressed are compressed at all + const auto compressed{ + encoding == MetapackEncoding::GZIP + ? gzip( + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + reinterpret_cast(contents.data()), + contents.size()) + : std::string{}}; + write_file(destination, [&](std::ostream &output) -> void { + write_header(output, media_type, encoding, extension, contents); + + if (encoding == MetapackEncoding::GZIP) { + output.write(compressed.data(), + static_cast(compressed.size())); + } else { + output.write(contents.data(), + static_cast(contents.size())); + } + }); +} + +} // namespace + +auto metapack_write_json(const std::filesystem::path &destination, + const JSON &document, + const std::string_view media_type, + const MetapackEncoding encoding, + const std::span extension) + -> void { + std::ostringstream buffer; + stringify(document, buffer); + write_metapack(destination, media_type, encoding, extension, + std::move(buffer).str()); +} + +auto metapack_write_json_pretty(const std::filesystem::path &destination, + const JSON &document, + const std::string_view media_type, + const MetapackEncoding encoding, + const std::span extension) + -> void { + std::ostringstream buffer; + prettify(document, buffer); + write_metapack(destination, media_type, encoding, extension, + std::move(buffer).str()); +} + +auto metapack_write_text(const std::filesystem::path &destination, + const std::string_view contents, + const std::string_view media_type, + const MetapackEncoding encoding, + const std::span extension) + -> void { + std::string line{contents}; + line += '\n'; + write_metapack(destination, media_type, encoding, extension, line); +} + +auto metapack_write_bytes(const std::filesystem::path &destination, + const std::span contents, + const std::string_view media_type, + const MetapackEncoding encoding, + const std::span extension) + -> void { + write_metapack( + destination, media_type, encoding, extension, + std::string_view{ + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + reinterpret_cast(contents.data()), contents.size()}); +} + +auto metapack_write_file(const std::filesystem::path &destination, + const std::filesystem::path &source, + const std::string_view media_type, + const MetapackEncoding encoding, + const std::span extension) + -> void { + write_metapack(destination, media_type, encoding, extension, + read_file_to_string(source)); +} + +} // namespace sourcemeta::core diff --git a/src/extra/metapack/signature.h b/src/extra/metapack/signature.h new file mode 100644 index 000000000..6245ca24e --- /dev/null +++ b/src/extra/metapack/signature.h @@ -0,0 +1,15 @@ +#ifndef SOURCEMETA_CORE_METAPACK_SIGNATURE_H_ +#define SOURCEMETA_CORE_METAPACK_SIGNATURE_H_ + +#include // std::uint16_t, std::uint32_t + +namespace sourcemeta::core { + +// Spells META when read as the four bytes it occupies +inline constexpr std::uint32_t METAPACK_MAGIC{0x4154454D}; + +inline constexpr std::uint16_t METAPACK_VERSION{1}; + +} // namespace sourcemeta::core + +#endif diff --git a/src/lang/io/include/sourcemeta/core/io_fileview.h b/src/lang/io/include/sourcemeta/core/io_fileview.h index c98c47f87..3ce3d2c1e 100644 --- a/src/lang/io/include/sourcemeta/core/io_fileview.h +++ b/src/lang/io/include/sourcemeta/core/io_fileview.h @@ -9,6 +9,7 @@ #include // std::size_t #include // std::uint8_t #include // std::filesystem::path +#include // std::span namespace sourcemeta::core { @@ -43,6 +44,21 @@ class SOURCEMETA_CORE_IO_EXPORT FileView { /// The size of the memory-mapped data in bytes [[nodiscard]] auto size() const noexcept -> std::size_t; + /// The memory-mapped data as a sequence of bytes, which is what lets a + /// reader hand out views into the mapping rather than copies of it. For + /// example: + /// + /// ```cpp + /// #include + /// #include + /// + /// const sourcemeta::core::FileView view{"/path/to/file.bin"}; + /// assert(view.bytes().size() == view.size()); + /// ``` + [[nodiscard]] auto bytes() const noexcept -> std::span { + return {this->data_, this->size_}; + } + /// Interpret the memory-mapped data as a pointer to T at the given offset. /// The caller must ensure that the offset yields a pointer suitably aligned /// for T, as dereferencing a misaligned pointer is undefined behavior. diff --git a/test/io/io_fileview_test.cc b/test/io/io_fileview_test.cc index 36751ea80..42924c26e 100644 --- a/test/io/io_fileview_test.cc +++ b/test/io/io_fileview_test.cc @@ -11,6 +11,29 @@ TEST(size) { EXPECT_EQ(view.size(), 20); } +TEST(bytes) { + const sourcemeta::core::FileView view{std::filesystem::path{STUBS_DIRECTORY} / + "fileview.bin"}; + const auto bytes{view.bytes()}; + EXPECT_EQ(bytes.size(), 20); + EXPECT_EQ(bytes.data(), view.as(0)); + EXPECT_EQ(bytes[0], 0x46); + EXPECT_EQ(bytes[1], 0x49); + EXPECT_EQ(bytes[2], 0x57); + EXPECT_EQ(bytes[3], 0x56); + EXPECT_EQ(bytes[19], 0xCA); +} + +TEST(bytes_on_an_empty_file) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".fileview-"}; + const auto path{directory.path() / "empty.bin"}; + sourcemeta::core::write_file(path, std::string_view{""}); + + const sourcemeta::core::FileView view{path}; + EXPECT_TRUE(view.bytes().empty()); +} + TEST(as_header) { struct Header { std::uint32_t magic; diff --git a/test/metapack/CMakeLists.txt b/test/metapack/CMakeLists.txt new file mode 100644 index 000000000..7d8ce0d58 --- /dev/null +++ b/test/metapack/CMakeLists.txt @@ -0,0 +1,9 @@ +sourcemeta_test(NAMESPACE sourcemeta PROJECT core NAME metapack + SOURCES + metapack_corrupt_test.cc + metapack_parse_test.cc + metapack_read_test.cc + metapack_write_test.cc) + +target_link_libraries(sourcemeta_core_metapack_unit + PRIVATE sourcemeta::core::metapack) diff --git a/test/metapack/metapack_corrupt_test.cc b/test/metapack/metapack_corrupt_test.cc new file mode 100644 index 000000000..41ee2b3fc --- /dev/null +++ b/test/metapack/metapack_corrupt_test.cc @@ -0,0 +1,289 @@ +#include +#include +#include +#include + +#include // offsetof, std::size_t +#include // std::uint16_t, std::uint64_t +#include // std::filesystem +#include // std::fstream, std::ofstream +#include // std::ios, std::streamoff +#include // std::string + +namespace { + +class MetapackCorruption { +protected: + // Overwrite the trailing bytes so that compressed contents fail to inflate + static auto corrupt_last_bytes(const std::filesystem::path &path, + const std::size_t count) -> void { + const auto size{std::filesystem::file_size(path)}; + std::fstream file{path, std::ios::binary | std::ios::in | std::ios::out}; + EXPECT_TRUE(file.good()); + file.seekp(static_cast(size - count), std::ios::beg); + const std::string replacement(count, static_cast(0xFF)); + file.write(replacement.data(), static_cast(count)); + } + + // Rewrite the decoded size the header advertises + static auto set_content_bytes(const std::filesystem::path &path, + const std::uint64_t value) -> void { + std::fstream file{path, std::ios::binary | std::ios::in | std::ios::out}; + EXPECT_TRUE(file.good()); + file.seekp(offsetof(sourcemeta::core::MetapackHeader, content_bytes), + std::ios::beg); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + file.write(reinterpret_cast(&value), sizeof(value)); + } + + // Rewrite the media type length the header advertises + static auto set_media_type_length(const std::filesystem::path &path, + const std::uint16_t value) -> void { + std::fstream file{path, std::ios::binary | std::ios::in | std::ios::out}; + EXPECT_TRUE(file.good()); + file.seekp(offsetof(sourcemeta::core::MetapackHeader, media_type_length), + std::ios::beg); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + file.write(reinterpret_cast(&value), sizeof(value)); + } + + // Rewrite the revision of the format the header declares + static auto set_format_version(const std::filesystem::path &path, + const std::uint16_t value) -> void { + std::fstream file{path, std::ios::binary | std::ios::in | std::ios::out}; + EXPECT_TRUE(file.good()); + file.seekp(offsetof(sourcemeta::core::MetapackHeader, format_version), + std::ios::beg); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + file.write(reinterpret_cast(&value), sizeof(value)); + } + + // Rewrite the encoding the header declares + static auto set_encoding(const std::filesystem::path &path, + const std::uint8_t value) -> void { + std::fstream file{path, std::ios::binary | std::ios::in | std::ios::out}; + EXPECT_TRUE(file.good()); + file.seekp(offsetof(sourcemeta::core::MetapackHeader, encoding), + std::ios::beg); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + file.write(reinterpret_cast(&value), sizeof(value)); + } + + // Rewrite the extension length, which the media type decides the position of + static auto set_extension_size(const std::filesystem::path &path, + const std::size_t media_type_length, + const std::uint32_t value) -> void { + std::fstream file{path, std::ios::binary | std::ios::in | std::ios::out}; + EXPECT_TRUE(file.good()); + const auto offset{sizeof(sourcemeta::core::MetapackHeader) + + media_type_length}; + file.seekp(static_cast(offset), std::ios::beg); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + file.write(reinterpret_cast(&value), sizeof(value)); + } + + // The mapping is released before returning, as Windows refuses to open a + // mapped file for writing and every caller goes on to rewrite the header + static auto payload_size_of(const std::filesystem::path &path) + -> std::size_t { + const sourcemeta::core::FileView view{path}; + return sourcemeta::core::metapack_parse(view).value().payload.size(); + } + + static auto write_document(const std::filesystem::path &path, + const sourcemeta::core::MetapackEncoding encoding) + -> void { + auto document{sourcemeta::core::JSON::make_object()}; + document.assign("hello", sourcemeta::core::JSON{"world"}); + sourcemeta::core::metapack_write_json(path, document, "application/json", + encoding); + } +}; + +} // namespace + +TEST_F(MetapackCorruption, parse_no_value_on_an_unknown_format_version) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + write_document(path, sourcemeta::core::MetapackEncoding::Identity); + + // A revision that this implementation does not speak + set_format_version(path, 2); + + const sourcemeta::core::FileView view{path}; + EXPECT_FALSE(sourcemeta::core::metapack_parse(view).has_value()); +} + +TEST_F(MetapackCorruption, + parse_no_value_when_the_media_type_runs_past_the_end) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + write_document(path, sourcemeta::core::MetapackEncoding::Identity); + + set_media_type_length(path, 4096); + + const sourcemeta::core::FileView view{path}; + EXPECT_FALSE(sourcemeta::core::metapack_parse(view).has_value()); +} + +TEST_F(MetapackCorruption, read_json_no_value_when_content_bytes_overstates) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + write_document(path, sourcemeta::core::MetapackEncoding::Identity); + + set_content_bytes(path, 999999); + + EXPECT_FALSE(sourcemeta::core::metapack_read_json(path).has_value()); +} + +TEST_F(MetapackCorruption, read_json_no_value_on_a_damaged_gzip_trailer) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + write_document(path, sourcemeta::core::MetapackEncoding::GZIP); + + corrupt_last_bytes(path, 4); + + EXPECT_FALSE(sourcemeta::core::metapack_read_json(path).has_value()); +} + +TEST_F(MetapackCorruption, read_json_no_value_on_contents_that_are_not_json) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + + sourcemeta::core::metapack_write_text( + path, "this is not json {{{", "text/plain", + sourcemeta::core::MetapackEncoding::Identity); + + EXPECT_FALSE(sourcemeta::core::metapack_read_json(path).has_value()); +} + +TEST_F(MetapackCorruption, read_json_no_value_when_the_decoded_size_disagrees) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + write_document(path, sourcemeta::core::MetapackEncoding::GZIP); + + // A small decoded size survives the ratio guard but no longer matches what + // the contents actually inflate to + set_content_bytes(path, 2); + + EXPECT_FALSE(sourcemeta::core::metapack_read_json(path).has_value()); +} + +TEST_F(MetapackCorruption, read_json_no_value_past_the_decompression_ratio) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + auto document{sourcemeta::core::JSON::make_object()}; + document.assign("hello", sourcemeta::core::JSON{std::string(4096, 'a')}); + + sourcemeta::core::metapack_write_json( + path, document, "application/json", + sourcemeta::core::MetapackEncoding::GZIP); + + EXPECT_TRUE(sourcemeta::core::metapack_read_json(path).has_value()); + EXPECT_FALSE(sourcemeta::core::metapack_read_json(path, 1).has_value()); +} + +TEST_F(MetapackCorruption, read_json_no_value_when_content_bytes_is_enormous) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + write_document(path, sourcemeta::core::MetapackEncoding::GZIP); + + set_content_bytes(path, 1099511627776); + + EXPECT_FALSE(sourcemeta::core::metapack_read_json(path).has_value()); +} + +TEST_F(MetapackCorruption, parse_no_value_on_an_encoding_this_revision_lacks) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + write_document(path, sourcemeta::core::MetapackEncoding::Identity); + + // Were this read as contents stored the way they are, the file would parse + // and hand back a payload that nothing knows how to interpret + set_encoding(path, 7); + + const sourcemeta::core::FileView view{path}; + EXPECT_FALSE(sourcemeta::core::metapack_parse(view).has_value()); + EXPECT_FALSE(sourcemeta::core::metapack_read_json(path).has_value()); +} + +TEST_F(MetapackCorruption, + parse_no_value_when_the_extension_runs_past_the_end) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + write_document(path, sourcemeta::core::MetapackEncoding::Identity); + + set_extension_size(path, std::string_view{"application/json"}.size(), 4096); + + const sourcemeta::core::FileView view{path}; + EXPECT_FALSE(sourcemeta::core::metapack_parse(view).has_value()); +} + +TEST_F(MetapackCorruption, read_json_no_value_when_content_bytes_understates) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + + sourcemeta::core::metapack_write_text( + path, "12345", "application/json", + sourcemeta::core::MetapackEncoding::Identity); + + // Were the declared size taken as a prefix of the contents, this would + // parse as the number one rather than being refused + set_content_bytes(path, 1); + + EXPECT_FALSE(sourcemeta::core::metapack_read_json(path).has_value()); +} + +TEST_F(MetapackCorruption, read_json_no_value_on_contents_appended_to) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + write_document(path, sourcemeta::core::MetapackEncoding::Identity); + + std::ofstream file{path, std::ios::binary | std::ios::app}; + EXPECT_TRUE(file.good()); + file << "trailing"; + file.close(); + + EXPECT_FALSE(sourcemeta::core::metapack_read_json(path).has_value()); +} + +TEST_F(MetapackCorruption, + read_json_no_value_on_a_ratio_that_lands_over_by_one) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + write_document(path, sourcemeta::core::MetapackEncoding::GZIP); + + const auto payload_size{payload_size_of(path)}; + + // One byte beyond what a ratio of two allows, which integer division alone + // would floor back into range + set_content_bytes(path, (payload_size * 2) + 1); + + EXPECT_FALSE(sourcemeta::core::metapack_read_json(path, 2).has_value()); +} + +TEST_F(MetapackCorruption, read_json_no_value_when_contents_inflate_short) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + write_document(path, sourcemeta::core::MetapackEncoding::GZIP); + + // Large enough to overstate what the contents inflate to, yet small enough + // that the expansion it declares is still a plausible one + set_content_bytes(path, 100); + + EXPECT_FALSE(sourcemeta::core::metapack_read_json(path).has_value()); +} diff --git a/test/metapack/metapack_parse_test.cc b/test/metapack/metapack_parse_test.cc new file mode 100644 index 000000000..67d267d1a --- /dev/null +++ b/test/metapack/metapack_parse_test.cc @@ -0,0 +1,195 @@ +#include +#include +#include +#include + +#include // std::array +#include // std::int32_t, std::uint8_t, std::uint16_t +#include // std::filesystem +#include // std::string +#include // std::string_view +#include // std::vector + +#pragma pack(push, 1) +struct TestExtension { + std::int32_t value; + std::uint16_t name_length; +}; +#pragma pack(pop) + +static auto extension_bytes() -> std::vector { + TestExtension header{}; + header.value = 42; + header.name_length = 5; + + std::vector result; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + const auto *begin{reinterpret_cast(&header)}; + result.insert(result.end(), begin, begin + sizeof(TestExtension)); + result.push_back('h'); + result.push_back('e'); + result.push_back('l'); + result.push_back('l'); + result.push_back('o'); + return result; +} + +TEST(parse_no_value_on_an_empty_file) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "empty.bin"}; + sourcemeta::core::write_file(path, std::string_view{""}); + + const sourcemeta::core::FileView view{path}; + EXPECT_FALSE(sourcemeta::core::metapack_parse(view).has_value()); +} + +TEST(parse_no_value_on_a_file_shorter_than_a_header) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "short.bin"}; + sourcemeta::core::write_file(path, std::string_view{"META"}); + + const sourcemeta::core::FileView view{path}; + EXPECT_FALSE(sourcemeta::core::metapack_parse(view).has_value()); +} + +TEST(parse_no_value_on_a_foreign_magic_number) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "foreign.bin"}; + const std::string contents(128, '\0'); + sourcemeta::core::write_file(path, contents); + + const sourcemeta::core::FileView view{path}; + EXPECT_FALSE(sourcemeta::core::metapack_parse(view).has_value()); +} + +TEST(parse_without_an_extension) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + const auto document{sourcemeta::core::JSON::make_object()}; + + sourcemeta::core::metapack_write_json( + path, document, "application/json", + sourcemeta::core::MetapackEncoding::Identity); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.media_type, "application/json"); + EXPECT_TRUE(parsed.extension.empty()); + EXPECT_EQ(parsed.payload.size(), 2); +} + +TEST(parse_with_an_empty_media_type) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + const auto document{sourcemeta::core::JSON::make_object()}; + + sourcemeta::core::metapack_write_json( + path, document, "", sourcemeta::core::MetapackEncoding::Identity); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.header->media_type_length, 0); + EXPECT_TRUE(parsed.media_type.empty()); + EXPECT_TRUE(parsed.extension.empty()); + EXPECT_EQ(parsed.payload.size(), 2); +} + +TEST(parse_with_an_extension) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + const auto document{sourcemeta::core::JSON::make_object()}; + const auto extension{extension_bytes()}; + + sourcemeta::core::metapack_write_json( + path, document, "application/json", + sourcemeta::core::MetapackEncoding::Identity, extension); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.extension.size(), 11); + + const auto *read_extension{ + sourcemeta::core::metapack_extension_as(parsed.extension)}; + EXPECT_NE(read_extension, nullptr); + EXPECT_EQ(read_extension->value, 42); + EXPECT_EQ(read_extension->name_length, 5); + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + const auto *name{reinterpret_cast(parsed.extension.data() + + sizeof(TestExtension))}; + EXPECT_EQ(std::string_view(name, read_extension->name_length), "hello"); + EXPECT_EQ(parsed.payload.size(), 2); +} + +TEST(extension_as_no_value_when_it_is_too_small) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + const auto document{sourcemeta::core::JSON::make_object()}; + const std::array extension{{0x01, 0x02}}; + + sourcemeta::core::metapack_write_json( + path, document, "application/json", + sourcemeta::core::MetapackEncoding::Identity, extension); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.extension.size(), 2); + EXPECT_EQ( + sourcemeta::core::metapack_extension_as(parsed.extension), + nullptr); +} + +TEST(parse_with_an_extension_and_compressed_contents) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + auto document{sourcemeta::core::JSON::make_object()}; + document.assign("hello", sourcemeta::core::JSON{std::string(4096, 'x')}); + const auto extension{extension_bytes()}; + + sourcemeta::core::metapack_write_json( + path, document, "application/json", + sourcemeta::core::MetapackEncoding::GZIP, extension); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.media_type, "application/json"); + EXPECT_EQ(parsed.header->encoding, sourcemeta::core::MetapackEncoding::GZIP); + EXPECT_EQ(parsed.extension.size(), 11); + EXPECT_LT(parsed.payload.size(), parsed.header->content_bytes); + + const auto *read_extension{ + sourcemeta::core::metapack_extension_as(parsed.extension)}; + EXPECT_NE(read_extension, nullptr); + EXPECT_EQ(read_extension->value, 42); + + const auto result{sourcemeta::core::metapack_read_json(parsed).value()}; + EXPECT_EQ(result.at("hello").to_string(), std::string(4096, 'x')); +} + +TEST(parse_with_a_long_media_type) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + const auto document{sourcemeta::core::JSON::make_object()}; + const std::string media_type{"application/" + std::string(1000, 'a')}; + const auto extension{extension_bytes()}; + + sourcemeta::core::metapack_write_json( + path, document, media_type, sourcemeta::core::MetapackEncoding::Identity, + extension); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.header->media_type_length, 1012); + EXPECT_EQ(parsed.media_type, media_type); + EXPECT_EQ(parsed.extension.size(), 11); + EXPECT_EQ(parsed.payload.size(), 2); +} diff --git a/test/metapack/metapack_read_test.cc b/test/metapack/metapack_read_test.cc new file mode 100644 index 000000000..54aa43d33 --- /dev/null +++ b/test/metapack/metapack_read_test.cc @@ -0,0 +1,170 @@ +#include +#include +#include +#include + +#include // std::array +#include // std::chrono +#include // std::uint8_t +#include // std::filesystem +#include // std::span +#include // std::string + +TEST(read_json_identity_from_a_path) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + auto document{sourcemeta::core::JSON::make_object()}; + document.assign("hello", sourcemeta::core::JSON{"world"}); + + sourcemeta::core::metapack_write_json( + path, document, "application/json", + sourcemeta::core::MetapackEncoding::Identity); + + const auto result{sourcemeta::core::metapack_read_json(path).value()}; + EXPECT_TRUE(result.is_object()); + EXPECT_EQ(result.size(), 1); + EXPECT_EQ(result.at("hello").to_string(), "world"); +} + +TEST(read_json_gzip_from_a_path) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + auto document{sourcemeta::core::JSON::make_object()}; + document.assign("foo", sourcemeta::core::JSON{42}); + + sourcemeta::core::metapack_write_json( + path, document, "application/json", + sourcemeta::core::MetapackEncoding::GZIP); + + const auto result{sourcemeta::core::metapack_read_json(path).value()}; + EXPECT_TRUE(result.is_object()); + EXPECT_EQ(result.size(), 1); + EXPECT_TRUE(result.at("foo").is_integer()); + EXPECT_EQ(result.at("foo").to_integer(), 42); +} + +TEST(read_json_pretty_from_a_path) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + auto document{sourcemeta::core::JSON::make_object()}; + document.assign("key", sourcemeta::core::JSON{"value"}); + + sourcemeta::core::metapack_write_json_pretty( + path, document, "application/schema+json", + sourcemeta::core::MetapackEncoding::GZIP); + + const auto result{sourcemeta::core::metapack_read_json(path).value()}; + EXPECT_TRUE(result.is_object()); + EXPECT_EQ(result.size(), 1); + EXPECT_EQ(result.at("key").to_string(), "value"); +} + +TEST(read_json_from_a_view) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + auto document{sourcemeta::core::JSON::make_object()}; + document.assign("hello", sourcemeta::core::JSON{"world"}); + + sourcemeta::core::metapack_write_json( + path, document, "application/json", + sourcemeta::core::MetapackEncoding::GZIP); + + const sourcemeta::core::FileView view{path}; + const auto result{sourcemeta::core::metapack_read_json(view).value()}; + EXPECT_TRUE(result.is_object()); + EXPECT_EQ(result.size(), 1); + EXPECT_EQ(result.at("hello").to_string(), "world"); +} + +TEST(read_json_from_a_parsed_file) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + auto document{sourcemeta::core::JSON::make_object()}; + document.assign("hello", sourcemeta::core::JSON{"world"}); + + sourcemeta::core::metapack_write_json( + path, document, "application/json", + sourcemeta::core::MetapackEncoding::Identity); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + const auto result{sourcemeta::core::metapack_read_json(parsed).value()}; + EXPECT_TRUE(result.is_object()); + EXPECT_EQ(result.size(), 1); + EXPECT_EQ(result.at("hello").to_string(), "world"); +} + +TEST(read_json_no_value_when_the_path_does_not_exist) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + EXPECT_FALSE(sourcemeta::core::metapack_read_json(directory.path() / + "missing.metapack") + .has_value()); +} + +TEST(read_json_no_value_when_the_file_has_no_header) { + const std::array bytes{{0x7B, 0x7D}}; + const sourcemeta::core::MetapackView parsed{ + .header = nullptr, + .media_type = {}, + .extension = {}, + .payload = std::span{bytes}}; + + EXPECT_FALSE(sourcemeta::core::metapack_read_json(parsed).has_value()); +} + +TEST(read_json_no_value_when_the_contents_are_empty) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "empty.metapack"}; + + sourcemeta::core::metapack_write_bytes( + path, {}, "application/json", + sourcemeta::core::MetapackEncoding::Identity); + + EXPECT_FALSE(sourcemeta::core::metapack_read_json(path).has_value()); +} + +TEST(checksum_hex_renders_every_digest_byte) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "greeting.metapack"}; + + sourcemeta::core::metapack_write_text( + path, "Hello World", "text/plain", + sourcemeta::core::MetapackEncoding::Identity); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + const auto checksum{sourcemeta::core::metapack_checksum_hex(*parsed.header)}; + + EXPECT_EQ(checksum.size(), 64); + // The SHA-256 of `Hello World` followed by a line feed + EXPECT_EQ(std::string(checksum.data(), checksum.size()), + "d2a84f4b8b650937ec8f73cd8be2c74add5a911ba64df27458ed8229da804a26"); +} + +TEST(last_modified_falls_between_the_writes_around_it) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "greeting.metapack"}; + + const auto before{std::chrono::system_clock::now()}; + sourcemeta::core::metapack_write_text( + path, "Hello World", "text/plain", + sourcemeta::core::MetapackEncoding::Identity); + const auto after{std::chrono::system_clock::now()}; + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + const auto last_modified{ + sourcemeta::core::metapack_last_modified(*parsed.header)}; + + EXPECT_GE(last_modified, before); + EXPECT_LE(last_modified, after); +} diff --git a/test/metapack/metapack_write_test.cc b/test/metapack/metapack_write_test.cc new file mode 100644 index 000000000..201f123ad --- /dev/null +++ b/test/metapack/metapack_write_test.cc @@ -0,0 +1,265 @@ +#include +#include +#include +#include + +#include // std::ranges::equal +#include // std::array +#include // std::uint8_t +#include // std::filesystem +#include // std::string +#include // std::string_view +#include // std::vector + +static auto payload_as_text(const sourcemeta::core::MetapackView &parsed) + -> std::string_view { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return std::string_view{reinterpret_cast(parsed.payload.data()), + parsed.payload.size()}; +} + +TEST(write_json_identity) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + auto document{sourcemeta::core::JSON::make_object()}; + document.assign("hello", sourcemeta::core::JSON{"world"}); + + sourcemeta::core::metapack_write_json( + path, document, "application/json", + sourcemeta::core::MetapackEncoding::Identity); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + // The magic number spells META when read as the four bytes it occupies + EXPECT_EQ(parsed.header->magic, 0x4154454D); + EXPECT_EQ(parsed.header->format_version, 1); + EXPECT_EQ(parsed.header->encoding, + sourcemeta::core::MetapackEncoding::Identity); + EXPECT_EQ(parsed.header->reserved, 0); + EXPECT_EQ(parsed.media_type, "application/json"); + EXPECT_TRUE(parsed.extension.empty()); + EXPECT_EQ(payload_as_text(parsed), "{\"hello\":\"world\"}"); + EXPECT_EQ(parsed.header->content_bytes, parsed.payload.size()); +} + +TEST(write_json_gzip) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + auto document{sourcemeta::core::JSON::make_object()}; + document.assign("hello", sourcemeta::core::JSON{std::string(4096, 'x')}); + + sourcemeta::core::metapack_write_json( + path, document, "application/json", + sourcemeta::core::MetapackEncoding::GZIP); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.header->encoding, sourcemeta::core::MetapackEncoding::GZIP); + EXPECT_EQ(parsed.media_type, "application/json"); + EXPECT_EQ(parsed.header->content_bytes, 4108); + EXPECT_LT(parsed.payload.size(), parsed.header->content_bytes); +} + +TEST(write_json_pretty_identity) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + auto document{sourcemeta::core::JSON::make_object()}; + document.assign("hello", sourcemeta::core::JSON{"world"}); + + sourcemeta::core::metapack_write_json_pretty( + path, document, "application/schema+json", + sourcemeta::core::MetapackEncoding::Identity); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.media_type, "application/schema+json"); + EXPECT_EQ(payload_as_text(parsed), "{\n \"hello\": \"world\"\n}"); +} + +TEST(write_text_appends_a_trailing_line_feed) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "greeting.metapack"}; + + sourcemeta::core::metapack_write_text( + path, "Hello World", "text/plain", + sourcemeta::core::MetapackEncoding::Identity); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.media_type, "text/plain"); + EXPECT_EQ(parsed.header->encoding, + sourcemeta::core::MetapackEncoding::Identity); + EXPECT_TRUE(parsed.extension.empty()); + EXPECT_EQ(payload_as_text(parsed), "Hello World\n"); + EXPECT_EQ(parsed.header->content_bytes, 12); +} + +TEST(write_text_gzip_records_the_decoded_size) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "greeting.metapack"}; + const std::string contents(4096, 'a'); + + sourcemeta::core::metapack_write_text( + path, contents, "text/plain", sourcemeta::core::MetapackEncoding::GZIP); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.media_type, "text/plain"); + EXPECT_EQ(parsed.header->encoding, sourcemeta::core::MetapackEncoding::GZIP); + EXPECT_EQ(parsed.header->content_bytes, 4097); + EXPECT_LT(parsed.payload.size(), parsed.header->content_bytes); +} + +TEST(write_bytes_identity_keeps_the_contents_exact) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "blob.metapack"}; + const std::array contents{{0x00, 0x01, 0xFF, 0x0A, 0x00}}; + + sourcemeta::core::metapack_write_bytes( + path, contents, "application/octet-stream", + sourcemeta::core::MetapackEncoding::Identity); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.media_type, "application/octet-stream"); + EXPECT_EQ(parsed.header->encoding, + sourcemeta::core::MetapackEncoding::Identity); + EXPECT_TRUE(parsed.extension.empty()); + EXPECT_EQ(parsed.payload.size(), 5); + EXPECT_EQ(parsed.header->content_bytes, 5); + EXPECT_TRUE(std::ranges::equal(parsed.payload, contents)); +} + +TEST(write_bytes_gzip_records_the_decoded_size) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "blob.metapack"}; + const std::vector contents(4096, 0x41); + + sourcemeta::core::metapack_write_bytes( + path, contents, "application/octet-stream", + sourcemeta::core::MetapackEncoding::GZIP); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.media_type, "application/octet-stream"); + EXPECT_EQ(parsed.header->encoding, sourcemeta::core::MetapackEncoding::GZIP); + EXPECT_EQ(parsed.header->content_bytes, 4096); + EXPECT_LT(parsed.payload.size(), parsed.header->content_bytes); +} + +TEST(write_file_stores_the_source_contents) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto source{directory.path() / "page.html"}; + const auto path{directory.path() / "page.metapack"}; + sourcemeta::core::write_file(source, std::string_view{"

Hello

"}); + + sourcemeta::core::metapack_write_file( + path, source, "text/html", sourcemeta::core::MetapackEncoding::Identity); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.media_type, "text/html"); + EXPECT_EQ(payload_as_text(parsed), "

Hello

"); + EXPECT_EQ(parsed.header->content_bytes, 12); +} + +TEST(write_json_records_the_checksum_of_the_decoded_contents) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto identity{directory.path() / "identity.metapack"}; + const auto compressed{directory.path() / "compressed.metapack"}; + auto document{sourcemeta::core::JSON::make_object()}; + document.assign("hello", sourcemeta::core::JSON{"world"}); + + sourcemeta::core::metapack_write_json( + identity, document, "application/json", + sourcemeta::core::MetapackEncoding::Identity); + sourcemeta::core::metapack_write_json( + compressed, document, "application/json", + sourcemeta::core::MetapackEncoding::GZIP); + + const sourcemeta::core::FileView identity_view{identity}; + const sourcemeta::core::FileView compressed_view{compressed}; + const auto parsed_identity{ + sourcemeta::core::metapack_parse(identity_view).value()}; + const auto parsed_compressed{ + sourcemeta::core::metapack_parse(compressed_view).value()}; + + const auto identity_checksum{ + sourcemeta::core::metapack_checksum_hex(*parsed_identity.header)}; + const auto compressed_checksum{ + sourcemeta::core::metapack_checksum_hex(*parsed_compressed.header)}; + + // The SHA-256 of `{"hello":"world"}` + EXPECT_EQ( + std::string_view(identity_checksum.data(), identity_checksum.size()), + "93a23971a914e5eacbf0a8d25154cda309c3c1c72fbb9914d47c60f3cb681588"); + EXPECT_EQ( + std::string_view(compressed_checksum.data(), compressed_checksum.size()), + "93a23971a914e5eacbf0a8d25154cda309c3c1c72fbb9914d47c60f3cb681588"); +} + +TEST(write_file_gzip_records_the_decoded_size) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto source{directory.path() / "page.html"}; + const auto path{directory.path() / "page.metapack"}; + const std::string contents(4096, 'z'); + sourcemeta::core::write_file(source, contents); + + sourcemeta::core::metapack_write_file( + path, source, "text/html", sourcemeta::core::MetapackEncoding::GZIP); + + const sourcemeta::core::FileView view{path}; + const auto parsed{sourcemeta::core::metapack_parse(view).value()}; + EXPECT_EQ(parsed.media_type, "text/html"); + EXPECT_EQ(parsed.header->encoding, sourcemeta::core::MetapackEncoding::GZIP); + EXPECT_EQ(parsed.header->content_bytes, 4096); + EXPECT_LT(parsed.payload.size(), parsed.header->content_bytes); +} + +TEST(write_refuses_a_media_type_the_header_cannot_record) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + const auto document{sourcemeta::core::JSON::make_object()}; + const std::string media_type( + sourcemeta::core::METAPACK_MAXIMUM_MEDIA_TYPE_BYTES + 1, 'a'); + + try { + sourcemeta::core::metapack_write_json( + path, document, media_type, + sourcemeta::core::MetapackEncoding::Identity); + FAIL(); + } catch (const sourcemeta::core::MetapackError &error) { + EXPECT_STREQ(error.what(), "The media type is too long for this format"); + } + + EXPECT_FALSE(std::filesystem::exists(path)); +} + +TEST(write_refuses_an_encoding_this_revision_lacks) { + const sourcemeta::core::TemporaryDirectory directory{ + std::filesystem::temp_directory_path(), ".metapack-"}; + const auto path{directory.path() / "document.metapack"}; + const auto document{sourcemeta::core::JSON::make_object()}; + + try { + sourcemeta::core::metapack_write_json( + path, document, "application/json", + static_cast(7)); + FAIL(); + } catch (const sourcemeta::core::MetapackError &error) { + EXPECT_STREQ(error.what(), "Unknown encoding"); + } + + EXPECT_FALSE(std::filesystem::exists(path)); +} diff --git a/test/packaging/find_package/CMakeLists.txt b/test/packaging/find_package/CMakeLists.txt index 4a23d1d60..219768be1 100644 --- a/test/packaging/find_package/CMakeLists.txt +++ b/test/packaging/find_package/CMakeLists.txt @@ -34,6 +34,7 @@ target_link_libraries(core_hello PRIVATE sourcemeta::core::gzip) target_link_libraries(core_hello PRIVATE sourcemeta::core::html) target_link_libraries(core_hello PRIVATE sourcemeta::core::css) target_link_libraries(core_hello PRIVATE sourcemeta::core::markdown) +target_link_libraries(core_hello PRIVATE sourcemeta::core::metapack) target_link_libraries(core_hello PRIVATE sourcemeta::core::diff) target_link_libraries(core_hello PRIVATE sourcemeta::core::options) target_link_libraries(core_hello PRIVATE sourcemeta::core::preprocessor) diff --git a/test/packaging/find_package/hello.cc b/test/packaging/find_package/hello.cc index 7d28ede7d..0bbb27dbf 100644 --- a/test/packaging/find_package/hello.cc +++ b/test/packaging/find_package/hello.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include