From ee62460ffc9d6d31ea90d44a987dd86c65791918 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 17 Oct 2026 14:58:47 -0600 Subject: [PATCH 01/24] tools: add the modern tree's save and bank codecs with CI rebuild/modern/save reads and rewrites retail saves and Profile.bin with portable C++23 types; every game-written save re-encodes byte for byte. LZO for the quest/faction text is an optional GPL download (FABLE_SAVE_WITH_LZO), never vendored. rebuild/modern/bank round-trips .big bank archives with payload views and the BIG.md minimal-repack writer. modern-tree.yml builds and tests the tree on Ubuntu GCC (with LZO) and Windows MSVC x64 (without). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/modern-tree.yml | 41 ++ rebuild/modern/CMakeLists.txt | 4 +- rebuild/modern/README.md | 8 + rebuild/modern/bank/CMakeLists.txt | 38 ++ rebuild/modern/bank/README.md | 51 +++ .../bank/include/fable/bank/BigBank.hpp | 113 +++++ rebuild/modern/bank/src/BigBank.cpp | 350 ++++++++++++++++ rebuild/modern/bank/tests/BigBankTests.cpp | 149 +++++++ .../modern/bank/tools/fable_bank_check.cpp | 157 +++++++ rebuild/modern/save/CMakeLists.txt | 127 ++++++ rebuild/modern/save/README.md | 121 ++++++ .../save/include/fable/save/Entities.hpp | 97 +++++ .../save/include/fable/save/PersistStream.hpp | 83 ++++ .../save/include/fable/save/Profile.hpp | 58 +++ .../save/include/fable/save/Regions.hpp | 38 ++ .../save/include/fable/save/SaveBody.hpp | 70 ++++ .../save/include/fable/save/SaveContainer.hpp | 67 +++ .../save/include/fable/save/SaveHeader.hpp | 57 +++ .../save/include/fable/save/TextSections.hpp | 69 +++ rebuild/modern/save/src/Entities.cpp | 261 ++++++++++++ rebuild/modern/save/src/Lzo.cpp | 257 ++++++++++++ rebuild/modern/save/src/Lzo.hpp | 33 ++ rebuild/modern/save/src/PersistStream.cpp | 241 +++++++++++ rebuild/modern/save/src/Profile.cpp | 116 ++++++ rebuild/modern/save/src/Regions.cpp | 67 +++ rebuild/modern/save/src/SaveBody.cpp | 152 +++++++ rebuild/modern/save/src/SaveContainer.cpp | 166 ++++++++ rebuild/modern/save/src/SaveHeader.cpp | 135 ++++++ rebuild/modern/save/src/TextSections.cpp | 222 ++++++++++ rebuild/modern/save/src/Zlib.cpp | 63 +++ rebuild/modern/save/src/Zlib.hpp | 28 ++ .../modern/save/tests/SaveContainerTests.cpp | 393 ++++++++++++++++++ .../modern/save/tools/fable_save_check.cpp | 220 ++++++++++ 33 files changed, 4051 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/modern-tree.yml create mode 100644 rebuild/modern/bank/CMakeLists.txt create mode 100644 rebuild/modern/bank/README.md create mode 100644 rebuild/modern/bank/include/fable/bank/BigBank.hpp create mode 100644 rebuild/modern/bank/src/BigBank.cpp create mode 100644 rebuild/modern/bank/tests/BigBankTests.cpp create mode 100644 rebuild/modern/bank/tools/fable_bank_check.cpp create mode 100644 rebuild/modern/save/CMakeLists.txt create mode 100644 rebuild/modern/save/README.md create mode 100644 rebuild/modern/save/include/fable/save/Entities.hpp create mode 100644 rebuild/modern/save/include/fable/save/PersistStream.hpp create mode 100644 rebuild/modern/save/include/fable/save/Profile.hpp create mode 100644 rebuild/modern/save/include/fable/save/Regions.hpp create mode 100644 rebuild/modern/save/include/fable/save/SaveBody.hpp create mode 100644 rebuild/modern/save/include/fable/save/SaveContainer.hpp create mode 100644 rebuild/modern/save/include/fable/save/SaveHeader.hpp create mode 100644 rebuild/modern/save/include/fable/save/TextSections.hpp create mode 100644 rebuild/modern/save/src/Entities.cpp create mode 100644 rebuild/modern/save/src/Lzo.cpp create mode 100644 rebuild/modern/save/src/Lzo.hpp create mode 100644 rebuild/modern/save/src/PersistStream.cpp create mode 100644 rebuild/modern/save/src/Profile.cpp create mode 100644 rebuild/modern/save/src/Regions.cpp create mode 100644 rebuild/modern/save/src/SaveBody.cpp create mode 100644 rebuild/modern/save/src/SaveContainer.cpp create mode 100644 rebuild/modern/save/src/SaveHeader.cpp create mode 100644 rebuild/modern/save/src/TextSections.cpp create mode 100644 rebuild/modern/save/src/Zlib.cpp create mode 100644 rebuild/modern/save/src/Zlib.hpp create mode 100644 rebuild/modern/save/tests/SaveContainerTests.cpp create mode 100644 rebuild/modern/save/tools/fable_save_check.cpp diff --git a/.github/workflows/modern-tree.yml b/.github/workflows/modern-tree.yml new file mode 100644 index 0000000000..62b2462d53 --- /dev/null +++ b/.github/workflows/modern-tree.yml @@ -0,0 +1,41 @@ +name: modern-tree + +# The C++23 modern tree (rebuild/modern) builds and tests on Linux and Windows x64. +# MODERNIZATION.md: "CI builds Linux ... from step 1 on". No retail data is needed; the retail +# save corpus check (fable_save_check) is run locally. +on: + push: + branches: [main] + paths: ["rebuild/modern/**", ".github/workflows/modern-tree.yml"] + pull_request: + paths: ["rebuild/modern/**", ".github/workflows/modern-tree.yml"] + +jobs: + linux-gcc: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Classic zlib (zlib-ng would break byte-identical saves) + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev ninja-build + # This job links the optional LZO (GPL, downloaded and hash-checked at configure time) so + # both the byte-exact and the stored-form text-blob paths are built; Windows leaves it off. + - name: Configure (with pinned LZO 2.10 download) + run: cmake -S rebuild/modern -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DFABLE_SAVE_WITH_LZO=ON + - name: Build + run: cmake --build build + - name: Test + run: ctest --test-dir build --output-on-failure + + windows-msvc-x64: + runs-on: windows-2022 + steps: + - uses: actions/checkout@v4 + - uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + - name: Configure (pinned zlib 1.3.1 download) + run: cmake -S rebuild/modern -B build -G Ninja -DCMAKE_BUILD_TYPE=Release + - name: Build + run: cmake --build build + - name: Test + run: ctest --test-dir build --output-on-failure diff --git a/rebuild/modern/CMakeLists.txt b/rebuild/modern/CMakeLists.txt index 9d4bbc8eb2..d57e4e1a45 100644 --- a/rebuild/modern/CMakeLists.txt +++ b/rebuild/modern/CMakeLists.txt @@ -1,7 +1,9 @@ cmake_minimum_required(VERSION 3.25) -project(FableTLCModern LANGUAGES CXX) +project(FableTLCModern LANGUAGES C CXX) include(CTest) add_subdirectory(multiplayer) +add_subdirectory(bank) +add_subdirectory(save) diff --git a/rebuild/modern/README.md b/rebuild/modern/README.md index 21b547a248..8996fe42c3 100644 --- a/rebuild/modern/README.md +++ b/rebuild/modern/README.md @@ -27,3 +27,11 @@ See `docs/ARCHITECTURE.md` for the promotion and modernization policy. codec for individual cut-co-op `CGameEvent` records, with boundary and stream tests. Package framing and gameplay dispatch remain future evidence-gated work. +- `save/` — retail saves and profiles: the `FableSave!` container (zlib, retail load gates), + every body section typed (HEADER, clock, PLAYER, ENTITIES cells, NPC names, REGIONS, + QUESTS/FACTIONS text and boasts) and `Profile.bin`. Every game-written save (402) and profile + (153) re-encodes byte-identically. LZO for the text blobs is an optional GPL download. See + `save/README.md`. +- `bank/` — the BIGB `.big` bank container (TOCs, payload views, minimal-repack writer). All 17 + bank files in the install round-trip byte-identically, and resize/revert edits hold. See + `bank/README.md`. diff --git a/rebuild/modern/bank/CMakeLists.txt b/rebuild/modern/bank/CMakeLists.txt new file mode 100644 index 0000000000..e1f756d40f --- /dev/null +++ b/rebuild/modern/bank/CMakeLists.txt @@ -0,0 +1,38 @@ +add_library(fable_bank + src/BigBank.cpp +) + +target_include_directories(fable_bank + PUBLIC + "${CMAKE_CURRENT_SOURCE_DIR}/include" +) + +target_compile_features(fable_bank PUBLIC cxx_std_23) +set_target_properties(fable_bank PROPERTIES + CXX_EXTENSIONS OFF +) + +if(MSVC) + set(FABLE_BANK_WARNINGS /W4 /permissive-) +else() + set(FABLE_BANK_WARNINGS -Wall -Wextra -Wconversion -Wpedantic) +endif() +target_compile_options(fable_bank PRIVATE ${FABLE_BANK_WARNINGS}) + +add_executable(fable_bank_check + tools/fable_bank_check.cpp +) +target_link_libraries(fable_bank_check PRIVATE fable_bank) +target_compile_options(fable_bank_check PRIVATE ${FABLE_BANK_WARNINGS}) + +if(BUILD_TESTING) + add_executable(fable_bank_tests + tests/BigBankTests.cpp + ) + target_link_libraries(fable_bank_tests PRIVATE fable_bank) + target_compile_options(fable_bank_tests PRIVATE ${FABLE_BANK_WARNINGS}) + + add_test(NAME fable_bank.BigBank + COMMAND fable_bank_tests + ) +endif() diff --git a/rebuild/modern/bank/README.md b/rebuild/modern/bank/README.md new file mode 100644 index 0000000000..f452b5d012 --- /dev/null +++ b/rebuild/modern/bank/README.md @@ -0,0 +1,51 @@ +# Bank archives (`.big`) + +Portable C++23 codec for the retail BIGB bank container ([BIG.md](../../../docs/formats/BIG.md)). +It holds textures, meshes, animations, text, fonts, lipsync, effects and shaders. This is the +fork-side "bank headers" item of MODERNIZATION step 2 (x64 hazard "Persisted layouts"). Every +field is an explicit little-endian `u32`, so nothing depends on pointer width or on retail struct +images. + +## Model + +- `BigBank`: version, then sub-banks in footer order. +- `SubBank`: name, version, alignment, the optional `(type, count)` stats header, entries. +- `BankEntry`: magic, ID, type, CRC (a per-format class fingerprint), name and dependency strings + as raw on-disk bytes (retail length prefixes include the NUL; tool-added entries may not), + timestamp, info sub-header and payload. +- `Payload`: a view into the file the bank was read from, or owned replacement bytes. A 533 MB + `textures.big` is not copied to rewrite one texture. + +## Writer + +`encode()` lays out every region (TOCs and non-empty payloads) in its original address order, +using the minimal-repack rule of BIG.md §9.1. Each region starts on `snap_up(cursor, B)`, where +`B` is the largest power of two, at most the file's largest sub-bank alignment, that divides the +region's original start. The footer follows the last region, and padding is zero. An unmodified +bank therefore re-encodes byte for byte. A resized payload moves only the regions after it, and +each keeps its alignment. + +## Verified (2026-10-17, MSVC x64 and GCC 15; MSVC x86 on the files up to 250 MB) + +`fable_bank_check --edit ` over the install's `data` folder: 17 bank files (the 8 retail +`.big` files plus 9 tool-written copies and backups, up to 535 MB). + +- **17/17 decode and re-encode byte-identically.** +- In every sub-bank, one payload is grown by the file's largest alignment plus one. The result + decodes, only that payload changed, and every payload that was aligned stays aligned. + Restoring the original bytes gives back the original file in 16/16 aligned files. +- The live `textures.big` was rewritten by the texture-upscale tool with its payloads packed + unaligned (6,321 of 6,325). It still round-trips exactly. But edit-then-revert cannot be + byte-exact there: a re-decode can only re-derive boundaries from the shifted offsets. The check + reports it as `exact+edit(packed)` and requires only that the reverted bank decodes. + +`fable_bank_tests` covers a new bank's round trip, empty payloads, a resize ripple with +alignment, grow-then-revert, and the magic, truncation and footer-range gates. + +## Not covered + +- Payload formats (texture, mesh, animation, text, lipsync decoders): those are per bank kind; + see BIG.md §4. +- Adding new entries or sub-banks. A bank built from scratch lays out in insertion order with + every region on the largest alignment, which the engine accepts (the Python `big_write.rebuild(adds=)` + path loads in-engine), but that path is not exercised against the engine from this codec. diff --git a/rebuild/modern/bank/include/fable/bank/BigBank.hpp b/rebuild/modern/bank/include/fable/bank/BigBank.hpp new file mode 100644 index 0000000000..e54af1adef --- /dev/null +++ b/rebuild/modern/bank/include/fable/bank/BigBank.hpp @@ -0,0 +1,113 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fable::bank { + +enum class BigError { + bad_magic, // not "BIGB" + truncated, // a field or region runs past the end of the file + footer_out_of_range, + bad_toc, // a TOC record does not parse, or its parsed length differs from the footer's size + payload_out_of_range, + overlap, // two regions claim the same bytes (encode: a layout the writer cannot place) + too_large, // an offset or size no longer fits the 32-bit fields +}; + +// An entry's payload: a view into the bank file it was read from, or bytes that replace it. +// Keeping the view avoids copying the large banks (textures.big is 533 MB) to rewrite them. +class Payload final { +public: + Payload() = default; + Payload(std::shared_ptr> source, std::size_t offset, std::size_t size) + : source_(std::move(source)), offset_(offset), size_(size) {} + explicit Payload(std::vector bytes) + : owned_(std::make_shared>(std::move(bytes))), size_(owned_->size()) {} + + [[nodiscard]] std::span bytes() const noexcept + { + if (owned_) { + return *owned_; + } + if (source_) { + return std::span(*source_).subspan(offset_, size_); + } + return {}; + } + [[nodiscard]] std::size_t size() const noexcept { return size_; } + + friend bool operator==(const Payload& a, const Payload& b) + { + const auto x = a.bytes(); + const auto y = b.bytes(); + return x.size() == y.size() && std::equal(x.begin(), x.end(), y.begin()); + } + +private: + std::shared_ptr> source_; + std::shared_ptr> owned_; + std::size_t offset_ = 0; + std::size_t size_ = 0; +}; + +// One TOC record. Strings are kept as their raw on-disk bytes: retail length prefixes include a +// trailing NUL, entries added by later tools may not, and the engine accepts both. +struct BankEntry { + std::uint32_t magic = 42; + std::uint32_t id = 0; + std::int32_t type = 0; + std::uint32_t crc = 0; // a per-format class fingerprint, not a payload hash (BIG.md §8.1) + std::vector name; // raw, as stored after its u32 length + std::uint32_t timestamp = 0; + std::vector> deps; + std::vector info; + Payload payload; + // Where the payload sat in the file it was read from. The writer keeps regions in this + // order and on the alignment boundary they had (BIG.md §9.1). For an empty payload this is + // the stored offset field, written back unchanged. + std::uint64_t original_offset = 0; + + [[nodiscard]] std::string_view name_text() const noexcept; + friend bool operator==(const BankEntry&, const BankEntry&) = default; +}; + +struct SubBank { + std::string name; + std::uint32_t version = 0; + std::uint32_t align = 1; + // (type, count) pairs at the head of the TOC; nullopt when the TOC has none. + std::optional>> stats; + std::vector entries; + std::uint64_t original_toc_offset = 0; + + friend bool operator==(const SubBank&, const SubBank&) = default; +}; + +// A `.big` bank archive (BIG.md §2): 16-byte header, payloads and per-sub-bank TOCs in any +// address order, then the sub-bank directory (footer). +struct BigBank { + std::uint32_t version = 100; + std::vector banks; + std::uint64_t original_footer_offset = 0; + + friend bool operator==(const BigBank&, const BigBank&) = default; + + // `file` is kept alive by the payload views. + [[nodiscard]] static std::expected decode(std::shared_ptr> file); + + // Re-lays every region in its original address order with the minimal-repack rule, so an + // unmodified bank re-encodes byte for byte and a resized payload ripples only what follows it. + [[nodiscard]] std::expected, BigError> encode() const; +}; + +} // namespace fable::bank diff --git a/rebuild/modern/bank/src/BigBank.cpp b/rebuild/modern/bank/src/BigBank.cpp new file mode 100644 index 0000000000..e618084bd0 --- /dev/null +++ b/rebuild/modern/bank/src/BigBank.cpp @@ -0,0 +1,350 @@ +#include + +#include +#include +#include + +namespace fable::bank { +namespace { + +constexpr std::size_t header_size = 16; +constexpr std::uint32_t max_stats_count = 1000; // EgoCore's rewind rule (BIG.md §2) + +class Reader final { +public: + Reader(std::span bytes, std::size_t pos) : bytes_(bytes), pos_(pos) {} + + [[nodiscard]] std::size_t position() const noexcept { return pos_; } + void seek(std::size_t pos) noexcept { pos_ = pos; } + + bool u32(std::uint32_t& value) + { + if (bytes_.size() < 4 || pos_ > bytes_.size() - 4) { + return false; + } + value = 0; + for (int i = 0; i < 4; ++i) { + value |= std::to_integer(bytes_[pos_ + static_cast(i)]) << (8 * i); + } + pos_ += 4; + return true; + } + + bool raw(std::size_t count, std::vector& out) + { + if (count > bytes_.size() - pos_) { + return false; + } + out.assign(bytes_.begin() + static_cast(pos_), + bytes_.begin() + static_cast(pos_ + count)); + pos_ += count; + return true; + } + + // u32 length, then that many bytes. + bool counted(std::vector& out) + { + std::uint32_t length = 0; + return u32(length) && raw(length, out); + } + + bool cstring(std::string& out) + { + for (std::size_t i = pos_; i < bytes_.size(); ++i) { + if (bytes_[i] == std::byte{0}) { + out.assign(reinterpret_cast(bytes_.data()) + pos_, i - pos_); + pos_ = i + 1; + return true; + } + } + return false; + } + +private: + std::span bytes_; + std::size_t pos_ = 0; +}; + +class Writer final { +public: + void u32(std::uint32_t value) + { + for (int i = 0; i < 4; ++i) { + bytes_.push_back(static_cast((value >> (8 * i)) & 0xFFU)); + } + } + void raw(std::span bytes) { bytes_.insert(bytes_.end(), bytes.begin(), bytes.end()); } + void counted(std::span bytes) + { + u32(static_cast(bytes.size())); + raw(bytes); + } + [[nodiscard]] std::vector take() noexcept { return std::move(bytes_); } + +private: + std::vector bytes_; +}; + +struct FooterRecord { + std::string name; + std::uint32_t version = 0; + std::uint32_t entry_count = 0; + std::uint32_t toc_offset = 0; + std::uint32_t toc_size = 0; + std::uint32_t align = 0; +}; + +// One TOC, with each payload's offset taken from `offsets` (index-parallel to the entries). +std::vector encode_toc(const SubBank& bank, const std::vector& offsets) +{ + Writer out; + if (bank.stats) { + out.u32(static_cast(bank.stats->size())); + for (const auto& [type, count] : *bank.stats) { + out.u32(type); + out.u32(count); + } + } + for (std::size_t i = 0; i < bank.entries.size(); ++i) { + const BankEntry& entry = bank.entries[i]; + out.u32(entry.magic); + out.u32(entry.id); + out.u32(static_cast(entry.type)); + out.u32(static_cast(entry.payload.size())); + out.u32(static_cast(offsets[i])); + out.u32(entry.crc); + out.counted(entry.name); + out.u32(entry.timestamp); + out.u32(static_cast(entry.deps.size())); + for (const auto& dep : entry.deps) { + out.counted(dep); + } + out.counted(entry.info); + } + return out.take(); +} + +// The largest power of two <= cap that divides x: the boundary the bank builder placed a region on. +std::uint64_t original_boundary(std::uint64_t x, std::uint64_t cap) +{ + std::uint64_t b = 1; + while (b * 2 <= cap && x % (b * 2) == 0) { + b *= 2; + } + return b; +} + +std::uint64_t snap_up(std::uint64_t x, std::uint64_t boundary) +{ + return (x + boundary - 1) / boundary * boundary; +} + +} // namespace + +std::string_view BankEntry::name_text() const noexcept +{ + std::string_view text(reinterpret_cast(name.data()), name.size()); + while (!text.empty() && text.back() == '\0') { + text.remove_suffix(1); + } + return text; +} + +std::expected BigBank::decode(std::shared_ptr> file) +{ + const std::span bytes(*file); + if (bytes.size() < header_size) { + return std::unexpected(BigError::truncated); + } + if (bytes[0] != std::byte{'B'} || bytes[1] != std::byte{'I'} || bytes[2] != std::byte{'G'} || bytes[3] != std::byte{'B'}) { + return std::unexpected(BigError::bad_magic); + } + Reader head(bytes, 4); + BigBank out; + std::uint32_t footer_offset = 0; + std::uint32_t footer_size = 0; + head.u32(out.version); + head.u32(footer_offset); + head.u32(footer_size); + if (footer_offset < header_size || footer_offset > bytes.size() || footer_size > bytes.size() - footer_offset) { + return std::unexpected(BigError::footer_out_of_range); + } + out.original_footer_offset = footer_offset; + + Reader footer(bytes.first(static_cast(footer_offset) + footer_size), footer_offset); + std::uint32_t bank_count = 0; + if (!footer.u32(bank_count)) { + return std::unexpected(BigError::truncated); + } + std::vector records; + for (std::uint32_t i = 0; i < bank_count; ++i) { + FooterRecord r; + if (!footer.cstring(r.name) || !footer.u32(r.version) || !footer.u32(r.entry_count) || !footer.u32(r.toc_offset) + || !footer.u32(r.toc_size) || !footer.u32(r.align)) { + return std::unexpected(BigError::truncated); + } + records.push_back(std::move(r)); + } + if (footer.position() != static_cast(footer_offset) + footer_size) { + return std::unexpected(BigError::footer_out_of_range); + } + + for (const FooterRecord& r : records) { + if (r.toc_offset > bytes.size() || r.toc_size > bytes.size() - r.toc_offset) { + return std::unexpected(BigError::bad_toc); + } + const auto toc_end = static_cast(r.toc_offset) + r.toc_size; + Reader toc(bytes.first(toc_end), r.toc_offset); + SubBank bank; + bank.name = r.name; + bank.version = r.version; + bank.align = r.align; + bank.original_toc_offset = r.toc_offset; + + std::uint32_t stats_count = 0; + if (!toc.u32(stats_count)) { + return std::unexpected(BigError::bad_toc); + } + if (stats_count < max_stats_count) { + bank.stats.emplace(); + for (std::uint32_t i = 0; i < stats_count; ++i) { + std::uint32_t type = 0; + std::uint32_t count = 0; + if (!toc.u32(type) || !toc.u32(count)) { + return std::unexpected(BigError::bad_toc); + } + bank.stats->emplace_back(type, count); + } + } else { + toc.seek(r.toc_offset); + } + + bank.entries.reserve(r.entry_count); + for (std::uint32_t i = 0; i < r.entry_count; ++i) { + BankEntry entry; + std::uint32_t type = 0; + std::uint32_t size = 0; + std::uint32_t offset = 0; + std::uint32_t dep_count = 0; + if (!toc.u32(entry.magic) || !toc.u32(entry.id) || !toc.u32(type) || !toc.u32(size) || !toc.u32(offset) + || !toc.u32(entry.crc) || !toc.counted(entry.name) || !toc.u32(entry.timestamp) || !toc.u32(dep_count)) { + return std::unexpected(BigError::bad_toc); + } + if (dep_count > r.toc_size / 4) { + return std::unexpected(BigError::bad_toc); + } + entry.deps.resize(dep_count); + for (auto& dep : entry.deps) { + if (!toc.counted(dep)) { + return std::unexpected(BigError::bad_toc); + } + } + if (!toc.counted(entry.info)) { + return std::unexpected(BigError::bad_toc); + } + entry.type = static_cast(type); + entry.original_offset = offset; + if (size != 0) { + if (offset < header_size || offset > bytes.size() || size > bytes.size() - offset) { + return std::unexpected(BigError::payload_out_of_range); + } + entry.payload = Payload(file, offset, size); + } + bank.entries.push_back(std::move(entry)); + } + if (toc.position() != toc_end) { + return std::unexpected(BigError::bad_toc); + } + out.banks.push_back(std::move(bank)); + } + return out; +} + +std::expected, BigError> BigBank::encode() const +{ + // Regions in original address order: each TOC, each non-empty payload, then the footer. + struct Region { + std::uint64_t original_start; + std::size_t bank; + std::size_t entry; // npos for a TOC + std::uint64_t size; + std::uint64_t start = 0; + }; + constexpr std::size_t toc_region = static_cast(-1); + + std::uint64_t cap = 1; + for (const SubBank& bank : banks) { + cap = std::max(cap, bank.align); + } + + // TOC lengths do not depend on the offset values, so they can be sized before layout. + std::vector> offsets(banks.size()); + std::vector regions; + for (std::size_t b = 0; b < banks.size(); ++b) { + offsets[b].resize(banks[b].entries.size()); + const auto toc_size = encode_toc(banks[b], offsets[b]).size(); + regions.push_back(Region{banks[b].original_toc_offset, b, toc_region, toc_size}); + for (std::size_t e = 0; e < banks[b].entries.size(); ++e) { + const BankEntry& entry = banks[b].entries[e]; + offsets[b][e] = entry.original_offset; + if (entry.payload.size() != 0) { + regions.push_back(Region{entry.original_offset, b, e, entry.payload.size()}); + } + } + } + std::stable_sort(regions.begin(), regions.end(), + [](const Region& x, const Region& y) { return x.original_start < y.original_start; }); + + std::uint64_t cursor = header_size; + for (Region& region : regions) { + region.start = snap_up(cursor, original_boundary(region.original_start, cap)); + cursor = region.start + region.size; + if (region.entry != toc_region) { + offsets[region.bank][region.entry] = region.start; + } + } + + // The footer follows the last region (BIG.md §5: it follows the last TOC contiguously). + Writer footer; + footer.u32(static_cast(banks.size())); + std::vector> tocs(banks.size()); + for (std::size_t b = 0; b < banks.size(); ++b) { + tocs[b] = encode_toc(banks[b], offsets[b]); + } + for (std::size_t b = 0; b < banks.size(); ++b) { + const auto toc = std::find_if(regions.begin(), regions.end(), + [&](const Region& r) { return r.bank == b && r.entry == toc_region; }); + const SubBank& bank = banks[b]; + footer.raw(std::span(reinterpret_cast(bank.name.data()), bank.name.size())); + footer.raw(std::array{std::byte{0}}); + footer.u32(bank.version); + footer.u32(static_cast(bank.entries.size())); + footer.u32(static_cast(toc->start)); + footer.u32(static_cast(tocs[b].size())); + footer.u32(bank.align); + } + const auto footer_bytes = footer.take(); + const std::uint64_t footer_offset = cursor; + const std::uint64_t total = footer_offset + footer_bytes.size(); + if (total > std::numeric_limits::max()) { + return std::unexpected(BigError::too_large); + } + + std::vector out(static_cast(total), std::byte{0}); + Writer header; + header.raw(std::array{std::byte{'B'}, std::byte{'I'}, std::byte{'G'}, std::byte{'B'}}); + header.u32(version); + header.u32(static_cast(footer_offset)); + header.u32(static_cast(footer_bytes.size())); + const auto header_bytes = header.take(); + std::copy(header_bytes.begin(), header_bytes.end(), out.begin()); + for (const Region& region : regions) { + const auto bytes = region.entry == toc_region ? std::span(tocs[region.bank]) + : banks[region.bank].entries[region.entry].payload.bytes(); + std::copy(bytes.begin(), bytes.end(), out.begin() + static_cast(region.start)); + } + std::copy(footer_bytes.begin(), footer_bytes.end(), out.begin() + static_cast(footer_offset)); + return out; +} + +} // namespace fable::bank diff --git a/rebuild/modern/bank/tests/BigBankTests.cpp b/rebuild/modern/bank/tests/BigBankTests.cpp new file mode 100644 index 0000000000..ae6eff08b7 --- /dev/null +++ b/rebuild/modern/bank/tests/BigBankTests.cpp @@ -0,0 +1,149 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +using fable::bank::BankEntry; +using fable::bank::BigBank; +using fable::bank::Payload; +using fable::bank::SubBank; + +int failures = 0; + +void expect(const bool condition, const std::string_view message) +{ + if (!condition) { + std::cerr << "FAIL: " << message << '\n'; + ++failures; + } +} + +std::vector bytes(const std::string_view text) +{ + std::vector out; + for (const char c : text) { + out.push_back(static_cast(c)); + } + return out; +} + +std::uint32_t u32_at(const std::vector& b, const std::size_t at) +{ + std::uint32_t v = 0; + for (std::size_t i = 0; i < 4; ++i) { + v |= std::to_integer(b[at + i]) << (8 * i); + } + return v; +} + +BankEntry entry(const std::uint32_t id, const std::string_view name, const std::string_view payload) +{ + BankEntry e; + e.id = id; + e.type = 1; + e.crc = 0x1234; + e.name = bytes(name); + e.name.push_back(std::byte{0}); // retail length prefixes include the NUL + e.deps.push_back(bytes("\\Dev\\source.tga")); + e.info = bytes("INFO"); + if (!payload.empty()) { + e.payload = Payload(bytes(payload)); + } + return e; +} + +BigBank sample() +{ + BigBank bank; + SubBank text; + text.name = "TEXT_ENGLISH_MAIN"; + text.version = 106; + text.align = 1; + text.stats = std::vector>{{1, 2}}; + text.entries.push_back(entry(1, "HELLO", "hello world")); + text.entries.push_back(entry(2, "EMPTY", "")); + SubBank tex; + tex.name = "GBANK_MAIN_PC"; + tex.version = 1; + tex.align = 16; + tex.stats = std::vector>{{0, 1}}; + tex.entries.push_back(entry(7, "TEX", "pixels")); + bank.banks = {text, tex}; + return bank; +} + +void round_trip_is_exact() +{ + const auto first = sample().encode(); + expect(first.has_value(), "a new bank encodes"); + if (!first) { + return; + } + expect(std::to_integer((*first)[0]) == 'B' && u32_at(*first, 4) == 100, "BIGB header, version 100"); + const auto file = std::make_shared>(*first); + const auto decoded = BigBank::decode(file); + expect(decoded.has_value(), "the encoded bank decodes"); + if (!decoded) { + return; + } + expect(decoded->banks.size() == 2 && decoded->banks[0].entries[0].name_text() == "HELLO", "names survive"); + expect(decoded->banks[0].entries[1].payload.size() == 0, "empty payloads stay empty"); + const auto again = decoded->encode(); + expect(again.has_value() && *again == *first, "decode then encode is byte-identical"); + for (const auto& sub : decoded->banks) { + for (const auto& e : sub.entries) { + expect(e.payload.size() == 0 || e.original_offset % 16 == 0, "regions sit on the file's largest alignment"); + } + } +} + +void resize_ripples_and_reverts() +{ + const auto file = std::make_shared>(*sample().encode()); + auto bank = *BigBank::decode(file); + bank.banks[0].entries[0].payload = Payload(bytes("hello world, now longer than before")); + const auto grown = bank.encode(); + const auto reread = BigBank::decode(std::make_shared>(*grown)); + expect(reread.has_value() && reread->banks[0].entries[0].payload == bank.banks[0].entries[0].payload, + "the resized payload reads back"); + expect(reread.has_value() && reread->banks[1].entries[0].payload == Payload(bytes("pixels")), + "the other payloads are unchanged"); + expect(reread.has_value() && reread->banks[1].entries[0].original_offset % 16 == 0, "rippled payloads stay aligned"); + auto reverted = *reread; + reverted.banks[0].entries[0].payload = Payload(bytes("hello world")); + const auto restored = reverted.encode(); + expect(restored.has_value() && *restored == *file, "grow then revert restores the original bytes"); +} + +void gates_are_enforced() +{ + auto bad = std::make_shared>(*sample().encode()); + (*bad)[0] = std::byte{'X'}; + expect(BigBank::decode(bad).error() == fable::bank::BigError::bad_magic, "bad magic is rejected"); + auto short_file = std::make_shared>(bytes("BIGB")); + expect(!BigBank::decode(short_file).has_value(), "a truncated header is rejected"); + auto footer = std::make_shared>(*sample().encode()); + (*footer)[8] = std::byte{0xFF}; + (*footer)[9] = std::byte{0xFF}; + (*footer)[10] = std::byte{0xFF}; + expect(BigBank::decode(footer).error() == fable::bank::BigError::footer_out_of_range, "a footer past the end is rejected"); +} + +} // namespace + +int main() +{ + round_trip_is_exact(); + resize_ripples_and_reverts(); + gates_are_enforced(); + if (failures == 0) { + std::cout << "fable_bank: all tests passed\n"; + } + return failures == 0 ? 0 : 1; +} diff --git a/rebuild/modern/bank/tools/fable_bank_check.cpp b/rebuild/modern/bank/tools/fable_bank_check.cpp new file mode 100644 index 0000000000..78e12dc13b --- /dev/null +++ b/rebuild/modern/bank/tools/fable_bank_check.cpp @@ -0,0 +1,157 @@ +// fable_bank_check [--edit] ... +// +// For each `.big` (BIGB) bank: decode, re-encode, and report whether the rewrite is byte-identical. +// With --edit, also grow one payload in every sub-bank (by one byte more than the file's largest +// alignment), check that the result decodes with only that payload changed and every payload on +// its sub-bank's alignment, then restore the original bytes and check that the file comes back +// exactly (the writer's invertibility, BIG.md §9.1). Exit status 1 on any failure. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using fable::bank::BigBank; +using fable::bank::Payload; + +std::shared_ptr> read_file(const std::filesystem::path& path) +{ + std::ifstream in(path, std::ios::binary); + auto bytes = std::make_shared>(static_cast(std::filesystem::file_size(path))); + in.read(reinterpret_cast(bytes->data()), static_cast(bytes->size())); + return bytes; +} + +bool has_magic(const std::filesystem::path& path) +{ + std::ifstream in(path, std::ios::binary); + char magic[4] = {}; + in.read(magic, 4); + return in && std::equal(magic, magic + 4, "BIGB"); +} + +// "" if the edit round trip holds, else what broke. +std::string edit_round_trip(const BigBank& bank, const std::vector& original, const bool packed) +{ + std::uint32_t cap = 1; + for (const auto& sub : bank.banks) { + cap = std::max(cap, sub.align); + } + for (std::size_t b = 0; b < bank.banks.size(); ++b) { + const auto& entries = bank.banks[b].entries; + const auto target = std::find_if(entries.begin(), entries.end(), [](const auto& e) { return e.payload.size() != 0; }); + if (target == entries.end()) { + continue; + } + const auto e = static_cast(target - entries.begin()); + BigBank edited = bank; + std::vector grown(target->payload.bytes().begin(), target->payload.bytes().end()); + grown.resize(grown.size() + cap + 1, std::byte{0x5A}); + edited.banks[b].entries[e].payload = Payload(grown); + + const auto bytes = edited.encode(); + if (!bytes) { + return "edit encode failed in " + bank.banks[b].name; + } + const auto again = BigBank::decode(std::make_shared>(*bytes)); + if (!again) { + return "edited bank does not decode (" + bank.banks[b].name + ")"; + } + for (std::size_t bb = 0; bb < bank.banks.size(); ++bb) { + const auto& sub = again->banks[bb]; + for (std::size_t ee = 0; ee < sub.entries.size(); ++ee) { + const auto& got = sub.entries[ee]; + const auto& want = edited.banks[bb].entries[ee]; + if (got.payload != want.payload || got.name != want.name || got.info != want.info || got.crc != want.crc) { + return "edit changed another entry in " + sub.name; + } + // Only payloads the original file had aligned must stay aligned: tool-written banks + // (e.g. a repacked textures.big) can store payloads off their sub-bank boundary. + const bool was_aligned = bank.banks[bb].entries[ee].original_offset % sub.align == 0; + if (got.payload.size() != 0 && was_aligned && got.original_offset % sub.align != 0) { + return "edit broke alignment in " + sub.name; + } + } + } + BigBank reverted = *again; + reverted.banks[b].entries[e].payload = Payload(std::vector(target->payload.bytes().begin(), target->payload.bytes().end())); + const auto restored = reverted.encode(); + // Invertibility needs the original boundaries, which a re-decode recovers only when the + // payloads were aligned: in a packed bank they are re-derived from the shifted offsets. + if (packed) { + if (!restored || !BigBank::decode(std::make_shared>(*restored))) { + return "reverted packed bank does not decode (" + bank.banks[b].name + ")"; + } + } else if (!restored || *restored != original) { + return "grow-then-revert is not the original (" + bank.banks[b].name + ")"; + } + } + return ""; +} + +} // namespace + +int main(int argc, char** argv) +{ + bool edit = false; + std::vector files; + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "--edit") { + edit = true; + } else if (std::filesystem::is_directory(arg)) { + for (const auto& entry : std::filesystem::recursive_directory_iterator(arg)) { + if (entry.is_regular_file() && has_magic(entry.path())) { + files.push_back(entry.path()); + } + } + } else { + files.push_back(arg); + } + } + + int failures = 0; + for (const auto& path : files) { + const auto file = read_file(path); + const auto bank = BigBank::decode(file); + std::string verdict; + std::size_t entries = 0; + std::size_t unaligned = 0; + if (!bank) { + verdict = "rejected_" + std::to_string(static_cast(bank.error())); + } else { + for (const auto& sub : bank->banks) { + entries += sub.entries.size(); + for (const auto& e : sub.entries) { + unaligned += e.payload.size() != 0 && e.original_offset % sub.align != 0; + } + } + const auto encoded = bank->encode(); + if (!encoded) { + verdict = "encode_failed_" + std::to_string(static_cast(encoded.error())); + } else if (*encoded != *file) { + verdict = "encode_mismatch"; + } else { + verdict = "exact"; + if (edit) { + const auto problem = edit_round_trip(*bank, *file, unaligned != 0); + verdict = !problem.empty() ? "edit:" + problem : unaligned != 0 ? "exact+edit(packed)" : "exact+edit"; + } + } + } + if (verdict.rfind("exact", 0) != 0) { + ++failures; + } + std::cout << verdict << '\t' << (bank ? bank->banks.size() : 0) << " banks\t" << entries << " entries\t" + << unaligned << " unaligned\t" << file->size() << " bytes\t" << path.string() << '\n'; + } + return failures == 0 ? 0 : 1; +} diff --git a/rebuild/modern/save/CMakeLists.txt b/rebuild/modern/save/CMakeLists.txt new file mode 100644 index 0000000000..3e9dfc1a73 --- /dev/null +++ b/rebuild/modern/save/CMakeLists.txt @@ -0,0 +1,127 @@ +# zlib: an installed package, a local source tree (-DFABLE_ZLIB_SOURCE_DIR=...), or a pinned +# download. Use classic zlib (1.2.x/1.3.x). zlib-ng deflates differently and would break the +# byte-identical save rewrite. +set(FABLE_ZLIB_SOURCE_DIR "" CACHE PATH "Classic zlib source tree to build instead of find_package(ZLIB)") + +if(FABLE_ZLIB_SOURCE_DIR) + add_library(fable_zlib STATIC + "${FABLE_ZLIB_SOURCE_DIR}/adler32.c" + "${FABLE_ZLIB_SOURCE_DIR}/crc32.c" + "${FABLE_ZLIB_SOURCE_DIR}/deflate.c" + "${FABLE_ZLIB_SOURCE_DIR}/inffast.c" + "${FABLE_ZLIB_SOURCE_DIR}/inflate.c" + "${FABLE_ZLIB_SOURCE_DIR}/inftrees.c" + "${FABLE_ZLIB_SOURCE_DIR}/trees.c" + "${FABLE_ZLIB_SOURCE_DIR}/zutil.c" + ) + target_include_directories(fable_zlib PUBLIC "${FABLE_ZLIB_SOURCE_DIR}") + set(FABLE_ZLIB_TARGET fable_zlib) +else() + find_package(ZLIB QUIET) + if(ZLIB_FOUND) + set(FABLE_ZLIB_TARGET ZLIB::ZLIB) + else() + include(FetchContent) + FetchContent_Declare(zlib + URL https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz + URL_HASH SHA256=9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 + ) + set(ZLIB_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(zlib) + set(FABLE_ZLIB_TARGET zlibstatic) + target_include_directories(zlibstatic PUBLIC "${zlib_SOURCE_DIR}" "${zlib_BINARY_DIR}") + endif() +endif() + +# LZO: optional, OFF by default. The QUESTS and FACTIONS text blobs are LZO1X streams from +# lzo1x_999_compress (retail 0xC08130). Upstream LZO 2.10 reproduces every game-written blob +# byte for byte, but LZO is GPL-2.0-or-later, so it is never vendored in this MIT tree: with +# the option ON it is fetched (or taken from FABLE_LZO_SOURCE_DIR) at configure time, and a +# binary built that way is a combined work under the GPL. With it OFF, text blobs still decode +# (our own decoder) and re-encode in retail's stored form, which the retail reader accepts. +option(FABLE_SAVE_WITH_LZO "Link upstream LZO 2.10 (GPL-2.0-or-later) for byte-exact QUESTS/FACTIONS re-encoding" OFF) +set(FABLE_LZO_SOURCE_DIR "" CACHE PATH "LZO 2.10 source tree to build instead of downloading it") + +if(FABLE_SAVE_WITH_LZO) + if(NOT FABLE_LZO_SOURCE_DIR) + include(FetchContent) + FetchContent_Declare(lzo + URL https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz + URL_HASH SHA256=c0f892943208266f9b6543b3ae308fab6284c5c90e627931446fb49b4221a072 + SOURCE_SUBDIR no-cmake-build + ) + FetchContent_MakeAvailable(lzo) + set(FABLE_LZO_SOURCE_DIR "${lzo_SOURCE_DIR}") + endif() + add_library(fable_lzo STATIC + "${FABLE_LZO_SOURCE_DIR}/src/lzo1x_9x.c" + "${FABLE_LZO_SOURCE_DIR}/src/lzo_init.c" + ) + target_include_directories(fable_lzo PUBLIC "${FABLE_LZO_SOURCE_DIR}/include") +endif() + +add_library(fable_save + src/Entities.cpp + src/Lzo.cpp + src/PersistStream.cpp + src/Profile.cpp + src/Regions.cpp + src/SaveBody.cpp + src/SaveContainer.cpp + src/SaveHeader.cpp + src/TextSections.cpp + src/Zlib.cpp +) + +target_include_directories(fable_save + PUBLIC + "${CMAKE_CURRENT_SOURCE_DIR}/include" +) +target_link_libraries(fable_save PRIVATE ${FABLE_ZLIB_TARGET}) +if(FABLE_SAVE_WITH_LZO) + target_link_libraries(fable_save PRIVATE fable_lzo) + target_compile_definitions(fable_save PRIVATE FABLE_SAVE_HAVE_LZO=1) +endif() + +target_compile_features(fable_save PUBLIC cxx_std_23) +set_target_properties(fable_save PROPERTIES + CXX_EXTENSIONS OFF +) + +if(MSVC) + target_compile_options(fable_save PRIVATE /W4 /permissive-) +else() + target_compile_options(fable_save PRIVATE + -Wall + -Wextra + -Wconversion + -Wpedantic + ) +endif() + +add_executable(fable_save_check + tools/fable_save_check.cpp +) +target_link_libraries(fable_save_check PRIVATE fable_save) + +if(BUILD_TESTING) + add_executable(fable_save_tests + tests/SaveContainerTests.cpp + ) + target_link_libraries(fable_save_tests PRIVATE fable_save) + + if(MSVC) + target_compile_options(fable_save_tests PRIVATE /W4 /permissive-) + else() + target_compile_options(fable_save_tests PRIVATE + -Wall + -Wextra + -Wconversion + -Wpedantic + ) + endif() + + add_test(NAME fable_save.SaveContainer + COMMAND fable_save_tests + ) +endif() diff --git a/rebuild/modern/save/README.md b/rebuild/modern/save/README.md new file mode 100644 index 0000000000..9d603fedbc --- /dev/null +++ b/rebuild/modern/save/README.md @@ -0,0 +1,121 @@ +# Save files (`.sav`, `Profile.bin`) + +Portable C++23 codec for the retail `FableSave!` container: the fork-side piece of +MODERNIZATION step 2 ("retail saves load; a save round-trips"). It decompresses and recompresses +the two persist-stream sections, enforces the retail load gates, and decodes HEADER, the body's +section list, PLAYER, ENTITIES down to the inflated SAVED_ENTITIES cells and SAVED_NPC_NAMES, +REGIONS, and the QUESTS and FACTIONS persist text (LZO). The entity streams inside the cells +stay opaque bytes. + +## Evidence boundary + +- Layout and gates: [SAVE.md](../../../docs/formats/SAVE.md) "Container and header layout" and + "Signature enforcement". The decoder accepts exactly what retail + `CUserProfileManager::VerifySignature` (0x00409730) accepts: the magic, `total_data_len + 4 <= + size`, the seed-0 `CCRC::Calc` trailer, size exactly 0x4B000, all-zero padding. It then applies the + chunk framing `CWorld::LoadGameStateInternal` (0x004A21F0) reads. +- Compression: retail links zlib **1.2.1** (`deflate 1.2.1 Copyright` in `Fable.exe`), driven by + `CDataOutputStream::WriteZlibCompressed` (0x009967E0). Level 9, 15-bit window, memLevel 8 and the + default strategy reproduce retail's compressed bytes. That holds with upstream 1.2.1 and with + classic zlib 1.3.1. **zlib-ng does not reproduce them** (Python's `zlib` here is + `1.3.1.zlib-ng`), which is why SAVE.md §2.4 originally concluded byte identity was unreachable. +- Text blobs: QUESTS and FACTIONS hold persist text compressed by `lzo1x_999_compress` + (0x00C08130, called from the blob writer 0x00996610 on all but the last 3 bytes, which are + stored raw). Upstream LZO 2.10 reproduces 8,698 of the corpus's 8,700 blobs (288 of 290 QUESTS, + all 8,410 FACTIONS records) byte for byte. The other two are QUESTS streams our Python editor + wrote in store mode. + +## Licensing (LZO) + +This tree is MIT. LZO is GPL-2.0-or-later, so it is never committed here. The decoder in +`src/Lzo.cpp` is this module's own implementation of the LZO1X stream format, and without LZO the +writer emits retail's own stored form (`i16 0` then the raw text). That is the shape retail +writes when compression does not help, and its reader accepts it. Configure with +`-DFABLE_SAVE_WITH_LZO=ON` to download upstream LZO 2.10 (SHA-256 pinned) or build it from +`FABLE_LZO_SOURCE_DIR`, and get byte-exact re-encoding. A binary built that way is a combined +work under the GPL, so whoever distributes it must also offer its source. Either way, everyone +gets the full source and can modify it. + +## Verified + +`fable_save_check` over every `FableSave!` file in the local save folder (409 files, 2026-10-17), +built x86 and x64: + +- 402 **exact**: the container, the typed HEADER, the body's section list, the typed PLAYER + section, ENTITIES (all 1,921 cells inflated and re-deflated at level 1, plus the NPC-name map) + and the QUESTS/FACTIONS text all decode, and every layer re-encodes byte for byte (the text + blobs only with LZO linked; without it they must decode back to the same text) (`encode(decode(file)) == file`, + all 307,200 bytes). This is every save the game itself wrote. +- 2 **sections**: valid at every layer, but recompressed by our Python tools (zlib-ng, and a + store-mode LZO stream in QUESTS), so the compressed bytes differ. +- 5 **malformed**, all backups made by our Python save editors. Three have body sections that + don't tile the stream (`body:decode_error`). Two have ENTITIES one byte short, so the next + section reads as `LAYER` (`body:unexpected_sections`). + +All 153 `Profile.bin` files (`FablePro`) also decode and re-encode byte for byte +(`profile:exact`). The same results come from MSVC x86, MSVC x64 and GCC 15 builds, with and +without LZO. + +Body shapes in the corpus: 291 full (game clock + ENTITIES, PLAYER, QUESTS, REGIONS, FACTIONS) +and 113 minimal (`.qs.hs`, ENTITIES only), matching the writer's two branches. + +`fable_save_tests` covers the CRC check value, layout, determinism, each retail gate, and the +HEADER round trip, including the minimal-context form and the tag check. It also covers a +hand-assembled LZO1X stream and the text-blob framing (empty, short, stored, and over 0x7FFF). + +## Layers + +- `SaveContainer.hpp`: `FableSave!` container, zlib sections, retail gates, `ccrc`. +- `PersistStream.hpp`: binary `CPersistContext` primitives. Each field is `[u32 field_tag(name)]` + then its value. The tag is checked the way retail `CheckCRC` asserts. Widths are pinned: + `long` → `int32_t`, `CWideString` → UTF-16 `char16_t`, UIDs → `uint64_t`, `CFloatRange`/vectors + → 3 × `float`. +- `SaveHeader.hpp`: the HEADER section (SAVE.md §3.4). +- `SaveBody.hpp`: the body. It holds the game clock (`Time`, `TimeStep`, `DayCount`) and the + named sections (`name\0`, i32 length, bytes; `CPersistContext::StartSection`/`EndSection` + 0x009BAB60/0x009BABF0), plus the typed `PlayerSection` (SAVE.md §3.5). +- `Entities.hpp`: ENTITIES (head, SAVED_ENTITIES, SAVED_NPC_NAMES) and the SAVED_ENTITIES slot + table: empty slots, cell records (zlib level 1) and the dead-entity table (SAVE.md §3.6, the + 2026-10-17 grammar). The head is the format version (2) and `CThingManager::CurrentUniqueID`. + The tail is `PersistedDeadEntities`: the UID of each killed game-persistent entity, plus the + seed-0 CRC of the quest script it belonged to. `SavedNpcNames` is `CThingManager::NPCNameMap` (UID → group, name; writer 0x00524440). + An empty group is written as `UNATTACHED`. The section is empty in the minimal `.qs.hs` form. +- `Regions.hpp`: REGIONS, the region manager's `vector` (SaveGameState 0x004FDE80). + For elements 1 onwards it stores `Name` and the `AppearOnWorldMap`, `CreatureGenerationEnabled` + and `SoundThemesEnabled` flags. Every field is `Transfer("")`, so every tag is 0. All 290 + REGIONS sections re-encode exactly. +- `TextSections.hpp`: the LZ text blob (`read_lz_text`/`write_lz_text`; its `clen` is a + signed i16, so above 0x7FFF it is -1 followed by a u32), `QuestsSection` (text plus the accepted + boasts: `NumAcceptedBoasts`, then `BoastScriptName`, `BoastId`, `BoastFailed` and + `BoastCompleted`) and `FactionsSection` (`NumberOfFactions`, then a `Faction` text each). + Every field tag in the save is now named. +- `Profile.hpp`: `Profile.bin` (`FablePro`, text-mode persist script). Entries are kept in + order with their raw values; `save_game_names()` gives the 50-slot manual-save registry. + `set_value` plus `encode` re-signs the file. Retail `VerifySignature` also has a + 0x4000-size / 0x20-pad branch for this magic, but no real profile satisfies it (all are + variable-length and unpadded). The decoder applies the CRC rule the files actually follow. + +## Build + +``` +cmake -S rebuild/modern -B build -G Ninja # find_package(ZLIB), else pinned zlib 1.3.1 +cmake -S rebuild/modern -B build -DFABLE_ZLIB_SOURCE_DIR= # offline +cmake -S rebuild/modern -B build -DFABLE_SAVE_WITH_LZO=ON # byte-exact text blobs (GPL build) +cmake --build build && ctest --test-dir build +build/save/fable_save_check "/My Games/Fable/Saves" +``` + +## Intentional modernization + +- Sections are `std::vector` and every on-disk field is an explicit little-endian + `u32`, so nothing depends on pointer width or on the retail struct images. +- Oversized output is an error (`too_large`), not a silently truncated file. + +## Not covered + +- Typed profile settings (values stay raw text; `CActionInputControl(...)` bindings unparsed). +- The entity and component records inside each cell (SAVE.md "Entity graph"). Those are the + reconstructed engine's own `Transfer` code, not a separate model. +- A typed model of the quest and faction persist text (it stays one string per blob). +- A size-0 text blob is written as a bare `u32 0`, as the writer does. No save in the corpus has + one, so how the reader handles it is unchecked. diff --git a/rebuild/modern/save/include/fable/save/Entities.hpp b/rebuild/modern/save/include/fable/save/Entities.hpp new file mode 100644 index 0000000000..681c2193c9 --- /dev/null +++ b/rebuild/modern/save/include/fable/save/Entities.hpp @@ -0,0 +1,97 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fable::save { + +// One SAVED_ENTITIES record: a map cell's persisted entities, stored zlib-compressed at level 1. +// The data is the inflated entity stream (SAVE.md "Entity graph"); it stays opaque bytes here. +struct EntityCell { + std::uint64_t cell_id = 0; + std::vector data; // empty = a record with no stream (clen 0) + std::array trailer{}; // all zero in every observed save + + friend bool operator==(const EntityCell&, const EntityCell&) = default; +}; + +// The SAVED_ENTITIES section: +// u32 0, u32 slot_count, u32 0 +// slot_count slots, each either u64 0 (empty) or +// u64 29+clen, u64 cell_id, u64 clen, u32 ulen, clen bytes zlib (level 1), 9 trailer bytes +// [tag][u32 dead_count], dead_count × { [tag][u64 uid] [tag][u32 section_name_crc] } +// The tail is CEntityRuntimePersistence::PersistedDeadEntities, filtered to the game-persistent +// entries (writer 0x521620). Every tag is field_tag("") == 0. +struct PersistedDeadEntity { + std::uint64_t uid = 0; + // Seed-0 CRC of the owning section's name: the quest script that created the entity + // (e.g. "Q_GuildTraining", "V_SickChild_Activate"), or "NULL" (0x321ACEE7) for none. + std::uint32_t section_name_crc = 0; + + friend bool operator==(const PersistedDeadEntity&, const PersistedDeadEntity&) = default; +}; + +struct SavedEntities { + std::vector> slots; + std::vector dead_entities; + + friend bool operator==(const SavedEntities&, const SavedEntities&) = default; + + [[nodiscard]] static std::expected decode(std::span section); + [[nodiscard]] std::expected, PersistError> encode() const; +}; + +// SAVED_NPC_NAMES: CThingManager::NPCNameMap, map>, written +// by 0x524440 in key order. Binary saves write u32 count, then per entry u64 uid, the group as a +// counted string and the name as NUL-terminated UTF-16. An empty group is written as "UNATTACHED" +// and read back as empty (loader 0x525A62), so `group` holds the in-memory value. A context +// with the +0x102 flag set (the minimal no-clock `.qs.hs` form) writes an empty section +// (`entries` = nullopt). +struct NpcName { + std::uint64_t uid = 0; + std::string group; + std::u16string name; + + friend bool operator==(const NpcName&, const NpcName&) = default; +}; + +struct SavedNpcNames { + std::optional> entries; + + friend bool operator==(const SavedNpcNames&, const SavedNpcNames&) = default; + + [[nodiscard]] static std::expected decode(std::span section); + [[nodiscard]] std::vector encode() const; +}; + +// The ENTITIES head of a full save (writer 0x524390): [tag][u32 format version] [tag][u64 +// CThingManager::CurrentUniqueID], the next free global UID. Every tag is field_tag("") == 0. +// The minimal `.qs.hs` form has no head. +struct EntitiesHead { + std::uint32_t format_version = 2; // 2 in every observed save + std::uint64_t current_unique_id = 0; + + friend bool operator==(const EntitiesHead&, const EntitiesHead&) = default; +}; + +// The ENTITIES section: the optional head, then the SAVED_ENTITIES and SAVED_NPC_NAMES sections. +struct EntitiesSection { + std::optional head; + SavedEntities saved_entities; + SavedNpcNames saved_npc_names; + + friend bool operator==(const EntitiesSection&, const EntitiesSection&) = default; + + [[nodiscard]] static std::expected decode(std::span section); + [[nodiscard]] std::expected, PersistError> encode() const; +}; + +} // namespace fable::save diff --git a/rebuild/modern/save/include/fable/save/PersistStream.hpp b/rebuild/modern/save/include/fable/save/PersistStream.hpp new file mode 100644 index 0000000000..d11b93a5ac --- /dev/null +++ b/rebuild/modern/save/include/fable/save/PersistStream.hpp @@ -0,0 +1,83 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fable::save { + +enum class PersistError { + truncated, // the stream ends inside a tag or value + tag_mismatch, // retail CheckCRC asserts stream_crc == CCharString(name).GetCRC() + invalid_bool, // a bool byte other than 0 or 1 + unterminated, // a string runs to the end of the stream without its NUL + bad_section, // section name or length framing does not match +}; + +// The on-disk tag of a field: seed-0 CCRC::Calc over the field name's bytes, no terminator. +[[nodiscard]] std::uint32_t field_tag(std::string_view name) noexcept; + +// Reader for the binary CPersistContext stream (mode 2). Every Transfer(name, value) is +// `[u32 field_tag(name)][value]`, and value widths are fixed by T, not by the host: long and +// unsigned long are 4 bytes, CWideString is UTF-16LE, UIDs are 8 bytes (docs/modern/PERSISTED_TYPES.md). +class PersistReader final { +public: + explicit PersistReader(std::span bytes) noexcept : bytes_(bytes) {} + + [[nodiscard]] std::size_t position() const noexcept { return pos_; } + [[nodiscard]] std::size_t remaining() const noexcept { return bytes_.size() - pos_; } + + [[nodiscard]] std::expected tag(std::string_view name); + + [[nodiscard]] std::expected u32(); + [[nodiscard]] std::expected i32(); + [[nodiscard]] std::expected u64(); + [[nodiscard]] std::expected f32(); + [[nodiscard]] std::expected boolean(); + [[nodiscard]] std::expected cstring(); // CCharString: bytes + NUL + [[nodiscard]] std::expected wstring(); // CWideString: UTF-16LE + NUL + [[nodiscard]] std::expected counted_string(); // u32 length, then bytes (0x995A80) + [[nodiscard]] std::expected, PersistError> vector3(); + [[nodiscard]] std::expected, PersistError> raw(std::size_t count); + + // `name\0` then u32 length; returns the section's bytes and advances past them. + [[nodiscard]] std::expected, PersistError> section(std::string_view name); + +private: + [[nodiscard]] std::expected, PersistError> take(std::size_t count); + + std::span bytes_; + std::size_t pos_ = 0; +}; + +// Writer for the binary CPersistContext stream (mode 3); the exact inverse of PersistReader. +class PersistWriter final { +public: + void tag(std::string_view name); + void u32(std::uint32_t value); + void i32(std::int32_t value); + void u64(std::uint64_t value); + void f32(float value); + void boolean(bool value); + void cstring(std::string_view value); + void wstring(std::u16string_view value); + void counted_string(std::string_view value); + void vector3(const std::array& value); + + // Wraps `body` as a named section: `name\0`, u32 body length, body. + void section(std::string_view name, std::span body); + void bytes_raw(std::span raw); + + [[nodiscard]] const std::vector& bytes() const noexcept { return bytes_; } + [[nodiscard]] std::vector take() noexcept { return std::move(bytes_); } + +private: + std::vector bytes_; +}; + +} // namespace fable::save diff --git a/rebuild/modern/save/include/fable/save/Profile.hpp b/rebuild/modern/save/include/fable/save/Profile.hpp new file mode 100644 index 0000000000..71e8c0c7a2 --- /dev/null +++ b/rebuild/modern/save/include/fable/save/Profile.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace fable::save { + +enum class ProfileError { + bad_magic, // not "FablePro" + u32 0 + truncated, + data_length_out_of_range, // total_data_len must equal file size - 4 + bad_signature, // trailer != ccrc(file[0 : total_data_len]) + bad_line, // a line that is not `Name value;` + CRLF +}; + +// `Profile.bin`: the `FablePro` container around a text-mode CPersistContext script +// (docs/formats/SAVE.md "Profile index"): +// 0x00 "FablePro", u32 0, u32 total_data_len (= file size - 4), latin1 text, u32 ccrc trailer. +// The text is one `Name value;\r\n` line per field, in writer order, so entries keep their order +// and raw value spelling and re-encode byte for byte. +// +// Retail VerifySignature (0x00409730) also has a 0x4000-size / 0x20-padding branch for this magic, +// but every real profile is variable-length and unpadded (153/153 here). The decoder applies the +// CRC rule the files actually satisfy. +class Profile final { +public: + struct Entry { + std::string name; // e.g. "SaveGameNames1", "AssignedInputsPassive[3]" + std::string value; // raw text between the space and the ';', e.g. "\"Manual - Save1\"" + + friend bool operator==(const Entry&, const Entry&) = default; + }; + + static constexpr std::size_t save_slot_count = 50; + + [[nodiscard]] static std::expected decode(std::span file); + [[nodiscard]] std::vector encode() const; + + [[nodiscard]] const std::vector& entries() const noexcept { return entries_; } + [[nodiscard]] std::optional value(std::string_view name) const noexcept; + // Replaces an existing entry's raw value; returns false if there is no such entry. + bool set_value(std::string_view name, std::string value); + + // The manual-save registry `SaveGameNames1..50`, unquoted. An empty string is a free slot. + [[nodiscard]] std::vector save_game_names() const; + + friend bool operator==(const Profile&, const Profile&) = default; + +private: + std::vector entries_; +}; + +} // namespace fable::save diff --git a/rebuild/modern/save/include/fable/save/Regions.hpp b/rebuild/modern/save/include/fable/save/Regions.hpp new file mode 100644 index 0000000000..079bf0a0a1 --- /dev/null +++ b/rebuild/modern/save/include/fable/save/Regions.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace fable::save { + +// One CRegion's saved state (the region manager's vector, 0x58-byte elements in retail). +struct Region { + std::string name; // CRegion::Name + bool appear_on_world_map = false; // +0x54 + bool creature_generation_enabled = false; // +0x55 + bool sound_themes_enabled = false; // +0x56 + + friend bool operator==(const Region&, const Region&) = default; +}; + +// REGIONS (SaveGameState 0x4FDE80, per region 0x6BC290). Every field is Transfer("", ...), so +// every tag is field_tag("") == 0: +// [tag][u32 table size] +// (table size - 1) × { [tag][name\0] [tag][bool] [tag][bool] [tag][bool] } +// Element 0 of the engine's table is never written; `regions` holds elements 1 onwards, so the +// stored table size is regions.size() + 1. +struct RegionsSection { + std::vector regions; + + friend bool operator==(const RegionsSection&, const RegionsSection&) = default; + + [[nodiscard]] static std::expected decode(std::span section); + [[nodiscard]] std::vector encode() const; +}; + +} // namespace fable::save diff --git a/rebuild/modern/save/include/fable/save/SaveBody.hpp b/rebuild/modern/save/include/fable/save/SaveBody.hpp new file mode 100644 index 0000000000..c95c281fcc --- /dev/null +++ b/rebuild/modern/save/include/fable/save/SaveBody.hpp @@ -0,0 +1,70 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace fable::save { + +// The body section, chunk1 of a `.sav`, as CWorld::SaveGameStateInternal (0x0049F4C0) writes it: +// +// [game clock: Time f32, TimeStep f32, DayCount i32] (written by 0x006BBB30) +// ENTITIES section (CThingManager, via 0x00524290) +// PLAYER section (CPlayerManager::SaveGameState 0x00449F90) +// QUESTS section (0x004B64E0) +// REGIONS section (0x004FDE80) +// FACTIONS section (0x006B9560) +// +// In the writer's minimal-context branch (`this+0x102 != 0`, the `.qs.hs` files) only ENTITIES +// is written. Each section is `name\0`, an i32 length patched by EndSection (0x009BABF0), then its +// bytes. Section contents other than PLAYER stay opaque bytes at this layer. +struct SaveBody { + struct GameClock { + float time = 0.0F; + float time_step = 0.0F; + std::int32_t day_count = 0; + + friend bool operator==(const GameClock&, const GameClock&) = default; + }; + + struct Section { + std::string name; + std::vector bytes; + + friend bool operator==(const Section&, const Section&) = default; + }; + + std::optional clock; + std::vector
sections; + + friend bool operator==(const SaveBody&, const SaveBody&) = default; + + // Requires the sections to tile the stream exactly. + [[nodiscard]] static std::expected decode(std::span body); + [[nodiscard]] std::vector encode() const; + + [[nodiscard]] const Section* find(std::string_view name) const noexcept; + + // True for the two shapes the retail writer produces: clock + ENTITIES, PLAYER, QUESTS, + // REGIONS, FACTIONS, or no clock + ENTITIES alone (minimal context). + [[nodiscard]] bool has_retail_shape() const noexcept; +}; + +// The PLAYER section (SAVE.md §3.5): exactly two tagged fields. +struct PlayerSection { + std::uint64_t player_character_uid = 0; // UID of the hero CThing in SAVED_ENTITIES + std::string player_region_name; // e.g. "Greatwood" + + friend bool operator==(const PlayerSection&, const PlayerSection&) = default; + + [[nodiscard]] static std::expected decode(std::span section); + [[nodiscard]] std::vector encode() const; +}; + +} // namespace fable::save diff --git a/rebuild/modern/save/include/fable/save/SaveContainer.hpp b/rebuild/modern/save/include/fable/save/SaveContainer.hpp new file mode 100644 index 0000000000..9b1fc2f815 --- /dev/null +++ b/rebuild/modern/save/include/fable/save/SaveContainer.hpp @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace fable::save { + +// Retail CCRC::Calc: reflected CRC-32 (poly 0xEDB88320, the zlib table), caller seed, no final +// inversion. Every on-disk hash in Fable uses seed 0 (save trailer, field tags, def tags). +[[nodiscard]] std::uint32_t ccrc(std::span data, std::uint32_t seed = 0) noexcept; + +enum class SaveContainerError { + bad_magic, // not "FableSave!" + 00 00 + truncated_header, // shorter than the 0x1C-byte header + data_length_out_of_range, // total_data_len + 4 > file size + bad_signature, // trailer != ccrc(file[0 : total_data_len]) + bad_slab_size, // retail requires exactly 0x4B000 bytes + nonzero_padding, // bytes after the trailer must all be 0x00 + chunk_out_of_bounds, // a chunk length runs past total_data_len + chunk_layout_mismatch, // chunk1 does not end exactly at total_data_len + inflate_failed, + inflated_size_mismatch, // a section's size differs from its header field + deflate_failed, + too_large, // the rewritten data does not fit the slab +}; + +// The two persist-stream sections of a retail `.sav` (`FableSave!` container), decompressed. +// +// On disk (docs/formats/SAVE.md "Container and header layout"): +// 0x00 "FableSave!" 00 00 +// 0x0C u32 total_data_len (= trailer offset) +// 0x10 u32 header_size (inflated HEADER section) +// 0x14 u32 body_size (inflated main section) +// 0x18 u32 header_zlib_size +// 0x1C header zlib stream, u32 body_zlib_size, body zlib stream +// total_data_len: u32 ccrc(file[0 : total_data_len]), then zeros to 0x4B000. +// +// All fields are explicit little-endian 32-bit values, so the layout is the same on every pointer +// width; nothing here depends on the retail 32-bit struct images. +struct SaveContainer { + std::vector header_section; + std::vector body_section; + + friend bool operator==(const SaveContainer&, const SaveContainer&) = default; +}; + +class SaveContainerCodec final { +public: + static constexpr std::size_t slab_size = 0x4B000; + static constexpr std::size_t header_size = 0x1C; + + // Accepts exactly what retail CUserProfileManager::VerifySignature (0x00409730) accepts, plus + // the chunk framing CWorld::LoadGameStateInternal (0x004A21F0) then reads. + [[nodiscard]] static std::expected + decode(std::span file); + + // Deflates each section the way retail CDataOutputStream::WriteZlibCompressed (0x009967E0) + // does: zlib level 9, 15-bit window, memLevel 8, default strategy. With classic zlib (1.2.1 + // through 1.3.x; not zlib-ng) this reproduces retail-written saves byte for byte. + [[nodiscard]] static std::expected, SaveContainerError> + encode(const SaveContainer& container); +}; + +} // namespace fable::save diff --git a/rebuild/modern/save/include/fable/save/SaveHeader.hpp b/rebuild/modern/save/include/fable/save/SaveHeader.hpp new file mode 100644 index 0000000000..ff20dd1e27 --- /dev/null +++ b/rebuild/modern/save/include/fable/save/SaveHeader.hpp @@ -0,0 +1,57 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fable::save { + +// The HEADER section, chunk0 of a `.sav`. Written by CWorld::SaveGameStateInternal (0x0049F4C0); +// field order is its Transfer call order (docs/formats/SAVE.md §3.4, 23/23 fields confirmed). +struct SaveHeader { + // Fields 2-22. The writer skips them in its minimal-context branch (`this+0x102 != 0`). + struct Details { + bool teleporting_enabled = false; + bool saving_enabled = false; + bool experience_spending_enabled = false; + bool creature_generation_enabled = false; + std::int32_t creature_generation_disabled_groups = 0; // bitmask + bool hero_sleeping_enabled = false; + bool map_table_show_quest_cards_on_used = false; + bool mini_map_enabled = false; + bool mini_map_active_before_disabled = false; + bool guild_master_messages_enabled = false; + bool summoner_death_explosion_affects_hero = false; + std::int32_t most_recent_save_type = 0; // CWorld::ESaveType + std::int32_t most_recent_save_type_before_manual_save = 0; + std::u16string most_recent_manual_save_name; // empty in autosaves + std::array save_game_marker_pos{}; + float save_game_marker_angle_xy = 0.0F; + std::array guild_seal_recall_pos{}; + float guild_seal_recall_angle_xy = 0.0F; + std::string current_region_name; // e.g. TXT_REGION_OAKVALE + std::string current_region_minimap_graphic_name; // e.g. MINIMAP_GREATWOOD + float total_time_played = 0.0F; // 4 bytes on disk + + friend bool operator==(const Details&, const Details&) = default; + }; + + std::string world_name; // "FinalAlbion" + std::int32_t world_frame = 0; // world frames (15 per second) at save time + std::optional
details; + + friend bool operator==(const SaveHeader&, const SaveHeader&) = default; + + // `section` is the whole chunk0 stream: "HEADER\0", u32 length, tagged fields. + [[nodiscard]] static std::expected decode(std::span section); + [[nodiscard]] std::vector encode() const; +}; + +} // namespace fable::save diff --git a/rebuild/modern/save/include/fable/save/TextSections.hpp b/rebuild/modern/save/include/fable/save/TextSections.hpp new file mode 100644 index 0000000000..25f2e24af0 --- /dev/null +++ b/rebuild/modern/save/include/fable/save/TextSections.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace fable::save { + +// A compressed persist-text blob, as QUESTS and FACTIONS store them: +// u32 size +// size == 0: nothing more +// otherwise: i16 clen; clen == -1: u32 clen follows +// clen <= 0: `size` raw bytes (retail's stored form) +// clen > 0: clen bytes LZO1X decoding to the first size - 3 bytes, then the +// last 3 bytes raw +// Writer 0x996610 / reader 0x995FE0. The reader takes clen as signed, so the writer stores at +// most 0x7FFF in 16 bits and uses -1 + u32 above that. It writes the stored form for size <= 3 +// and when the compressed stream would not be shorter than size - 3. +[[nodiscard]] std::expected read_lz_text(PersistReader& in); + +// Byte-exact with retail when the build links LZO (lzo_compressor_available()); otherwise the +// stored form, which decodes to the same text. +void write_lz_text(PersistWriter& out, std::string_view text); + +[[nodiscard]] bool lzo_compressor_available() noexcept; + +// One accepted boast, as the QUESTS section's tagged tail stores it. +struct AcceptedBoast { + std::string script_name; // BoastScriptName: the quest the boast belongs to + std::uint32_t id = 0; // BoastId + bool failed = false; // BoastFailed + bool completed = false; // BoastCompleted + + friend bool operator==(const AcceptedBoast&, const AcceptedBoast&) = default; +}; + +// QUESTS: the quest manager's persist text (START_QUEST_CARDS ... END_SAVED_QUESTS), then the +// accepted boasts as tagged binary fields: NumAcceptedBoasts, then per boast i the fields +// BoastScriptName, BoastId, BoastFailed, BoastCompleted (the index is part of the +// field name, so of its tag). +struct QuestsSection { + std::string text; + std::vector accepted_boasts; + + friend bool operator==(const QuestsSection&, const QuestsSection&) = default; + + [[nodiscard]] static std::expected decode(std::span section); + [[nodiscard]] std::vector encode() const; +}; + +// FACTIONS: tagged NumberOfFactions (u32), then per faction i the tagged Faction LZ text +// blob. Each text is one faction's `Name "FACTION_X"; NumberOfAllies n; Ally "..."; ... +// EndFaction`. The tags follow from the names, so only the texts are kept. +struct FactionsSection { + std::vector factions; + + friend bool operator==(const FactionsSection&, const FactionsSection&) = default; + + [[nodiscard]] static std::expected decode(std::span section); + [[nodiscard]] std::vector encode() const; +}; + +} // namespace fable::save diff --git a/rebuild/modern/save/src/Entities.cpp b/rebuild/modern/save/src/Entities.cpp new file mode 100644 index 0000000000..bbcf47f7f5 --- /dev/null +++ b/rebuild/modern/save/src/Entities.cpp @@ -0,0 +1,261 @@ +#include + +#include "Zlib.hpp" + +#include +#include + +namespace fable::save { +namespace { + +constexpr std::uint64_t record_overhead = 29; // cell_id 8 + clen 8 + ulen 4 + trailer 9 +constexpr int cell_zlib_level = 1; +constexpr std::string_view unattached_group = "UNATTACHED"; + +// The position of the first `name\0` in `bytes`, as a section start. +std::optional find_section(std::span bytes, std::string_view name) +{ + const auto it = std::search(bytes.begin(), bytes.end(), name.begin(), name.end(), + [](std::byte b, char c) { return b == static_cast(c); }); + if (it == bytes.end()) { + return std::nullopt; + } + const auto at = static_cast(it - bytes.begin()); + if (at + name.size() >= bytes.size() || bytes[at + name.size()] != std::byte{0}) { + return std::nullopt; + } + return at; +} + +} // namespace + +std::expected SavedEntities::decode(std::span section) +{ + PersistReader in(section); + const auto lead = in.u32(); + const auto slot_count = in.u32(); + const auto pad = in.u32(); + if (!lead || !slot_count || !pad) { + return std::unexpected(PersistError::truncated); + } + if (*lead != 0 || *pad != 0 || *slot_count > in.remaining() / 8) { + return std::unexpected(PersistError::bad_section); + } + + SavedEntities out; + out.slots.reserve(*slot_count); + for (std::uint32_t i = 0; i < *slot_count; ++i) { + const auto record_length = in.u64(); + if (!record_length) { + return std::unexpected(record_length.error()); + } + if (*record_length == 0) { + out.slots.emplace_back(std::nullopt); + continue; + } + const auto cell_id = in.u64(); + const auto clen = in.u64(); + const auto ulen = in.u32(); + if (!cell_id || !clen || !ulen) { + return std::unexpected(PersistError::truncated); + } + if (*record_length != record_overhead + *clen || *clen > in.remaining()) { + return std::unexpected(PersistError::bad_section); + } + const auto compressed = in.raw(static_cast(*clen)); + const auto trailer = in.raw(9); + if (!compressed || !trailer) { + return std::unexpected(PersistError::truncated); + } + EntityCell cell; + cell.cell_id = *cell_id; + std::copy(trailer->begin(), trailer->end(), cell.trailer.begin()); + if (*clen != 0) { + auto data = detail::inflate_exact(*compressed, *ulen); + if (!data) { + return std::unexpected(PersistError::bad_section); + } + cell.data = std::move(*data); + } else if (*ulen != 0) { + return std::unexpected(PersistError::bad_section); + } + out.slots.emplace_back(std::move(cell)); + } + + const auto dead_count = in.u32(); + if (!dead_count) { + return std::unexpected(dead_count.error()); + } + if (in.remaining() != static_cast(*dead_count) * 20) { + return std::unexpected(PersistError::bad_section); + } + out.dead_entities.resize(*dead_count); + for (PersistedDeadEntity& dead : out.dead_entities) { + if (const auto tag = in.tag(""); !tag) { + return std::unexpected(tag.error()); + } + const auto uid = in.u64(); + if (const auto tag = in.tag(""); !tag) { + return std::unexpected(tag.error()); + } + const auto crc = in.u32(); + dead = PersistedDeadEntity{*uid, *crc}; // the size check above covers both reads + } + return out; +} + +std::expected, PersistError> SavedEntities::encode() const +{ + PersistWriter out; + out.u32(0); + out.u32(static_cast(slots.size())); + out.u32(0); + for (const auto& slot : slots) { + if (!slot) { + out.u64(0); + continue; + } + std::vector compressed; + if (!slot->data.empty()) { + auto deflated = detail::deflate_retail(slot->data, cell_zlib_level); + if (!deflated) { + return std::unexpected(PersistError::bad_section); + } + compressed = std::move(*deflated); + } + out.u64(record_overhead + compressed.size()); + out.u64(slot->cell_id); + out.u64(compressed.size()); + out.u32(static_cast(slot->data.size())); + out.bytes_raw(compressed); + out.bytes_raw(slot->trailer); + } + out.u32(static_cast(dead_entities.size())); + for (const PersistedDeadEntity& dead : dead_entities) { + out.tag(""); + out.u64(dead.uid); + out.tag(""); + out.u32(dead.section_name_crc); + } + return out.take(); +} + +std::expected SavedNpcNames::decode(std::span section) +{ + SavedNpcNames out; + if (section.empty()) { + return out; + } + PersistReader in(section); + const auto count = in.u32(); + if (!count) { + return std::unexpected(count.error()); + } + // Each entry is at least u64 + u32 + a UTF-16 NUL. + if (*count > in.remaining() / 14) { + return std::unexpected(PersistError::bad_section); + } + out.entries.emplace(); + out.entries->reserve(*count); + for (std::uint32_t i = 0; i < *count; ++i) { + const auto uid = in.u64(); + if (!uid) { + return std::unexpected(uid.error()); + } + auto group = in.counted_string(); + if (!group) { + return std::unexpected(group.error()); + } + auto name = in.wstring(); + if (!name) { + return std::unexpected(name.error()); + } + if (*group == unattached_group) { + group->clear(); + } + out.entries->push_back(NpcName{*uid, std::move(*group), std::move(*name)}); + } + if (in.remaining() != 0) { + return std::unexpected(PersistError::bad_section); + } + return out; +} + +std::vector SavedNpcNames::encode() const +{ + PersistWriter out; + if (!entries) { + return out.take(); + } + out.u32(static_cast(entries->size())); + for (const NpcName& entry : *entries) { + out.u64(entry.uid); + out.counted_string(entry.group.empty() ? unattached_group : std::string_view(entry.group)); + out.wstring(entry.name); + } + return out.take(); +} + +std::expected EntitiesSection::decode(std::span section) +{ + const auto at = find_section(section, "SAVED_ENTITIES"); + if (!at) { + return std::unexpected(PersistError::bad_section); + } + EntitiesSection out; + if (*at != 0) { + PersistReader head(section.first(*at)); + const auto version_tag = head.tag(""); + const auto version = head.u32(); + const auto uid_tag = head.tag(""); + const auto uid = head.u64(); + if (!version_tag || !version || !uid_tag || !uid || head.remaining() != 0) { + return std::unexpected(PersistError::bad_section); + } + out.head = EntitiesHead{*version, *uid}; + } + + PersistReader in(section.subspan(*at)); + const auto saved = in.section("SAVED_ENTITIES"); + if (!saved) { + return std::unexpected(saved.error()); + } + auto entities = SavedEntities::decode(*saved); + if (!entities) { + return std::unexpected(entities.error()); + } + out.saved_entities = std::move(*entities); + const auto names = in.section("SAVED_NPC_NAMES"); + if (!names) { + return std::unexpected(names.error()); + } + auto npc_names = SavedNpcNames::decode(*names); + if (!npc_names) { + return std::unexpected(npc_names.error()); + } + out.saved_npc_names = std::move(*npc_names); + if (in.remaining() != 0) { + return std::unexpected(PersistError::bad_section); + } + return out; +} + +std::expected, PersistError> EntitiesSection::encode() const +{ + auto saved = saved_entities.encode(); + if (!saved) { + return std::unexpected(saved.error()); + } + PersistWriter out; + if (head) { + out.tag(""); + out.u32(head->format_version); + out.tag(""); + out.u64(head->current_unique_id); + } + out.section("SAVED_ENTITIES", *saved); + out.section("SAVED_NPC_NAMES", saved_npc_names.encode()); + return out.take(); +} + +} // namespace fable::save diff --git a/rebuild/modern/save/src/Lzo.cpp b/rebuild/modern/save/src/Lzo.cpp new file mode 100644 index 0000000000..1347bdf10f --- /dev/null +++ b/rebuild/modern/save/src/Lzo.cpp @@ -0,0 +1,257 @@ +#include "Lzo.hpp" + +#include + +#if defined(FABLE_SAVE_HAVE_LZO) +#include +#endif + +namespace fable::save::detail { +namespace { + +// LZO1X stream decoder. Token grammar (t = the token byte): +// first byte > 17 initial literal run of t - 17 bytes +// t < 16 at loop top literal run of t + 3 bytes (t == 0: zero-byte extension, base 15) +// t >= 64 M2: length (t >> 5) + 1, distance 1 + ((t >> 2) & 7) + (next << 3) +// t >= 32 M3: length (t & 31) + 2 (extension base 31), 14-bit distance + 1 +// t >= 16 M4: length (t & 7) + 2 (extension base 7), distance 0x4000 + ...; +// an encoded distance of 0 is the end of the stream +// t < 16 after a run a 3-byte match at distance 0x801 + (t >> 2) + (next << 2) +// t < 16 after a match a 2-byte match at distance 1 + (t >> 2) + (next << 2) +// After every match, the low two bits of the byte two before the input position give 0-3 +// literals to copy; a nonzero count is followed directly by the next match token. +class Decoder final { +public: + Decoder(std::span in, std::size_t expected) : in_(in) { out_.reserve(expected); } + + std::expected, LzoError> run() + { + enum class State { loop, after_run, match, after_match }; + State state = State::loop; + std::size_t t = 0; + + if (!in_.empty() && byte_at(0) > 17) { + t = byte_at(0) - 17; + ip_ = 1; + if (!literals(t)) { + return std::unexpected(LzoError::truncated); + } + state = State::after_run; + if (t < 4) { + // A short initial run plays the role of a match's trailing literals. + if (!next(t)) { + return std::unexpected(LzoError::truncated); + } + state = State::match; + } + } + + for (;;) { + switch (state) { + case State::loop: { + if (!next(t)) { + return std::unexpected(LzoError::truncated); + } + if (t >= 16) { + state = State::match; + break; + } + if (t == 0 && !extend(t, 15)) { + return std::unexpected(LzoError::truncated); + } + if (!literals(t + 3)) { + return std::unexpected(LzoError::truncated); + } + state = State::after_run; + break; + } + case State::after_run: { + std::size_t h = 0; + if (!next(t)) { + return std::unexpected(LzoError::truncated); + } + if (t >= 16) { + state = State::match; + break; + } + if (!next(h)) { + return std::unexpected(LzoError::truncated); + } + if (!copy_match(0x801 + (t >> 2) + (h << 2), 3)) { + return std::unexpected(LzoError::bad_distance); + } + state = State::after_match; + break; + } + case State::match: { + std::size_t distance = 0; + std::size_t length = 0; + std::size_t h = 0; + if (t >= 64) { + if (!next(h)) { + return std::unexpected(LzoError::truncated); + } + distance = 1 + ((t >> 2) & 7) + (h << 3); + length = (t >> 5) + 1; + } else if (t >= 32) { + std::size_t n = t & 31; + if (n == 0 && !extend(n, 31)) { + return std::unexpected(LzoError::truncated); + } + std::size_t lo = 0; + if (!next(lo) || !next(h)) { + return std::unexpected(LzoError::truncated); + } + distance = 1 + ((h << 6) | (lo >> 2)); + length = n + 2; + } else if (t >= 16) { + std::size_t n = t & 7; + if (n == 0 && !extend(n, 7)) { + return std::unexpected(LzoError::truncated); + } + std::size_t lo = 0; + if (!next(lo) || !next(h)) { + return std::unexpected(LzoError::truncated); + } + const std::size_t encoded = ((t & 8) << 11) + ((h << 6) | (lo >> 2)); + if (encoded == 0) { + return finish(); + } + distance = 0x4000 + encoded; + length = n + 2; + } else { + if (!next(h)) { + return std::unexpected(LzoError::truncated); + } + distance = 1 + (t >> 2) + (h << 2); + length = 2; + } + if (!copy_match(distance, length)) { + return std::unexpected(LzoError::bad_distance); + } + state = State::after_match; + break; + } + case State::after_match: { + const std::size_t trailing = byte_at(ip_ - 2) & 3; + if (trailing == 0) { + state = State::loop; + break; + } + if (!literals(trailing) || !next(t)) { + return std::unexpected(LzoError::truncated); + } + state = State::match; + break; + } + } + } + } + +private: + std::size_t byte_at(std::size_t at) const { return std::to_integer(in_[at]); } + + bool next(std::size_t& value) + { + if (ip_ >= in_.size()) { + return false; + } + value = byte_at(ip_++); + return true; + } + + // Zero bytes add 255 each; the first nonzero byte adds itself plus `base`. + bool extend(std::size_t& value, std::size_t base) + { + std::size_t total = 0; + while (ip_ < in_.size() && in_[ip_] == std::byte{0}) { + total += 255; + ++ip_; + } + std::size_t last = 0; + if (!next(last)) { + return false; + } + value = total + base + last; + return true; + } + + bool literals(std::size_t count) + { + if (count > in_.size() - ip_) { + return false; + } + out_.insert(out_.end(), in_.begin() + static_cast(ip_), + in_.begin() + static_cast(ip_ + count)); + ip_ += count; + return true; + } + + bool copy_match(std::size_t distance, std::size_t length) + { + if (distance > out_.size()) { + return false; + } + std::size_t from = out_.size() - distance; + for (std::size_t i = 0; i < length; ++i) { + out_.push_back(out_[from++]); // overlapping copies repeat, as in the format + } + return true; + } + + std::expected, LzoError> finish() + { + if (ip_ != in_.size()) { + return std::unexpected(LzoError::trailing_bytes); + } + return std::move(out_); + } + + std::span in_; + std::size_t ip_ = 0; + std::vector out_; +}; + +} // namespace + +std::expected, LzoError> lzo1x_decompress(std::span stream, + std::size_t expected_size) +{ + auto out = Decoder(stream, expected_size).run(); + if (out && out->size() != expected_size) { + return std::unexpected(LzoError::size_mismatch); + } + return out; +} + +bool lzo_compressor_available() noexcept +{ +#if defined(FABLE_SAVE_HAVE_LZO) + return true; +#else + return false; +#endif +} + +std::optional> lzo1x_999_compress(std::span data) +{ +#if defined(FABLE_SAVE_HAVE_LZO) + // lzo1x_999 never expands by more than size / 16 + 64 + 3 (LZO's documented bound). + std::vector out(data.size() + data.size() / 16 + 64 + 3); + std::vector work(LZO1X_999_MEM_COMPRESS); + lzo_uint out_size = 0; + const int status = ::lzo1x_999_compress(reinterpret_cast(data.data()), + static_cast(data.size()), + reinterpret_cast(out.data()), &out_size, work.data()); + if (status != LZO_E_OK) { + return std::nullopt; + } + out.resize(out_size); + return out; +#else + (void)data; + return std::nullopt; +#endif +} + +} // namespace fable::save::detail diff --git a/rebuild/modern/save/src/Lzo.hpp b/rebuild/modern/save/src/Lzo.hpp new file mode 100644 index 0000000000..8f4f5d1572 --- /dev/null +++ b/rebuild/modern/save/src/Lzo.hpp @@ -0,0 +1,33 @@ +#pragma once + +// Internal: the LZO1X operations the QUESTS and FACTIONS text blobs need. +// +// The decoder is this module's own implementation of the LZO1X stream format (MIT, like the +// rest of the tree). The compressor is upstream LZO's lzo1x_999_compress, which retail calls +// through 0xC08130 (level-8 parameters, 0x70000 bytes of work memory). LZO is GPL-2.0-or-later, +// so it is linked only when the build defines FABLE_SAVE_HAVE_LZO (CMake FABLE_SAVE_WITH_LZO). + +#include +#include +#include +#include +#include + +namespace fable::save::detail { + +enum class LzoError { + truncated, // the stream ends inside a token, a literal run or a match + bad_distance, // a match reaches back before the start of the output + size_mismatch, // the stream decodes to a different size than expected + trailing_bytes, // bytes follow the end-of-stream marker +}; + +[[nodiscard]] std::expected, LzoError> lzo1x_decompress( + std::span stream, std::size_t expected_size); + +[[nodiscard]] bool lzo_compressor_available() noexcept; + +// lzo1x_999_compress(data); nullopt when the build has no LZO or compression fails. +[[nodiscard]] std::optional> lzo1x_999_compress(std::span data); + +} // namespace fable::save::detail diff --git a/rebuild/modern/save/src/PersistStream.cpp b/rebuild/modern/save/src/PersistStream.cpp new file mode 100644 index 0000000000..a0aaa073d5 --- /dev/null +++ b/rebuild/modern/save/src/PersistStream.cpp @@ -0,0 +1,241 @@ +#include + +#include + +#include + +namespace fable::save { +namespace { + +std::uint32_t le32(std::span b) noexcept +{ + return static_cast(b[0]) | static_cast(b[1]) << 8 + | static_cast(b[2]) << 16 | static_cast(b[3]) << 24; +} + +std::span as_bytes(std::string_view text) noexcept +{ + return {reinterpret_cast(text.data()), text.size()}; +} + +} // namespace + +std::uint32_t field_tag(std::string_view name) noexcept +{ + return ccrc(as_bytes(name)); +} + +std::expected, PersistError> PersistReader::take(std::size_t count) +{ + if (count > remaining()) { + return std::unexpected(PersistError::truncated); + } + const auto out = bytes_.subspan(pos_, count); + pos_ += count; + return out; +} + +std::expected PersistReader::tag(std::string_view name) +{ + const auto value = u32(); + if (!value) { + return std::unexpected(value.error()); + } + if (*value != field_tag(name)) { + return std::unexpected(PersistError::tag_mismatch); + } + return {}; +} + +std::expected PersistReader::u32() +{ + return take(4).transform(le32); +} + +std::expected PersistReader::i32() +{ + return u32().transform([](std::uint32_t v) { return static_cast(v); }); +} + +std::expected PersistReader::u64() +{ + return take(8).transform([](std::span b) { + return static_cast(le32(b.first(4))) | static_cast(le32(b.subspan(4))) << 32; + }); +} + +std::expected PersistReader::f32() +{ + return u32().transform([](std::uint32_t v) { return std::bit_cast(v); }); +} + +std::expected PersistReader::boolean() +{ + const auto b = take(1); + if (!b) { + return std::unexpected(b.error()); + } + const auto v = std::to_integer((*b)[0]); + if (v > 1) { + return std::unexpected(PersistError::invalid_bool); + } + return v == 1; +} + +std::expected PersistReader::cstring() +{ + std::string out; + for (std::size_t i = pos_; i < bytes_.size(); ++i) { + if (bytes_[i] == std::byte{0}) { + pos_ = i + 1; + return out; + } + out.push_back(static_cast(bytes_[i])); + } + return std::unexpected(PersistError::unterminated); +} + +std::expected PersistReader::counted_string() +{ + const auto length = u32(); + if (!length) { + return std::unexpected(length.error()); + } + const auto raw = take(*length); + if (!raw) { + return std::unexpected(raw.error()); + } + std::string out; + out.reserve(raw->size()); + for (const std::byte b : *raw) { + out.push_back(static_cast(b)); + } + return out; +} + +std::expected PersistReader::wstring() +{ + std::u16string out; + for (std::size_t i = pos_; i + 1 < bytes_.size(); i += 2) { + const auto unit = static_cast(std::to_integer(bytes_[i]) + | std::to_integer(bytes_[i + 1]) << 8); + if (unit == 0) { + pos_ = i + 2; + return out; + } + out.push_back(unit); + } + return std::unexpected(PersistError::unterminated); +} + +std::expected, PersistError> PersistReader::vector3() +{ + std::array out{}; + for (float& v : out) { + const auto f = f32(); + if (!f) { + return std::unexpected(f.error()); + } + v = *f; + } + return out; +} + +std::expected, PersistError> PersistReader::raw(std::size_t count) +{ + return take(count); +} + +std::expected, PersistError> PersistReader::section(std::string_view name) +{ + const auto got = cstring(); + if (!got) { + return std::unexpected(got.error()); + } + if (*got != name) { + return std::unexpected(PersistError::bad_section); + } + const auto length = u32(); + if (!length) { + return std::unexpected(length.error()); + } + return take(*length); +} + +void PersistWriter::tag(std::string_view name) +{ + u32(field_tag(name)); +} + +void PersistWriter::u32(std::uint32_t value) +{ + for (int i = 0; i < 4; ++i) { + bytes_.push_back(static_cast((value >> (8 * i)) & 0xFFU)); + } +} + +void PersistWriter::i32(std::int32_t value) +{ + u32(static_cast(value)); +} + +void PersistWriter::u64(std::uint64_t value) +{ + u32(static_cast(value & 0xFFFFFFFFU)); + u32(static_cast(value >> 32)); +} + +void PersistWriter::f32(float value) +{ + u32(std::bit_cast(value)); +} + +void PersistWriter::boolean(bool value) +{ + bytes_.push_back(value ? std::byte{1} : std::byte{0}); +} + +void PersistWriter::cstring(std::string_view value) +{ + const auto raw = as_bytes(value); + bytes_.insert(bytes_.end(), raw.begin(), raw.end()); + bytes_.push_back(std::byte{0}); +} + +void PersistWriter::counted_string(std::string_view value) +{ + u32(static_cast(value.size())); + const auto raw = as_bytes(value); + bytes_.insert(bytes_.end(), raw.begin(), raw.end()); +} + +void PersistWriter::wstring(std::u16string_view value) +{ + for (const char16_t unit : value) { + bytes_.push_back(static_cast(unit & 0xFFU)); + bytes_.push_back(static_cast(unit >> 8)); + } + bytes_.push_back(std::byte{0}); + bytes_.push_back(std::byte{0}); +} + +void PersistWriter::vector3(const std::array& value) +{ + for (const float v : value) { + f32(v); + } +} + +void PersistWriter::section(std::string_view name, std::span body) +{ + cstring(name); + u32(static_cast(body.size())); + bytes_raw(body); +} + +void PersistWriter::bytes_raw(std::span raw) +{ + bytes_.insert(bytes_.end(), raw.begin(), raw.end()); +} + +} // namespace fable::save diff --git a/rebuild/modern/save/src/Profile.cpp b/rebuild/modern/save/src/Profile.cpp new file mode 100644 index 0000000000..4993cbe08f --- /dev/null +++ b/rebuild/modern/save/src/Profile.cpp @@ -0,0 +1,116 @@ +#include + +#include + +#include +#include +#include + +namespace fable::save { +namespace { + +constexpr std::array magic{ + std::byte{'F'}, std::byte{'a'}, std::byte{'b'}, std::byte{'l'}, std::byte{'e'}, std::byte{'P'}, + std::byte{'r'}, std::byte{'o'}, std::byte{0}, std::byte{0}, std::byte{0}, std::byte{0}, +}; +constexpr std::size_t text_start = 0x10; + +std::uint32_t read_u32(std::span bytes, std::size_t offset) noexcept +{ + return static_cast(bytes[offset]) | static_cast(bytes[offset + 1]) << 8 + | static_cast(bytes[offset + 2]) << 16 | static_cast(bytes[offset + 3]) << 24; +} + +void append_u32(std::vector& out, std::uint32_t value) +{ + for (int i = 0; i < 4; ++i) { + out.push_back(static_cast((value >> (8 * i)) & 0xFFU)); + } +} + +} // namespace + +std::expected Profile::decode(std::span file) +{ + if (file.size() < text_start + 4) { + return std::unexpected(ProfileError::truncated); + } + if (!std::equal(magic.begin(), magic.end(), file.begin())) { + return std::unexpected(ProfileError::bad_magic); + } + const std::size_t total = read_u32(file, 0x0C); + if (total != file.size() - 4 || total < text_start) { + return std::unexpected(ProfileError::data_length_out_of_range); + } + if (read_u32(file, total) != ccrc(file.first(total))) { + return std::unexpected(ProfileError::bad_signature); + } + + const auto text = file.subspan(text_start, total - text_start); + const std::string_view all(reinterpret_cast(text.data()), text.size()); + Profile out; + for (std::size_t at = 0; at < all.size();) { + const auto end = all.find("\r\n", at); + if (end == std::string_view::npos) { + return std::unexpected(ProfileError::bad_line); + } + const std::string_view line = all.substr(at, end - at); + const auto space = line.find(' '); + if (space == std::string_view::npos || space == 0 || !line.ends_with(';')) { + return std::unexpected(ProfileError::bad_line); + } + out.entries_.push_back(Entry{std::string(line.substr(0, space)), + std::string(line.substr(space + 1, line.size() - space - 2))}); + at = end + 2; + } + return out; +} + +std::vector Profile::encode() const +{ + std::vector out(magic.begin(), magic.end()); + append_u32(out, 0); // total_data_len, patched below + for (const Entry& e : entries_) { + for (const char c : e.name + ' ' + e.value + ";\r\n") { + out.push_back(static_cast(c)); + } + } + const auto total = static_cast(out.size()); + for (int i = 0; i < 4; ++i) { + out[0x0C + static_cast(i)] = static_cast((total >> (8 * i)) & 0xFFU); + } + append_u32(out, ccrc(out)); + return out; +} + +std::optional Profile::value(std::string_view name) const noexcept +{ + const auto it = std::find_if(entries_.begin(), entries_.end(), [&](const Entry& e) { return e.name == name; }); + if (it == entries_.end()) { + return std::nullopt; + } + return std::string_view(it->value); +} + +bool Profile::set_value(std::string_view name, std::string value) +{ + const auto it = std::find_if(entries_.begin(), entries_.end(), [&](const Entry& e) { return e.name == name; }); + if (it == entries_.end()) { + return false; + } + it->value = std::move(value); + return true; +} + +std::vector Profile::save_game_names() const +{ + std::vector names; + for (std::size_t slot = 1; slot <= save_slot_count; ++slot) { + const auto raw = value("SaveGameNames" + std::to_string(slot)).value_or(""); + const bool quoted = raw.size() >= 2 && raw.front() == '"' && raw.back() == '"'; + names.emplace_back(quoted ? raw.substr(1, raw.size() - 2) : raw); + } + return names; +} + +} // namespace fable::save diff --git a/rebuild/modern/save/src/Regions.cpp b/rebuild/modern/save/src/Regions.cpp new file mode 100644 index 0000000000..4679f0e723 --- /dev/null +++ b/rebuild/modern/save/src/Regions.cpp @@ -0,0 +1,67 @@ +#include + +namespace fable::save { + +std::expected RegionsSection::decode(std::span section) +{ + PersistReader in(section); + if (const auto tag = in.tag(""); !tag) { + return std::unexpected(tag.error()); + } + const auto table_size = in.u32(); + if (!table_size) { + return std::unexpected(table_size.error()); + } + // A world always has its unwritten element 0; each record is at least 4 tags, a NUL and 3 bools. + if (*table_size == 0 || *table_size - 1 > in.remaining() / 20) { + return std::unexpected(PersistError::bad_section); + } + RegionsSection out; + out.regions.reserve(*table_size - 1); + for (std::uint32_t i = 1; i < *table_size; ++i) { + Region region; + if (const auto tag = in.tag(""); !tag) { + return std::unexpected(tag.error()); + } + auto name = in.cstring(); + if (!name) { + return std::unexpected(name.error()); + } + region.name = std::move(*name); + for (bool* flag : {®ion.appear_on_world_map, ®ion.creature_generation_enabled, + ®ion.sound_themes_enabled}) { + if (const auto tag = in.tag(""); !tag) { + return std::unexpected(tag.error()); + } + const auto value = in.boolean(); + if (!value) { + return std::unexpected(value.error()); + } + *flag = *value; + } + out.regions.push_back(std::move(region)); + } + if (in.remaining() != 0) { + return std::unexpected(PersistError::bad_section); + } + return out; +} + +std::vector RegionsSection::encode() const +{ + PersistWriter out; + out.tag(""); + out.u32(static_cast(regions.size() + 1)); + for (const Region& region : regions) { + out.tag(""); + out.cstring(region.name); + for (const bool flag : {region.appear_on_world_map, region.creature_generation_enabled, + region.sound_themes_enabled}) { + out.tag(""); + out.boolean(flag); + } + } + return out.take(); +} + +} // namespace fable::save diff --git a/rebuild/modern/save/src/SaveBody.cpp b/rebuild/modern/save/src/SaveBody.cpp new file mode 100644 index 0000000000..10d152ce0d --- /dev/null +++ b/rebuild/modern/save/src/SaveBody.cpp @@ -0,0 +1,152 @@ +#include + +#include +#include +#include + +namespace fable::save { +namespace { + +constexpr std::size_t max_section_name = 64; + +// A section name is printable ASCII, so a stray length or tag is never mistaken for one. +bool plausible_name(std::span bytes, std::size_t at) +{ + std::size_t i = at; + for (; i < bytes.size() && i - at <= max_section_name; ++i) { + const auto c = std::to_integer(bytes[i]); + if (c == 0) { + return i > at; + } + if (c < 0x20 || c > 0x7E) { + return false; + } + } + return false; +} + +} // namespace + +std::expected SaveBody::decode(std::span body) +{ + PersistReader in(body); + SaveBody out; + + // The clock is the only content that is not a section; its first tag tells them apart. + if (body.size() >= 4 && !plausible_name(body, 0)) { + GameClock clock; + for (auto [name, target] : {std::pair{"Time", &clock.time}, std::pair{"TimeStep", &clock.time_step}}) { + if (auto t = in.tag(name); !t) { + return std::unexpected(t.error()); + } + const auto v = in.f32(); + if (!v) { + return std::unexpected(v.error()); + } + *target = *v; + } + if (auto t = in.tag("DayCount"); !t) { + return std::unexpected(t.error()); + } + const auto days = in.i32(); + if (!days) { + return std::unexpected(days.error()); + } + clock.day_count = *days; + out.clock = clock; + } + + while (in.remaining() != 0) { + if (!plausible_name(body, in.position())) { + return std::unexpected(PersistError::bad_section); + } + auto name = in.cstring(); + if (!name) { + return std::unexpected(name.error()); + } + const auto length = in.i32(); + if (!length) { + return std::unexpected(length.error()); + } + if (*length < 0) { + return std::unexpected(PersistError::bad_section); + } + const auto bytes = in.raw(static_cast(*length)); + if (!bytes) { + return std::unexpected(PersistError::bad_section); + } + out.sections.push_back(Section{std::move(*name), {bytes->begin(), bytes->end()}}); + } + return out; +} + +std::vector SaveBody::encode() const +{ + PersistWriter out; + if (clock) { + out.tag("Time"); + out.f32(clock->time); + out.tag("TimeStep"); + out.f32(clock->time_step); + out.tag("DayCount"); + out.i32(clock->day_count); + } + for (const Section& s : sections) { + out.section(s.name, s.bytes); + } + return out.take(); +} + +const SaveBody::Section* SaveBody::find(std::string_view name) const noexcept +{ + const auto it = std::find_if(sections.begin(), sections.end(), [&](const Section& s) { return s.name == name; }); + return it == sections.end() ? nullptr : &*it; +} + +bool SaveBody::has_retail_shape() const noexcept +{ + constexpr std::array full{"ENTITIES", "PLAYER", "QUESTS", "REGIONS", "FACTIONS"}; + const auto names_are = [&](std::span expected) { + return std::equal(sections.begin(), sections.end(), expected.begin(), expected.end(), + [](const Section& s, std::string_view n) { return s.name == n; }); + }; + return clock ? names_are(full) : names_are(std::span(full).first(1)); +} + +std::expected PlayerSection::decode(std::span section) +{ + PersistReader in(section); + PlayerSection out; + if (auto t = in.tag("PlayerCharacterUID"); !t) { + return std::unexpected(t.error()); + } + const auto uid = in.u64(); + if (!uid) { + return std::unexpected(uid.error()); + } + out.player_character_uid = *uid; + if (auto t = in.tag("PlayerRegionName"); !t) { + return std::unexpected(t.error()); + } + auto region = in.cstring(); + if (!region) { + return std::unexpected(region.error()); + } + out.player_region_name = std::move(*region); + if (in.remaining() != 0) { + return std::unexpected(PersistError::bad_section); + } + return out; +} + +std::vector PlayerSection::encode() const +{ + PersistWriter out; + out.tag("PlayerCharacterUID"); + out.u64(player_character_uid); + out.tag("PlayerRegionName"); + out.cstring(player_region_name); + return out.take(); +} + +} // namespace fable::save diff --git a/rebuild/modern/save/src/SaveContainer.cpp b/rebuild/modern/save/src/SaveContainer.cpp new file mode 100644 index 0000000000..fdae8e039b --- /dev/null +++ b/rebuild/modern/save/src/SaveContainer.cpp @@ -0,0 +1,166 @@ +#include + +#include "Zlib.hpp" + +#include +#include +#include + +namespace fable::save { +namespace { + +constexpr std::array magic{ + std::byte{'F'}, std::byte{'a'}, std::byte{'b'}, std::byte{'l'}, std::byte{'e'}, std::byte{'S'}, + std::byte{'a'}, std::byte{'v'}, std::byte{'e'}, std::byte{'!'}, std::byte{0}, std::byte{0}, +}; + +constexpr std::array make_crc_table() noexcept +{ + std::array table{}; + for (std::uint32_t n = 0; n < table.size(); ++n) { + std::uint32_t c = n; + for (int bit = 0; bit < 8; ++bit) { + c = (c & 1U) != 0 ? (c >> 1) ^ 0xEDB88320U : c >> 1; + } + table[n] = c; + } + return table; +} + +constexpr auto crc_table = make_crc_table(); + +std::uint32_t read_u32(std::span bytes, std::size_t offset) noexcept +{ + return static_cast(bytes[offset]) + | static_cast(bytes[offset + 1]) << 8 + | static_cast(bytes[offset + 2]) << 16 + | static_cast(bytes[offset + 3]) << 24; +} + +void write_u32(std::vector& out, std::size_t offset, std::uint32_t value) noexcept +{ + for (int i = 0; i < 4; ++i) { + out[offset + static_cast(i)] = static_cast((value >> (8 * i)) & 0xFFU); + } +} + +SaveContainerError to_container_error(detail::ZlibError error) noexcept +{ + switch (error) { + case detail::ZlibError::inflate_failed: return SaveContainerError::inflate_failed; + case detail::ZlibError::size_mismatch: return SaveContainerError::inflated_size_mismatch; + case detail::ZlibError::deflate_failed: return SaveContainerError::deflate_failed; + case detail::ZlibError::too_large: return SaveContainerError::too_large; + } + return SaveContainerError::inflate_failed; +} + +std::expected, SaveContainerError> inflate_exact( + std::span compressed, std::uint32_t expected_size) +{ + return detail::inflate_exact(compressed, expected_size).transform_error(to_container_error); +} + +// Container chunks are deflated at level 9 (SaveContainerCodec::encode). +std::expected, SaveContainerError> deflate_retail(std::span data) +{ + return detail::deflate_retail(data, 9).transform_error(to_container_error); +} + +} // namespace + +std::uint32_t ccrc(std::span data, std::uint32_t seed) noexcept +{ + std::uint32_t crc = seed; + for (const std::byte b : data) { + crc = (crc >> 8) ^ crc_table[(crc ^ static_cast(b)) & 0xFFU]; + } + return crc; +} + +std::expected SaveContainerCodec::decode(std::span file) +{ + if (file.size() < header_size) { + return std::unexpected(SaveContainerError::truncated_header); + } + if (!std::equal(magic.begin(), magic.end(), file.begin())) { + return std::unexpected(SaveContainerError::bad_magic); + } + + // VerifySignature order: bounds, CRC, then slab size and padding. + const std::size_t total = read_u32(file, 0x0C); + if (total < header_size || total > file.size() - 4) { + return std::unexpected(SaveContainerError::data_length_out_of_range); + } + if (read_u32(file, total) != ccrc(file.first(total))) { + return std::unexpected(SaveContainerError::bad_signature); + } + if (file.size() != slab_size) { + return std::unexpected(SaveContainerError::bad_slab_size); + } + const auto padding = file.subspan(total + 4); + if (std::any_of(padding.begin(), padding.end(), [](std::byte b) { return b != std::byte{0}; })) { + return std::unexpected(SaveContainerError::nonzero_padding); + } + + const std::uint32_t header_bytes = read_u32(file, 0x10); + const std::uint32_t body_bytes = read_u32(file, 0x14); + const std::size_t header_zlib = read_u32(file, 0x18); + if (header_zlib > total - header_size || total - header_size - header_zlib < 4) { + return std::unexpected(SaveContainerError::chunk_out_of_bounds); + } + const std::size_t body_zlib_at = header_size + header_zlib + 4; + const std::size_t body_zlib = read_u32(file, header_size + header_zlib); + if (body_zlib > total - body_zlib_at) { + return std::unexpected(SaveContainerError::chunk_out_of_bounds); + } + if (body_zlib_at + body_zlib != total) { + return std::unexpected(SaveContainerError::chunk_layout_mismatch); + } + + auto header = inflate_exact(file.subspan(header_size, header_zlib), header_bytes); + if (!header) { + return std::unexpected(header.error()); + } + auto body = inflate_exact(file.subspan(body_zlib_at, body_zlib), body_bytes); + if (!body) { + return std::unexpected(body.error()); + } + return SaveContainer{std::move(*header), std::move(*body)}; +} + +std::expected, SaveContainerError> SaveContainerCodec::encode(const SaveContainer& container) +{ + if (container.header_section.size() > std::numeric_limits::max() + || container.body_section.size() > std::numeric_limits::max()) { + return std::unexpected(SaveContainerError::too_large); + } + auto header = deflate_retail(container.header_section); + if (!header) { + return std::unexpected(header.error()); + } + auto body = deflate_retail(container.body_section); + if (!body) { + return std::unexpected(body.error()); + } + + const std::size_t total = header_size + header->size() + 4 + body->size(); + if (total + 4 > slab_size) { + return std::unexpected(SaveContainerError::too_large); + } + + std::vector file(slab_size, std::byte{0}); + std::copy(magic.begin(), magic.end(), file.begin()); + write_u32(file, 0x0C, static_cast(total)); + write_u32(file, 0x10, static_cast(container.header_section.size())); + write_u32(file, 0x14, static_cast(container.body_section.size())); + write_u32(file, 0x18, static_cast(header->size())); + std::copy(header->begin(), header->end(), file.begin() + header_size); + const std::size_t body_len_at = header_size + header->size(); + write_u32(file, body_len_at, static_cast(body->size())); + std::copy(body->begin(), body->end(), file.begin() + static_cast(body_len_at + 4)); + write_u32(file, total, ccrc(std::span(file).first(total))); + return file; +} + +} // namespace fable::save diff --git a/rebuild/modern/save/src/SaveHeader.cpp b/rebuild/modern/save/src/SaveHeader.cpp new file mode 100644 index 0000000000..157a076ca1 --- /dev/null +++ b/rebuild/modern/save/src/SaveHeader.cpp @@ -0,0 +1,135 @@ +#include + +namespace fable::save { +namespace { + +// Reads `[tag(name)][value]` with the reader's `method` into `target`; returns the first error. +#define FABLE_READ_FIELD(reader, name, method, target) \ + do { \ + if (auto t = (reader).tag(name); !t) { \ + return std::unexpected(t.error()); \ + } \ + auto v = (reader).method(); \ + if (!v) { \ + return std::unexpected(v.error()); \ + } \ + (target) = std::move(*v); \ + } while (false) + +std::expected read_details(PersistReader& in) +{ + SaveHeader::Details d; + FABLE_READ_FIELD(in, "TeleportingEnabled", boolean, d.teleporting_enabled); + FABLE_READ_FIELD(in, "SavingEnabled", boolean, d.saving_enabled); + FABLE_READ_FIELD(in, "ExperienceSpendingEnabled", boolean, d.experience_spending_enabled); + FABLE_READ_FIELD(in, "CreatureGenerationEnabled", boolean, d.creature_generation_enabled); + FABLE_READ_FIELD(in, "CreatureGenerationDisabledGroups", i32, d.creature_generation_disabled_groups); + FABLE_READ_FIELD(in, "HeroSleepingEnabled", boolean, d.hero_sleeping_enabled); + FABLE_READ_FIELD(in, "MapTableShowQuestCardsOnUsed", boolean, d.map_table_show_quest_cards_on_used); + FABLE_READ_FIELD(in, "MiniMapEnabled", boolean, d.mini_map_enabled); + FABLE_READ_FIELD(in, "MiniMapActiveBeforeDisabled", boolean, d.mini_map_active_before_disabled); + FABLE_READ_FIELD(in, "GuildMasterMessagesEnabled", boolean, d.guild_master_messages_enabled); + FABLE_READ_FIELD(in, "SummonerDeathExplosionAffectsHero", boolean, d.summoner_death_explosion_affects_hero); + FABLE_READ_FIELD(in, "MostRecentSaveType", i32, d.most_recent_save_type); + FABLE_READ_FIELD(in, "MostRecentSaveTypeBeforeManualSave", i32, d.most_recent_save_type_before_manual_save); + FABLE_READ_FIELD(in, "MostRecentManualSaveName", wstring, d.most_recent_manual_save_name); + FABLE_READ_FIELD(in, "SaveGameMarkerPos", vector3, d.save_game_marker_pos); + FABLE_READ_FIELD(in, "SaveGameMarkerAngleXY", f32, d.save_game_marker_angle_xy); + FABLE_READ_FIELD(in, "GuildSealRecallPos", vector3, d.guild_seal_recall_pos); + FABLE_READ_FIELD(in, "GuildSealRecallAngleXY", f32, d.guild_seal_recall_angle_xy); + FABLE_READ_FIELD(in, "CurrentRegionName", cstring, d.current_region_name); + FABLE_READ_FIELD(in, "CurrentRegionMinimapGraphicName", cstring, d.current_region_minimap_graphic_name); + FABLE_READ_FIELD(in, "TotalTimePlayed", f32, d.total_time_played); + return d; +} + +} // namespace + +std::expected SaveHeader::decode(std::span section) +{ + PersistReader outer(section); + const auto body = outer.section("HEADER"); + if (!body) { + return std::unexpected(body.error()); + } + if (outer.remaining() != 0) { + return std::unexpected(PersistError::bad_section); + } + + PersistReader in(*body); + SaveHeader header; + FABLE_READ_FIELD(in, "WorldName", cstring, header.world_name); + FABLE_READ_FIELD(in, "WorldFrame", i32, header.world_frame); + if (in.remaining() != 0) { + auto details = read_details(in); + if (!details) { + return std::unexpected(details.error()); + } + header.details = std::move(*details); + } + if (in.remaining() != 0) { + return std::unexpected(PersistError::bad_section); + } + return header; +} + +#undef FABLE_READ_FIELD + +std::vector SaveHeader::encode() const +{ + PersistWriter out; + out.tag("WorldName"); + out.cstring(world_name); + out.tag("WorldFrame"); + out.i32(world_frame); + if (details) { + const Details& d = *details; + out.tag("TeleportingEnabled"); + out.boolean(d.teleporting_enabled); + out.tag("SavingEnabled"); + out.boolean(d.saving_enabled); + out.tag("ExperienceSpendingEnabled"); + out.boolean(d.experience_spending_enabled); + out.tag("CreatureGenerationEnabled"); + out.boolean(d.creature_generation_enabled); + out.tag("CreatureGenerationDisabledGroups"); + out.i32(d.creature_generation_disabled_groups); + out.tag("HeroSleepingEnabled"); + out.boolean(d.hero_sleeping_enabled); + out.tag("MapTableShowQuestCardsOnUsed"); + out.boolean(d.map_table_show_quest_cards_on_used); + out.tag("MiniMapEnabled"); + out.boolean(d.mini_map_enabled); + out.tag("MiniMapActiveBeforeDisabled"); + out.boolean(d.mini_map_active_before_disabled); + out.tag("GuildMasterMessagesEnabled"); + out.boolean(d.guild_master_messages_enabled); + out.tag("SummonerDeathExplosionAffectsHero"); + out.boolean(d.summoner_death_explosion_affects_hero); + out.tag("MostRecentSaveType"); + out.i32(d.most_recent_save_type); + out.tag("MostRecentSaveTypeBeforeManualSave"); + out.i32(d.most_recent_save_type_before_manual_save); + out.tag("MostRecentManualSaveName"); + out.wstring(d.most_recent_manual_save_name); + out.tag("SaveGameMarkerPos"); + out.vector3(d.save_game_marker_pos); + out.tag("SaveGameMarkerAngleXY"); + out.f32(d.save_game_marker_angle_xy); + out.tag("GuildSealRecallPos"); + out.vector3(d.guild_seal_recall_pos); + out.tag("GuildSealRecallAngleXY"); + out.f32(d.guild_seal_recall_angle_xy); + out.tag("CurrentRegionName"); + out.cstring(d.current_region_name); + out.tag("CurrentRegionMinimapGraphicName"); + out.cstring(d.current_region_minimap_graphic_name); + out.tag("TotalTimePlayed"); + out.f32(d.total_time_played); + } + PersistWriter section; + section.section("HEADER", out.bytes()); + return section.take(); +} + +} // namespace fable::save diff --git a/rebuild/modern/save/src/TextSections.cpp b/rebuild/modern/save/src/TextSections.cpp new file mode 100644 index 0000000000..030cc2113e --- /dev/null +++ b/rebuild/modern/save/src/TextSections.cpp @@ -0,0 +1,222 @@ +#include + +#include "Lzo.hpp" + +#include +#include +#include + +namespace fable::save { +namespace { + +constexpr std::size_t raw_tail = 3; +constexpr std::uint32_t max_short_clen = 0x7FFF; + +std::span as_bytes(std::string_view text) +{ + return {reinterpret_cast(text.data()), text.size()}; +} + +std::string as_text(std::span bytes) +{ + std::string out(bytes.size(), '\0'); + std::transform(bytes.begin(), bytes.end(), out.begin(), [](std::byte b) { return static_cast(b); }); + return out; +} + +void write_stored(PersistWriter& out, std::string_view text) +{ + out.u32(static_cast(text.size())); + out.bytes_raw(std::array{std::byte{0}, std::byte{0}}); + out.bytes_raw(as_bytes(text)); +} + +} // namespace + +bool lzo_compressor_available() noexcept +{ + return detail::lzo_compressor_available(); +} + +std::expected read_lz_text(PersistReader& in) +{ + const auto size = in.u32(); + if (!size) { + return std::unexpected(size.error()); + } + if (*size == 0) { + return std::string(); + } + const auto short_clen = in.raw(2); + if (!short_clen) { + return std::unexpected(short_clen.error()); + } + const auto clen16 = static_cast(std::to_integer((*short_clen)[0]) + | std::to_integer((*short_clen)[1]) << 8); + std::int64_t clen = clen16; + if (clen16 == -1) { + const auto wide = in.u32(); + if (!wide) { + return std::unexpected(wide.error()); + } + clen = *wide; + } + if (clen <= 0) { + const auto raw = in.raw(*size); + if (!raw) { + return std::unexpected(raw.error()); + } + return as_text(*raw); + } + if (*size <= raw_tail) { + return std::unexpected(PersistError::bad_section); + } + const auto stream = in.raw(static_cast(clen)); + const auto tail = in.raw(raw_tail); + if (!stream || !tail) { + return std::unexpected(PersistError::truncated); + } + auto head = detail::lzo1x_decompress(*stream, *size - raw_tail); + if (!head) { + return std::unexpected(PersistError::bad_section); + } + std::string text = as_text(*head); + text += as_text(*tail); + return text; +} + +void write_lz_text(PersistWriter& out, std::string_view text) +{ + if (text.empty()) { + out.u32(0); + return; + } + if (text.size() <= raw_tail) { + write_stored(out, text); + return; + } + const std::string_view head = text.substr(0, text.size() - raw_tail); + const auto stream = detail::lzo1x_999_compress(as_bytes(head)); + if (!stream || stream->size() >= head.size()) { + write_stored(out, text); + return; + } + out.u32(static_cast(text.size())); + if (stream->size() <= max_short_clen) { + const auto clen = static_cast(stream->size()); + out.bytes_raw(std::array{static_cast(clen & 0xFFU), static_cast(clen >> 8)}); + } else { + out.bytes_raw(std::array{std::byte{0xFF}, std::byte{0xFF}}); + out.u32(static_cast(stream->size())); + } + out.bytes_raw(*stream); + out.bytes_raw(as_bytes(text.substr(text.size() - raw_tail))); +} + +std::expected QuestsSection::decode(std::span section) +{ + PersistReader in(section); + auto text = read_lz_text(in); + if (!text) { + return std::unexpected(text.error()); + } + QuestsSection out; + out.text = std::move(*text); + if (const auto tag = in.tag("NumAcceptedBoasts"); !tag) { + return std::unexpected(tag.error()); + } + const auto count = in.u32(); + if (!count) { + return std::unexpected(count.error()); + } + if (*count > in.remaining() / 19) { // 4 tags, a NUL, a u32 and 2 bools per boast + return std::unexpected(PersistError::bad_section); + } + for (std::uint32_t i = 0; i < *count; ++i) { + const std::string index = std::to_string(i); + AcceptedBoast boast; + auto name = in.tag("BoastScriptName" + index).and_then([&] { return in.cstring(); }); + auto id = in.tag("BoastId" + index).and_then([&] { return in.u32(); }); + auto failed = in.tag("BoastFailed" + index).and_then([&] { return in.boolean(); }); + auto completed = in.tag("BoastCompleted" + index).and_then([&] { return in.boolean(); }); + if (!name || !id || !failed || !completed) { + return std::unexpected(!name ? name.error() : !id ? id.error() : !failed ? failed.error() : completed.error()); + } + boast.script_name = std::move(*name); + boast.id = *id; + boast.failed = *failed; + boast.completed = *completed; + out.accepted_boasts.push_back(std::move(boast)); + } + if (in.remaining() != 0) { + return std::unexpected(PersistError::bad_section); + } + return out; +} + +std::vector QuestsSection::encode() const +{ + PersistWriter out; + write_lz_text(out, text); + out.tag("NumAcceptedBoasts"); + out.u32(static_cast(accepted_boasts.size())); + for (std::size_t i = 0; i < accepted_boasts.size(); ++i) { + const std::string index = std::to_string(i); + const AcceptedBoast& boast = accepted_boasts[i]; + out.tag("BoastScriptName" + index); + out.cstring(boast.script_name); + out.tag("BoastId" + index); + out.u32(boast.id); + out.tag("BoastFailed" + index); + out.boolean(boast.failed); + out.tag("BoastCompleted" + index); + out.boolean(boast.completed); + } + return out.take(); +} + +std::expected FactionsSection::decode(std::span section) +{ + PersistReader in(section); + if (const auto tag = in.tag("NumberOfFactions"); !tag) { + return std::unexpected(tag.error()); + } + const auto count = in.u32(); + if (!count) { + return std::unexpected(count.error()); + } + // Each record is at least a tag and a size. + if (*count > in.remaining() / 8) { + return std::unexpected(PersistError::bad_section); + } + FactionsSection out; + out.factions.reserve(*count); + for (std::uint32_t i = 0; i < *count; ++i) { + if (const auto tag = in.tag("Faction" + std::to_string(i)); !tag) { + return std::unexpected(tag.error()); + } + auto text = read_lz_text(in); + if (!text) { + return std::unexpected(text.error()); + } + out.factions.push_back(std::move(*text)); + } + if (in.remaining() != 0) { + return std::unexpected(PersistError::bad_section); + } + return out; +} + +std::vector FactionsSection::encode() const +{ + PersistWriter out; + out.tag("NumberOfFactions"); + out.u32(static_cast(factions.size())); + for (std::size_t i = 0; i < factions.size(); ++i) { + out.tag("Faction" + std::to_string(i)); + write_lz_text(out, factions[i]); + } + return out.take(); +} + +} // namespace fable::save diff --git a/rebuild/modern/save/src/Zlib.cpp b/rebuild/modern/save/src/Zlib.cpp new file mode 100644 index 0000000000..acf188d7cb --- /dev/null +++ b/rebuild/modern/save/src/Zlib.cpp @@ -0,0 +1,63 @@ +#include "Zlib.hpp" + +#include + +#include + +namespace fable::save::detail { + +std::expected, ZlibError> inflate_exact( + std::span compressed, std::uint32_t expected_size) +{ + if (compressed.size() > std::numeric_limits::max()) { + return std::unexpected(ZlibError::too_large); + } + std::vector out(expected_size); + z_stream stream{}; + if (inflateInit(&stream) != Z_OK) { + return std::unexpected(ZlibError::inflate_failed); + } + stream.next_in = reinterpret_cast(const_cast(compressed.data())); + stream.avail_in = static_cast(compressed.size()); + stream.next_out = reinterpret_cast(out.data()); + stream.avail_out = static_cast(out.size()); + const int result = inflate(&stream, Z_FINISH); + const auto produced = stream.total_out; + const auto space_left = stream.avail_out; + inflateEnd(&stream); + if (result == Z_BUF_ERROR && space_left == 0) { + return std::unexpected(ZlibError::size_mismatch); // more output than promised + } + if (result != Z_STREAM_END) { + return std::unexpected(ZlibError::inflate_failed); + } + if (produced != expected_size) { + return std::unexpected(ZlibError::size_mismatch); + } + return out; +} + +std::expected, ZlibError> deflate_retail(std::span data, int level) +{ + if (data.size() > std::numeric_limits::max()) { + return std::unexpected(ZlibError::too_large); + } + z_stream stream{}; + if (deflateInit2(&stream, level, Z_DEFLATED, 15, 8, Z_DEFAULT_STRATEGY) != Z_OK) { + return std::unexpected(ZlibError::deflate_failed); + } + std::vector out(deflateBound(&stream, static_cast(data.size()))); + stream.next_in = reinterpret_cast(const_cast(data.data())); + stream.avail_in = static_cast(data.size()); + stream.next_out = reinterpret_cast(out.data()); + stream.avail_out = static_cast(out.size()); + const int result = deflate(&stream, Z_FINISH); + out.resize(stream.total_out); + deflateEnd(&stream); + if (result != Z_STREAM_END) { + return std::unexpected(ZlibError::deflate_failed); + } + return out; +} + +} // namespace fable::save::detail diff --git a/rebuild/modern/save/src/Zlib.hpp b/rebuild/modern/save/src/Zlib.hpp new file mode 100644 index 0000000000..57caab86b0 --- /dev/null +++ b/rebuild/modern/save/src/Zlib.hpp @@ -0,0 +1,28 @@ +#pragma once + +// Internal: the two zlib operations the save format needs. Retail drives zlib 1.2.1 through +// CCompressorZlib with a 15-bit window and memLevel 8; only the level differs by layer +// (9 for the container chunks, 1 for SAVED_ENTITIES cells). + +#include +#include +#include +#include +#include + +namespace fable::save::detail { + +enum class ZlibError { + inflate_failed, + size_mismatch, // the stream inflates to a different size than its header field says + deflate_failed, + too_large, +}; + +[[nodiscard]] std::expected, ZlibError> inflate_exact( + std::span compressed, std::uint32_t expected_size); + +[[nodiscard]] std::expected, ZlibError> deflate_retail( + std::span data, int level); + +} // namespace fable::save::detail diff --git a/rebuild/modern/save/tests/SaveContainerTests.cpp b/rebuild/modern/save/tests/SaveContainerTests.cpp new file mode 100644 index 0000000000..baecf5beb8 --- /dev/null +++ b/rebuild/modern/save/tests/SaveContainerTests.cpp @@ -0,0 +1,393 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using fable::save::ccrc; +using fable::save::SaveContainer; +using fable::save::SaveContainerCodec; +using fable::save::SaveContainerError; + +int failures = 0; + +void expect(const bool condition, const std::string_view message) +{ + if (!condition) { + std::cerr << "FAIL: " << message << '\n'; + ++failures; + } +} + +void expect_error(const auto& result, const SaveContainerError error, const std::string_view message) +{ + expect(!result && result.error() == error, message); +} + +std::vector bytes(const std::string_view text) +{ + std::vector out; + for (const char c : text) { + out.push_back(static_cast(c)); + } + return out; +} + +SaveContainer sample() +{ + // Shaped like a retail stream: repetitive tagged fields, so deflate has matches to find. + std::vector body; + for (std::uint32_t i = 0; i < 4000; ++i) { + body.push_back(static_cast(i & 0xFFU)); + body.push_back(static_cast((i * 7U) & 0x3FU)); + body.push_back(std::byte{0x2A}); + } + return SaveContainer{bytes("HEADER section"), body}; +} + +std::uint32_t u32_at(std::span file, std::size_t offset) +{ + return static_cast(file[offset]) | static_cast(file[offset + 1]) << 8 + | static_cast(file[offset + 2]) << 16 | static_cast(file[offset + 3]) << 24; +} + +void crc_matches_retail_ccrc() +{ + // Seed 0, no inversion: "123456789" is the classic check input; zlib's crc32 gives + // 0xCBF43926 with pre/post inversion, so the raw seed-0 form differs from it. + expect(ccrc(bytes("123456789")) == 0x2DFD2D88U, "seed-0 CRC of the check string"); + expect(ccrc(bytes("")) == 0U, "empty input keeps the seed"); + expect(ccrc(bytes("a"), 0xFFFFFFFFU) != ccrc(bytes("a")), "seed is honoured"); +} + +void encode_writes_retail_layout() +{ + const auto file = SaveContainerCodec::encode(sample()); + expect(file.has_value(), "sample encodes"); + if (!file) { + return; + } + expect(file->size() == SaveContainerCodec::slab_size, "file is the 0x4B000 slab"); + expect(std::string_view(reinterpret_cast(file->data()), 10) == "FableSave!", "magic"); + const std::uint32_t total = u32_at(*file, 0x0C); + expect(u32_at(*file, 0x10) == sample().header_section.size(), "header size field"); + expect(u32_at(*file, 0x14) == sample().body_section.size(), "body size field"); + expect(std::to_integer((*file)[0x1C]) == 0x78 && std::to_integer((*file)[0x1D]) == 0xDA, + "level-9 zlib stream header 78 DA"); + expect(u32_at(*file, total) == ccrc(std::span(*file).first(total)), "trailer CRC"); +} + +void round_trip_is_exact() +{ + const auto file = SaveContainerCodec::encode(sample()); + const auto decoded = file ? SaveContainerCodec::decode(*file) : std::unexpected(SaveContainerError::deflate_failed); + expect(decoded.has_value() && *decoded == sample(), "decode(encode(x)) == x"); + if (decoded) { + const auto again = SaveContainerCodec::encode(*decoded); + expect(again.has_value() && *again == *file, "encode is deterministic"); + } +} + +void retail_gates_are_enforced() +{ + auto file = *SaveContainerCodec::encode(sample()); + const std::uint32_t total = u32_at(file, 0x0C); + + auto bad = file; + bad[3] = std::byte{'X'}; + expect_error(SaveContainerCodec::decode(bad), SaveContainerError::bad_magic, "magic is checked"); + + bad = file; + bad[0x40] ^= std::byte{1}; + expect_error(SaveContainerCodec::decode(bad), SaveContainerError::bad_signature, "CRC covers the data"); + + bad = file; + bad.push_back(std::byte{0}); + expect_error(SaveContainerCodec::decode(bad), SaveContainerError::bad_slab_size, "exact slab size"); + + bad = file; + bad[total + 8] = std::byte{1}; + expect_error(SaveContainerCodec::decode(bad), SaveContainerError::nonzero_padding, "zero padding"); + + bad = file; + bad[0x0C] = std::byte{0xFF}; + bad[0x0D] = std::byte{0xFF}; + bad[0x0E] = std::byte{0xFF}; + expect_error(SaveContainerCodec::decode(bad), SaveContainerError::data_length_out_of_range, + "total_data_len bounds"); + + expect_error(SaveContainerCodec::decode(std::span(file).first(8)), SaveContainerError::truncated_header, + "short input"); +} + +void inflated_sizes_are_checked() +{ + auto file = *SaveContainerCodec::encode(sample()); + const std::uint32_t total = u32_at(file, 0x0C); + // Claim a larger body, then re-sign so only the size check can catch it. + file[0x14] = static_cast(std::to_integer(file[0x14]) + 1); + const std::uint32_t crc = ccrc(std::span(file).first(total)); + for (int i = 0; i < 4; ++i) { + file[total + static_cast(i)] = static_cast((crc >> (8 * i)) & 0xFFU); + } + expect_error(SaveContainerCodec::decode(file), SaveContainerError::inflated_size_mismatch, + "header size fields must match the streams"); +} + +void oversized_saves_are_refused() +{ + // Incompressible data larger than the slab cannot be written as a retail save. + std::vector noise(SaveContainerCodec::slab_size); + std::uint32_t x = 0x12345678U; + for (auto& b : noise) { + x = x * 1664525U + 1013904223U; + b = static_cast(x >> 24); + } + expect_error(SaveContainerCodec::encode(SaveContainer{{}, noise}), SaveContainerError::too_large, + "slab overflow is an error, not a truncated file"); +} + +void header_round_trips() +{ + using fable::save::PersistError; + using fable::save::SaveHeader; + + SaveHeader full; + full.world_name = "FinalAlbion"; + full.world_frame = 138230; + SaveHeader::Details d; + d.saving_enabled = true; + d.mini_map_enabled = true; + d.most_recent_save_type = 1; + d.most_recent_manual_save_name = u"Manual - Save1"; + d.save_game_marker_pos = {3114.79F, 3000.5F, -12.25F}; + d.current_region_name = "TXT_REGION_GREATWOOD_TELEPORT"; + d.current_region_minimap_graphic_name = "MINIMAP_GREATWOOD"; + full.details = d; + + const auto bytes = full.encode(); + expect(std::string_view(reinterpret_cast(bytes.data()), 7) == std::string_view("HEADER\0", 7), + "section name"); + // First field tag right after the 4-byte section length: WorldName = a8de4f22 (SAVE.md §3.4). + expect(u32_at(bytes, 11) == 0xA8DE4F22U, "WorldName tag"); + const auto back = SaveHeader::decode(bytes); + expect(back.has_value() && *back == full, "full header round-trips"); + + SaveHeader minimal{"FinalAlbion", 740, std::nullopt}; + const auto small = SaveHeader::decode(minimal.encode()); + expect(small.has_value() && *small == minimal, "minimal-context header (fields 0-1 only)"); + + auto broken = bytes; + broken[11] ^= std::byte{1}; + const auto bad = SaveHeader::decode(broken); + expect(!bad && bad.error() == PersistError::tag_mismatch, "tags are checked like retail CheckCRC"); +} + +void body_round_trips() +{ + using fable::save::PersistError; + using fable::save::PlayerSection; + using fable::save::SaveBody; + + const PlayerSection player{0xFFFFFF0000003AE3ULL, "Greatwood"}; + const auto player_bytes = player.encode(); + expect(u32_at(player_bytes, 0) == 0x96ADC827U, "PlayerCharacterUID tag (SAVE.md §3.5)"); + const auto player_back = PlayerSection::decode(player_bytes); + expect(player_back.has_value() && *player_back == player, "PLAYER round-trips"); + + SaveBody full; + full.clock = SaveBody::GameClock{0.393F, 0.000666F, 5}; + full.sections = {{"ENTITIES", bytes("entity stream")}, {"PLAYER", player_bytes}, + {"QUESTS", bytes("q")}, {"REGIONS", {}}, {"FACTIONS", bytes("f")}}; + const auto encoded = full.encode(); + expect(u32_at(encoded, 0) == 0xEEE2E867U, "body opens with the Time tag"); + const auto back = SaveBody::decode(encoded); + expect(back.has_value() && *back == full, "full body round-trips"); + expect(back && back->find("PLAYER") != nullptr && back->find("PLAYER")->bytes == player_bytes, "find"); + + SaveBody minimal; + minimal.sections = {{"ENTITIES", bytes("only")}}; + const auto small = SaveBody::decode(minimal.encode()); + expect(small.has_value() && *small == minimal && !small->clock, "minimal body has no clock"); + expect(full.has_retail_shape() && minimal.has_retail_shape(), "both writer shapes are retail"); + SaveBody renamed = full; + renamed.sections[1].name = "LAYER"; // what a one-byte-short ENTITIES length looks like + expect(!renamed.has_retail_shape(), "unexpected section names are flagged"); + + auto truncated = encoded; + truncated.pop_back(); + const auto bad = SaveBody::decode(truncated); + expect(!bad && bad.error() == PersistError::bad_section, "sections must tile the stream"); +} + +void entities_round_trip() +{ + using fable::save::EntitiesSection; + using fable::save::EntityCell; + using fable::save::SavedEntities; + + SavedEntities saved; + saved.slots.emplace_back(std::nullopt); // empty slot: u64 0 + saved.slots.emplace_back(EntityCell{0x27A89, sample().body_section, {}}); + saved.slots.emplace_back(EntityCell{0x1832A3, {}, {}}); // record with no stream + saved.dead_entities.push_back({0x027A89000000045CULL, 0x321ACEE7U}); + + const auto bytes_out = saved.encode(); + expect(bytes_out.has_value(), "SAVED_ENTITIES encodes"); + if (!bytes_out) { + return; + } + expect(u32_at(*bytes_out, 4) == 3, "slot count"); + expect(u32_at(*bytes_out, 12) == 0 && u32_at(*bytes_out, 16) == 0, "empty slot is u64 0"); + const std::uint32_t clen = u32_at(*bytes_out, 36); + expect(u32_at(*bytes_out, 20) == 29 + clen, "record length = 29 + compressed length"); + expect(std::to_integer((*bytes_out)[48]) == 0x78 && std::to_integer((*bytes_out)[49]) == 0x01, + "cells are zlib level 1 (78 01)"); + const auto back = SavedEntities::decode(*bytes_out); + expect(back.has_value() && *back == saved, "SAVED_ENTITIES round-trips"); + + fable::save::SavedNpcNames names; + names.entries.emplace(); + names.entries->push_back({0x3020DF0000002903ULL, "", u"Brom"}); + names.entries->push_back({0x3020DF000000290BULL, "GROUP", u"Theresa"}); + const auto names_bytes = names.encode(); + expect(u32_at(names_bytes, 0) == 2 && u32_at(names_bytes, 12) == 10, + "an empty NPC group is written as the 10-byte UNATTACHED"); + const auto names_back = fable::save::SavedNpcNames::decode(names_bytes); + expect(names_back.has_value() && *names_back == names, "SAVED_NPC_NAMES round-trips"); + expect(fable::save::SavedNpcNames{}.encode().empty(), "a minimal-context NPC name section is empty"); + + EntitiesSection entities{fable::save::EntitiesHead{2, 0xDE}, saved, names}; + const auto section = entities.encode(); + const auto entities_back = section ? EntitiesSection::decode(*section) : std::unexpected(fable::save::PersistError::truncated); + expect(entities_back.has_value() && *entities_back == entities, "ENTITIES round-trips with its head"); +} + +void text_sections_round_trip() +{ + using fable::save::PersistReader; + using fable::save::PersistWriter; + + // A hand-assembled LZO1X stream: initial run "abc" (0x14), an M2 match of 8 at distance 3 + // (0xE8 0x00), end of stream (0x11 0x00 0x00). Retail frames it as size, i16 clen, stream, + // then the last 3 bytes raw. + const auto blob = bytes(std::string_view("\x0E\x00\x00\x00\x09\x00\x14" "abc\xE8\x00\x11\x00\x00" "XYZ", 18)); + PersistReader in(blob); + const auto text = fable::save::read_lz_text(in); + expect(text.has_value() && *text == "abcabcabcabXYZ" && in.remaining() == 0, "LZO1X blob decodes"); + + std::string noise; + std::uint32_t seed = 12345; + for (int i = 0; i < 70000; ++i) { + seed = seed * 1103515245U + 12345U; + noise.push_back(static_cast('a' + (seed >> 16) % 26)); + } + for (const std::string& sample : {std::string(), std::string("ab"), std::string("abc"), + std::string("Name \"FACTION_HERO\";\nNumberOfAllies 0;\nEndFaction\n"), + std::string(5000, 'q'), noise}) { + PersistWriter out; + fable::save::write_lz_text(out, sample); + const auto written = out.take(); + PersistReader back(written); + const auto again = fable::save::read_lz_text(back); + expect(again.has_value() && *again == sample && back.remaining() == 0, "LZ text round-trips"); + } + + fable::save::FactionsSection factions{{"Name \"FACTION_HERO\";\nEndFaction\n", ""}}; + expect(fable::save::field_tag("NumberOfFactions") == 0x4F0E45C3U, "FACTIONS head tag is NumberOfFactions"); + const auto factions_back = fable::save::FactionsSection::decode(factions.encode()); + expect(factions_back.has_value() && *factions_back == factions, "FACTIONS round-trips"); + fable::save::QuestsSection quests{"START_QUEST_CARDS\nEND_QUEST_CARDS\n", + {{"Q_TraderEscort", 3, false, false}, {"Q_TraderEscort", 1, true, false}}}; + const auto quests_bytes = quests.encode(); + const auto quests_back = fable::save::QuestsSection::decode(quests_bytes); + expect(quests_back.has_value() && *quests_back == quests, "QUESTS round-trips with its boasts"); + // Retail Manual-Save1 starts its tail with the NumAcceptedBoasts tag 0xA4D9A09F. + expect(fable::save::field_tag("NumAcceptedBoasts") == 0xA4D9A09FU, "boast tags are seed-0 field tags"); +} + +void regions_round_trip() +{ + fable::save::RegionsSection regions{{{"LookoutPoint", false, true, true}, {"PicnicArea", true, false, true}}}; + const auto out = regions.encode(); + expect(u32_at(out, 0) == 0 && u32_at(out, 4) == 3, "REGIONS: tag(\"\") is 0 and the table size counts element 0"); + expect(out.size() == 8 + 2 * 20 + 12 + 10, "REGIONS: 20 bytes per record beyond the name"); + const auto back = fable::save::RegionsSection::decode(out); + expect(back.has_value() && *back == regions, "REGIONS round-trips"); +} + +void profile_round_trips() +{ + using fable::save::Profile; + using fable::save::ProfileError; + + std::string text = "ProfileName \"Cornelio\";\r\nSubtitlingEnabled TRUE;\r\n"; + for (int slot = 1; slot <= 50; ++slot) { + text += "SaveGameNames" + std::to_string(slot) + (slot == 1 ? " \"Manual - Save1\";\r\n" : " \"\";\r\n"); + } + std::vector file = bytes("FablePro"); + file.resize(16, std::byte{0}); + const auto body = bytes(text); + file.insert(file.end(), body.begin(), body.end()); + const auto total = static_cast(file.size()); + for (int i = 0; i < 4; ++i) { + file[12 + static_cast(i)] = static_cast((total >> (8 * i)) & 0xFFU); + } + const std::uint32_t crc = ccrc(file); + for (int i = 0; i < 4; ++i) { + file.push_back(static_cast((crc >> (8 * i)) & 0xFFU)); + } + + auto profile = Profile::decode(file); + expect(profile.has_value(), "profile decodes"); + if (!profile) { + return; + } + expect(profile->encode() == file, "profile re-encodes byte for byte"); + expect(profile->value("SubtitlingEnabled") == "TRUE", "raw values"); + const auto names = profile->save_game_names(); + expect(names.size() == 50 && names[0] == "Manual - Save1" && names[1].empty(), "save registry"); + + expect(profile->set_value("SaveGameNames2", "\"Manual - Save2\""), "registry edit"); + const auto edited = Profile::decode(profile->encode()); + expect(edited && edited->save_game_names()[1] == "Manual - Save2", "edited profile re-signs and reloads"); + + auto broken = file; + broken[20] ^= std::byte{1}; + const auto bad = Profile::decode(broken); + expect(!bad && bad.error() == ProfileError::bad_signature, "profile trailer is enforced"); +} + +} // namespace + +int main() +{ + profile_round_trips(); + header_round_trips(); + body_round_trips(); + entities_round_trip(); + text_sections_round_trip(); + regions_round_trip(); + crc_matches_retail_ccrc(); + encode_writes_retail_layout(); + round_trip_is_exact(); + retail_gates_are_enforced(); + inflated_sizes_are_checked(); + oversized_saves_are_refused(); + if (failures == 0) { + std::cout << "fable_save: all tests passed\n"; + } + return failures == 0 ? 0 : 1; +} diff --git a/rebuild/modern/save/tools/fable_save_check.cpp b/rebuild/modern/save/tools/fable_save_check.cpp new file mode 100644 index 0000000000..f2ccd85d9b --- /dev/null +++ b/rebuild/modern/save/tools/fable_save_check.cpp @@ -0,0 +1,220 @@ +// fable_save_check ... +// +// For each `FableSave!` file: decode with the retail gates, re-encode, and report whether the +// rewrite is byte-identical to the original. `FablePro` profiles get the same check +// (`profile:exact`). Directories are walked recursively; files with other magics are skipped. +// Exit status 1 if any file fails to decode or re-encode. +// +// exact decode passed and encode(decode(file)) == file +// sections decode passed, sections round-trip, but the compressed bytes differ (the file was +// written by a different deflate, e.g. zlib-ng in Python tools) +// rejected decode failed (retail would refuse to load it) +// +// The HEADER section (chunk0), the body's section list (chunk1), PLAYER and ENTITIES (down to +// the inflated SAVED_ENTITIES cells, which are re-deflated) are also decoded and re-encoded; a +// save where one of them does not reproduce byte for byte gets a `header:`, `body:`, `player:` +// or `entities:` verdict instead. +// +// REGIONS is decoded and must re-encode exactly. QUESTS and FACTIONS (LZO text blobs) are decoded too. A build with FABLE_SAVE_WITH_LZO must +// re-encode them byte for byte; without LZO they re-encode in the stored form, and the check +// is that the stored form decodes back to the same text (`quests:` / `factions:` verdicts). + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using fable::save::SaveContainerCodec; + +// "" when the section is absent or re-encodes byte for byte, else the verdict. +template +std::string exact_section_verdict(const fable::save::SaveBody& body, std::string_view name, std::string_view prefix) +{ + const auto* raw = body.find(name); + if (raw == nullptr) { + return ""; + } + const auto decoded = Section::decode(raw->bytes); + if (!decoded) { + return std::string(prefix) + "decode_error_" + std::to_string(static_cast(decoded.error())); + } + return decoded->encode() == raw->bytes ? "" : std::string(prefix) + "encode_mismatch"; +} + +// "" when the text section is absent or re-encodes as this build should, else the verdict. +template +std::string text_section_verdict(const fable::save::SaveBody& body, std::string_view name, std::string_view prefix) +{ + const auto* raw = body.find(name); + if (raw == nullptr) { + return ""; + } + const auto decoded = Section::decode(raw->bytes); + if (!decoded) { + return std::string(prefix) + "decode_error_" + std::to_string(static_cast(decoded.error())); + } + const auto encoded = decoded->encode(); + if (fable::save::lzo_compressor_available() && encoded == raw->bytes) { + return ""; + } + const auto again = Section::decode(encoded); + if (!again || *again != *decoded) { + return std::string(prefix) + "round_trip_mismatch"; + } + // Same text, different compressed bytes: not written by retail's lzo1x_999 (e.g. the Python + // tools' store-mode streams), like a whole-file `sections` verdict. + return fable::save::lzo_compressor_available() ? "sections" : ""; +} + +using fable::save::SaveContainerError; + +std::string_view error_name(SaveContainerError error) +{ + switch (error) { + case SaveContainerError::bad_magic: return "bad_magic"; + case SaveContainerError::truncated_header: return "truncated_header"; + case SaveContainerError::data_length_out_of_range: return "data_length_out_of_range"; + case SaveContainerError::bad_signature: return "bad_signature"; + case SaveContainerError::bad_slab_size: return "bad_slab_size"; + case SaveContainerError::nonzero_padding: return "nonzero_padding"; + case SaveContainerError::chunk_out_of_bounds: return "chunk_out_of_bounds"; + case SaveContainerError::chunk_layout_mismatch: return "chunk_layout_mismatch"; + case SaveContainerError::inflate_failed: return "inflate_failed"; + case SaveContainerError::inflated_size_mismatch: return "inflated_size_mismatch"; + case SaveContainerError::deflate_failed: return "deflate_failed"; + case SaveContainerError::too_large: return "too_large"; + } + return "unknown"; +} + +std::vector read_file(const std::filesystem::path& path) +{ + std::ifstream in(path, std::ios::binary); + const std::vector raw{std::istreambuf_iterator(in), std::istreambuf_iterator()}; + std::vector out(raw.size()); + for (std::size_t i = 0; i < raw.size(); ++i) { + out[i] = static_cast(raw[i]); + } + return out; +} + +bool has_magic(const std::vector& file, std::string_view magic) +{ + if (file.size() < magic.size()) { + return false; + } + for (std::size_t i = 0; i < magic.size(); ++i) { + if (file[i] != static_cast(magic[i])) { + return false; + } + } + return true; +} + +} // namespace + +int main(int argc, char** argv) +{ + std::vector files; + for (int i = 1; i < argc; ++i) { + const std::filesystem::path arg = argv[i]; + if (std::filesystem::is_directory(arg)) { + for (const auto& entry : std::filesystem::recursive_directory_iterator(arg)) { + if (entry.is_regular_file()) { + files.push_back(entry.path()); + } + } + } else { + files.push_back(arg); + } + } + + std::map totals; + for (const auto& path : files) { + const auto file = read_file(path); + if (has_magic(file, "FablePro")) { + const auto profile = fable::save::Profile::decode(file); + const std::string verdict = !profile ? "profile:rejected_" + std::to_string(static_cast(profile.error())) + : profile->encode() == file ? "profile:exact" + : "profile:encode_mismatch"; + ++totals[verdict]; + if (verdict != "profile:exact") { + std::cout << verdict << '\t' << path.string() << '\n'; + } + continue; + } + if (!has_magic(file, "FableSave!")) { + continue; + } + const auto decoded = SaveContainerCodec::decode(file); + std::string verdict; + if (!decoded) { + verdict = "rejected:" + std::string(error_name(decoded.error())); + } else if (const auto header = fable::save::SaveHeader::decode(decoded->header_section); !header) { + verdict = "header:decode_error_" + std::to_string(static_cast(header.error())); + } else if (header->encode() != decoded->header_section) { + verdict = "header:encode_mismatch"; + } else if (const auto body = fable::save::SaveBody::decode(decoded->body_section); !body) { + verdict = "body:decode_error_" + std::to_string(static_cast(body.error())); + } else if (body->encode() != decoded->body_section) { + verdict = "body:encode_mismatch"; + } else if (!body->has_retail_shape()) { + verdict = "body:unexpected_sections"; + } else if (const auto* player = body->find("PLAYER"); + player != nullptr && !fable::save::PlayerSection::decode(player->bytes)) { + verdict = "player:decode_error"; + } else if (player != nullptr && fable::save::PlayerSection::decode(player->bytes)->encode() != player->bytes) { + verdict = "player:encode_mismatch"; + } else if (const auto entities = fable::save::EntitiesSection::decode(body->find("ENTITIES")->bytes); + !entities) { + verdict = "entities:decode_error_" + std::to_string(static_cast(entities.error())); + } else if (const auto again = entities->encode(); !again || *again != body->find("ENTITIES")->bytes) { + verdict = "entities:encode_mismatch"; + } else if (verdict = exact_section_verdict(*body, "REGIONS", "regions:"); + !verdict.empty()) { + } else if (verdict = text_section_verdict(*body, "QUESTS", "quests:"); + !verdict.empty()) { + } else if (verdict = text_section_verdict(*body, "FACTIONS", "factions:"); + !verdict.empty()) { + } else { + const auto encoded = SaveContainerCodec::encode(*decoded); + if (!encoded) { + verdict = "encode_failed:" + std::string(error_name(encoded.error())); + } else if (*encoded == file) { + verdict = "exact"; + } else { + const auto again = SaveContainerCodec::decode(*encoded); + verdict = again && *again == *decoded ? "sections" : "encode_mismatch"; + } + } + ++totals[verdict]; + if (verdict != "exact") { + std::cout << verdict << '\t' << path.string() << '\n'; + } + } + std::cout << "text blobs: " << (fable::save::lzo_compressor_available() ? "LZO byte-exact" : "stored form (no LZO)") + << '\n'; + int rejected = 0; + for (const auto& [verdict, count] : totals) { + std::cout << count << '\t' << verdict << '\n'; + if (verdict != "exact" && verdict != "sections" && verdict != "profile:exact") { + rejected += count; + } + } + return rejected == 0 ? 0 : 1; +} From 128b916a2c48f261d065e59a59bea203d5d9de9e Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 17 Oct 2026 14:58:49 -0600 Subject: [PATCH 02/24] tools: add the modernization census, surface inventories and mouse buffer patcher modern_compile_census.py syntax-checks every reconstructed TU with VS 2022 as x86 and x64 and reports readiness by fix and layout drift; platform_surface, persisted_types and the D3D9 census tools inventory what the fork must replace; mouse_buffer_patch.py writes a copy of the Steam exe with a larger DirectInput mouse buffer (2 bytes). Co-Authored-By: Claude Opus 5.5 --- tools/modern/d3d9_device_census.py | 485 ++++++++++++++++++++++++++ tools/modern/d3d9_live_census.py | 172 +++++++++ tools/modern/d3d9_live_report.py | 113 ++++++ tools/modern/modern_compile_census.py | 378 ++++++++++++++++++++ tools/modern/mouse_buffer_patch.py | 84 +++++ tools/modern/persisted_types.py | 252 +++++++++++++ tools/modern/platform_surface.py | 179 ++++++++++ tools/modern/recorder_poke.py | 114 ++++++ tools/modern/rtti_live_globals.py | 94 +++++ 9 files changed, 1871 insertions(+) create mode 100644 tools/modern/d3d9_device_census.py create mode 100644 tools/modern/d3d9_live_census.py create mode 100644 tools/modern/d3d9_live_report.py create mode 100644 tools/modern/modern_compile_census.py create mode 100644 tools/modern/mouse_buffer_patch.py create mode 100644 tools/modern/persisted_types.py create mode 100644 tools/modern/platform_surface.py create mode 100644 tools/modern/recorder_poke.py create mode 100644 tools/modern/rtti_live_globals.py diff --git a/tools/modern/d3d9_device_census.py b/tools/modern/d3d9_device_census.py new file mode 100644 index 0000000000..7331f9bc74 --- /dev/null +++ b/tools/modern/d3d9_device_census.py @@ -0,0 +1,485 @@ +"""Census of Direct3D 9 interface call sites in the retail Fable.exe (sizes the modern RHI). + +Evidence for docs/modern/D3D9_DEVICE_CENSUS.md (+ .json). Static only: linear disassembly +(capstone) of every non-thunk function in ghidra_out/coverage.tsv with a typed register model. + +Types come from the PDB layouts (ghidra_out/struct_layouts_egor.tsv): a register that holds a +pointer to class T, loaded from `[r+d]`, gets the PDB type of T's member at offset d (descending +into embedded members, arrays and the base classes in BASES). Hand seeds are SEED_GLOBALS and +STATIC_INSTANCES; ghidra_out/rtti_live_globals.tsv and the file's RTTI vtables add the rest. Types spread: + + - into callees: a function whose callers all pass `ecx` of type T is a T method (`this`), + - out of globals: `mov [abs], r` with r typed makes abs a typed global, + - out of getters: a tiny `mov eax,[ecx+d] ; ret` method returns the member type. + +A COM call is `call [V+off]` where V = `[P]`, P holds a D3D9 interface pointer, and P (the very same +load) was the last register pushed: the stdcall `this`. Method names come from the interface +declarations in the DirectX SDK d3d9.h. The result is a lower bound: device pointers passed as +arguments, spilled to the stack, or reloaded through untyped objects are not seen. +""" +from __future__ import annotations + +import collections +import json +import re +import sys +from pathlib import Path + +import capstone +from capstone import x86 as cx + +ROOT = Path(__file__).resolve().parents[2] +EXE = Path(r"C:\Programs\Steam\steamapps\common\Fable The Lost Chapters\Fable.exe") +D3D9_H = Path(r"D:\Tools\dxsdk_feb10\include\d3d9.h") +COVERAGE = ROOT / "ghidra_out" / "coverage.tsv" +LAYOUTS = ROOT / "ghidra_out" / "struct_layouts_egor.tsv" +OUT_MD = ROOT / "docs" / "modern" / "D3D9_DEVICE_CENSUS.md" +OUT_JSON = ROOT / "docs" / "modern" / "D3D9_DEVICE_CENSUS.json" + +# 0x1436E18 holds the CRenderManagerCore*: 96 `mov r,[0x1436E18]` loads feed `[r+0x3CF0]` +# (CRenderManagerCore::PD3DDevice in the PDB), and CRenderManagerCore::Init stores the device there. +# 0x13CB508 is the render-state-manager singleton (written by Init, cleared by ~CRenderStateManager) +# that the frontend/2D paths use; live 2026-10-16 it pointed at the render core with the device at +# both +4 and +0x3CF0, while 0x1436E18 (CEngine's copy, set in CEngine::Initialise) was still 0. +# Live RTTI (2026-10-16) names [0x13CB508]'s object CRenderManager. +SEED_GLOBALS = {0x1436E18: "CRenderManagerCore", 0x13CB508: "CRenderManager"} +# Static objects (the address is the object): `mov r, imm` and `mov eax, imm ; ret` getters. +# 0x13CA618 is CSystemManager by live RTTI; getter 0x9A4EC0 returns it, and +# [[0x13CA618+0x60]+0x58] was the live device (CSystemManager::PDisplayManager -> D3DDevice). +STATIC_INSTANCES = {0x13CA618: "CSystemManager"} +# PDB layouts print a base class as `_padding_` ints; these bases are confirmed by size +# (CRenderStateManager is 14908 bytes, exactly the padding run before CRenderManagerCore's own members; +# CRenderManager lists no members of its own before CRenderManagerCore's 15648 bytes). +BASES = {"CRenderManagerCore": "CRenderStateManager", "CRenderManager": "CRenderManagerCore"} + +# Tier B: device members by retail offset, used where PDB typing cannot reach. +# - 0x3CF0 is the only PDB member at that offset in any class (CRenderManagerCore::PD3DDevice). +# - the others apply only inside functions whose name, in Ego_r.pdb, belongs to that class alone. +# CShaderRenderManager::D3DDevice is +0x1E4 in Ego_r.pdb but +0x1D4 in retail: retail shader-manager +# methods (ApplyVertexShader, SetUserVertexShaderConstant, UpdatePixelShader, ...) push [this+0x1D4] +# as the COM `this` of SetVertexShaderConstantF / SetPixelShader / SetStreamSource calls. +UNIQUE_DEVICE_OFFSETS = {0x3CF0} +CLASS_DEVICE_OFFSETS = {"CRenderManagerCore": {0x4, 0x3CF0}, "CRenderStateManager": {0x4}, + "CDisplayManager": {0x58}, "CShaderRenderManager": {0x1D4}} +EGOR_NAMES = ROOT / "ghidra_out" / "egor_pdb_names.tsv" +RTTI_GLOBALS = ROOT / "ghidra_out" / "rtti_live_globals.tsv" + +REGS32 = ("eax", "ebx", "ecx", "edx", "esi", "edi", "ebp") +VOLATILE = ("eax", "ecx", "edx") +ITERATIONS = 10 + + +def d3d9_interfaces() -> dict[str, list[str]]: + text = D3D9_H.read_text(errors="replace") + out = {} + for m in re.finditer(r"DECLARE_INTERFACE_\((IDirect3D\w+),\s*\w+\)", text): + body = text[m.end():text.index("};", m.end())] + out[m.group(1)] = re.findall(r"STDMETHOD(?:_\([^,]+,\s*|\()\s*(\w+)\)", body) + if len(out.get("IDirect3DDevice9", [])) != 119: + sys.exit("d3d9.h parse failed: IDirect3DDevice9 should have 119 methods") + return out + + +class Layouts: + def __init__(self, path: Path): + self.size: dict[str, int] = {} + self.fields: dict[str, list[tuple[int, str]]] = collections.defaultdict(list) + for line in path.read_text(errors="replace").splitlines(): + c = line.split("\t") + if len(c) < 5: + continue + if c[0] == "@STRUCT": + self.size[c[1]] = int(c[2]) + elif c[4] != "_padding_": + self.fields[c[0]].append((int(c[2]), c[3].strip())) + self.size.setdefault(c[0], int(c[1])) + + def member(self, t: str, off: int, depth: int = 0) -> str | None: + """Type of the member of `t` that starts exactly at `off` (through embedded members).""" + if depth > 6: + return None + base = BASES.get(t) + if base and off < self.size.get(base, 0) and not any(o == off for o, _ in self.fields.get(t, ())): + return self.member(base, off, depth + 1) + best = None + for o, ty in self.fields.get(t, ()): + if o == off and not ty.endswith("]"): + return ty + if o <= off and (best is None or o > best[0]): + best = (o, ty) + if best is None: + return None + o, ty = best + arr = re.fullmatch(r"(.+?)\s*\[(\d+)\]", ty) + if arr: + elem = arr.group(1) + esz = 4 if elem.endswith("*") else self.size.get(elem, 0) + if esz and off - o < esz * int(arr.group(2)): + rel = (off - o) % esz + return elem if rel == 0 else self.member(elem, rel, depth + 1) + return None + if ty in self.size and off - o < self.size[ty]: + return self.member(ty, off - o, depth + 1) + return None + + def load(self, t: str, off: int) -> str | None: + """Type of the dword loaded from `[T* + off]`: an embedded struct yields its first field + (e.g. CCountedPointer::Data).""" + ty = self.member(t, off) + for _ in range(4): + if ty is None or ty.endswith("*") or not (ty in self.fields or ty in self.size): + break + ty = self.member(ty, 0) + return ty + + +def functions() -> list[tuple[int, str, int]]: + out = [] + for line in COVERAGE.read_text(errors="replace").splitlines()[1:]: + c = line.split("\t") + if len(c) >= 5 and c[4] != "1" and int(c[3] or 0) > 0: + out.append((int(c[0], 16), c[1], int(c[3]))) + return out + + +def load_image(): + import pefile + pe = pefile.PE(str(EXE), fast_load=True) + return pe.OPTIONAL_HEADER.ImageBase, pe.get_memory_mapped_image() + + +def class_only_names() -> dict[str, str]: + """Bare method name -> device class, for names that no other class in Ego_r.pdb uses.""" + owners = collections.defaultdict(set) + for line in EGOR_NAMES.read_text(errors="replace").splitlines(): + m = re.match(r"[0-9a-fA-F]+ \?(\w+)@(\w+)@@", line) + if m and not m.group(1).startswith("?"): + owners[m.group(1)].add(m.group(2)) + return {n: next(iter(cs)) for n, cs in owners.items() + if len(cs) == 1 and next(iter(cs)) in CLASS_DEVICE_OFFSETS} + + +def rtti_vtable_methods(base: int, img: bytes, lay: "Layouts") -> dict[int, str]: + """Function -> `this` class from the retail file's own MSVC RTTI. Every vtable is preceded by a + complete-object locator (signature 0, offset 0 = primary vtable) naming its class; each slot that + points into .text is a method of that class. A function shared by several classes' vtables (an + inherited, non-overridden method) gets the smallest class with a PDB layout: its offsets are + valid in every derived class.""" + import pefile + pe = pefile.PE(str(EXE), fast_load=True) + sec = {s.Name.rstrip(b"\0").decode(): (base + s.VirtualAddress, base + s.VirtualAddress + s.Misc_VirtualSize) + for s in pe.sections} + text, rdata = sec[".text"], sec[".rdata"] + u32 = lambda a: int.from_bytes(img[a - base:a - base + 4], "little") + owners: dict[int, set[str]] = collections.defaultdict(set) + for a in range(rdata[0], rdata[1] - 8, 4): + col = u32(a) + if not rdata[0] <= col < rdata[1] - 16 or u32(col) != 0 or u32(col + 4) != 0: + continue + td = u32(col + 12) + if not sec[".data"][0] <= td < sec[".data"][1]: + continue + raw = img[td + 8 - base:td + 8 - base + 128].split(b"\0")[0].decode("latin1") + if not raw.startswith((".?AV", ".?AU")) or not raw.endswith("@@") or "@" in raw[4:-2]: + continue + cls = raw[4:-2] + if cls not in lay.size and cls not in lay.fields: + continue + v = a + 4 + while v < rdata[1] and text[0] <= u32(v) < text[1]: + owners[u32(v)].add(cls) + v += 4 + return {f: min(cs, key=lambda c: lay.size.get(c, 1 << 30)) for f, cs in owners.items()} + + +def pointee(ty: str | None) -> str | None: + if ty and ty.endswith("*") and not ty.endswith("**"): + return ty[:-1].strip() + return None + + +class Census: + def __init__(self): + self.ifaces = d3d9_interfaces() + self.lay = Layouts(LAYOUTS) + self.funcs = functions() + self.base, self.img = load_image() + self.md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32) + self.md.detail = True + self.globals: dict[int, str] = dict(SEED_GLOBALS) + # Live RTTI sweep (tools/modern/rtti_live_globals.py): typed statics and pointer globals. + self.statics: dict[int, str] = dict(STATIC_INSTANCES) + if RTTI_GLOBALS.exists(): + for line in RTTI_GLOBALS.read_text().splitlines()[1:]: + kind, addr, cls = line.split("\t")[:3] + if cls in self.lay.size or cls in self.lay.fields: + (self.statics if kind == "static" else self.globals).setdefault(int(addr, 16), cls) + self.this_type: dict[int, str] = rtti_vtable_methods(self.base, self.img, self.lay) + self.rtti_methods = set(self.this_type) # seeds that caller agreement must not override + self.returns: dict[int, str] = {} + self.owned = class_only_names() + self.tier_b: set[int] = set() # identities of tier-B device loads + + def scan(self): + calls = [] + callee_this = collections.defaultdict(collections.Counter) + new_globals = collections.defaultdict(collections.Counter) + new_returns = {} + lay, ifaces = self.lay, self.ifaces + for fva, fname, size in self.funcs: + code = self.img[fva - self.base: fva - self.base + size] + # reg -> (type, identity); vt: reg -> (type, identity) of the object whose vtable it holds + val: dict[str, tuple[str, int]] = {} + vt: dict[str, tuple[str, int]] = {} + if fva in self.this_type: + val["ecx"] = (self.this_type[fva], fva) + last_push = None + esp4 = None + dev_offs = set(UNIQUE_DEVICE_OFFSETS) | CLASS_DEVICE_OFFSETS.get(self.owned.get(fname, ""), set()) + ins = list(self.md.disasm(code, fva)) + if (len(ins) == 2 and ins[1].mnemonic == "ret" and ins[0].mnemonic == "mov" + and ins[0].operands[1].type == cx.X86_OP_IMM + and ins[0].operands[1].imm in self.statics): + new_returns[fva] = self.statics[ins[0].operands[1].imm] + if (len(ins) == 2 and ins[1].mnemonic == "ret" and ins[0].mnemonic == "mov" + and fva in self.this_type): + o = ins[0].operands + if (o[0].type == cx.X86_OP_REG and ins[0].reg_name(o[0].reg) == "eax" + and o[1].type == cx.X86_OP_MEM and o[1].mem.index == 0 + and ins[0].reg_name(o[1].mem.base) == "ecx"): + ty = pointee(lay.load(self.this_type[fva], o[1].mem.disp)) + if ty: + new_returns[fva] = ty + for insn in ins: + mn, ops = insn.mnemonic, insn.operands + if mn in ("mov", "lea") and len(ops) == 2 and ops[0].type == cx.X86_OP_MEM and mn == "mov": + m = ops[0].mem + if m.base == 0 and m.index == 0 and ops[1].type == cx.X86_OP_REG: + s = insn.reg_name(ops[1].reg) + if s in val and not val[s][0].startswith("@"): + new_globals[m.disp & 0xFFFFFFFF][val[s][0]] += 1 + elif (m.index == 0 and m.disp == 4 and ops[1].type == cx.X86_OP_REG + and insn.reg_name(m.base) == "esp"): + esp4 = val.get(insn.reg_name(ops[1].reg)) # `this` of a COM tail jump + continue + if (mn == "add" and len(ops) == 2 and ops[0].type == cx.X86_OP_REG + and ops[1].type == cx.X86_OP_IMM and insn.reg_name(ops[0].reg) in val): + # pointer to a member: `add edi, 0x3cf0` then `mov eax, [edi]` + r = insn.reg_name(ops[0].reg) + t, ident = val[r] + vt.pop(r, None) + if t in ifaces: + val.pop(r) + else: + base_t, off0 = (t[1:].rsplit("@", 1)[0], int(t.rsplit("@", 1)[1])) if t.startswith("@") else (t, 0) + val[r] = (f"@{base_t}@{off0 + ops[1].imm}", ident) + continue + if mn in ("mov", "lea") and len(ops) == 2 and ops[0].type == cx.X86_OP_REG: + dst = insn.reg_name(ops[0].reg) + if dst not in REGS32: + continue + src = ops[1] + nv = nvt = None + if src.type == cx.X86_OP_REG and mn == "mov": + s = insn.reg_name(src.reg) + nv, nvt = val.get(s), vt.get(s) + elif src.type == cx.X86_OP_IMM and mn == "mov" and src.imm in self.statics: + nv = (self.statics[src.imm], insn.address) + elif src.type == cx.X86_OP_MEM and src.mem.index == 0: + b = insn.reg_name(src.mem.base) if src.mem.base else None + d = src.mem.disp + if mn == "lea": + if b in val and val[b][0] not in ifaces: + ty = val[b][0] if d == 0 else lay.member(val[b][0], d) + if ty in lay.size or ty in lay.fields: + nv = (ty, insn.address) + elif b is None: + g = self.globals.get(d & 0xFFFFFFFF) + if g: + nv = (g, insn.address) + elif b in val: + t = val[b][0] + if d == 0 and t in ifaces: + nvt = val[b] + elif t.startswith("@"): + if mn == "mov": + bt, o0 = t[1:].rsplit("@", 1) + ty = pointee(lay.load(bt, int(o0) + d)) + if ty: + nv = (ty, insn.address) + else: + ty = pointee(lay.load(t, d)) + if ty: + nv = (ty, insn.address) + if nv is None and nvt is None and mn == "mov" and b and d in dev_offs: + nv = ("IDirect3DDevice9", insn.address) + self.tier_b.add(insn.address) + val.pop(dst, None) + vt.pop(dst, None) + if nv: + val[dst] = nv + if nvt: + vt[dst] = nvt + continue + if mn == "push" and len(ops) == 1: + last_push = insn.reg_name(ops[0].reg) if ops[0].type == cx.X86_OP_REG else None + continue + if mn == "call" and len(ops) == 1: + op = ops[0] + if op.type == cx.X86_OP_MEM and op.mem.base and op.mem.index == 0: + vreg = insn.reg_name(op.mem.base) + obj = vt.get(vreg) + if obj and last_push and val.get(last_push) == obj: + calls.append((fva, fname, insn.address, obj[0], op.mem.disp, + "B" if obj[1] in self.tier_b else "A")) + if (op.type == cx.X86_OP_IMM and "ecx" in val and val["ecx"][0] not in ifaces + and not val["ecx"][0].startswith("@")): + callee_this[op.imm][val["ecx"][0]] += 1 + esp4 = None + for r in VOLATILE: + val.pop(r, None) + vt.pop(r, None) + if op.type == cx.X86_OP_IMM and op.imm in self.returns: + val["eax"] = (self.returns[op.imm], insn.address) + last_push = None + continue + if mn == "jmp" and len(ops) == 1 and ops[0].type == cx.X86_OP_MEM and ops[0].mem.base: + # COM tail call: `mov [esp+4], this ; jmp [vtbl+off]` + obj = vt.get(insn.reg_name(ops[0].mem.base)) + if obj and ops[0].mem.index == 0 and esp4 == obj: + calls.append((fva, fname, insn.address, obj[0], ops[0].mem.disp, + "B" if obj[1] in self.tier_b else "A")) + continue + if mn == "ret": + val.clear(); vt.clear() + continue + if ops and ops[0].type == cx.X86_OP_REG and mn not in ("cmp", "test", "push"): + dst = insn.reg_name(ops[0].reg) + if dst in REGS32: + val.pop(dst, None) + vt.pop(dst, None) + self.callee_this = callee_this # kept for diagnostics + # Learn: `this` types agreed by every typed caller; globals written with one type only. + changed = 0 + for callee, cnt in callee_this.items(): + # Unanimous, or dominant (>= 3 typed callers, >= 80%): one caller mistyped by PDB offset + # drift must not hide a method (ClearRenderTarget 0x9BE420: 5 CDisplayManager, 1 CStateInfo). + t, top = cnt.most_common(1)[0] + total = sum(cnt.values()) + if callee not in self.rtti_methods and (len(cnt) == 1 or (total >= 3 and top >= 0.8 * total)): + if self.this_type.get(callee) != t: + self.this_type[callee] = t + changed += 1 + for g, cnt in new_globals.items(): + if len(cnt) == 1 and g not in self.globals: + self.globals[g] = next(iter(cnt)) + changed += 1 + for f, t in new_returns.items(): + if self.returns.get(f) != t: + self.returns[f] = t + changed += 1 + return calls, changed + + def run(self): + for i in range(ITERATIONS): + calls, changed = self.scan() + print(f"pass {i + 1}: {len(calls)} D3D9 calls, {len(self.this_type)} typed methods, " + f"{len(self.globals)} typed globals, {len(self.returns)} getters, {changed} new facts") + if not changed: + break + return calls + + +def write_report(c: Census, calls) -> None: + ifaces = c.ifaces + per = collections.defaultdict(collections.Counter) + per_funcs = collections.defaultdict(lambda: collections.defaultdict(set)) + dev_funcs = collections.Counter() + sites, bad = [], 0 + tiers = collections.Counter() + for fva, fname, cva, iface, off, tier in sorted(calls, key=lambda t: t[2]): + methods = ifaces[iface] + if off % 4 or off // 4 >= len(methods): + bad += 1 + continue + name = methods[off // 4] + per[iface][name] += 1 + tiers[(iface == "IDirect3DDevice9", tier)] += 1 + per_funcs[iface][name].add(fva) + if iface == "IDirect3DDevice9": + dev_funcs[(fva, fname)] += 1 + sites.append({"function": f"{fva:08X}", "name": fname, "call": f"{cva:08X}", + "interface": iface, "method": name, "tier": tier}) + dev = ifaces["IDirect3DDevice9"] + data = { + "source": "tools/modern/d3d9_device_census.py", + "functions_scanned": len(c.funcs), + "call_sites": len(sites), + "out_of_range_slots": bad, + "seed_globals": {f"{k:08X}": v for k, v in SEED_GLOBALS.items()}, + "static_instances": len(c.statics), + "rtti_typed_methods": len(c.rtti_methods), + "typed_globals": len(c.globals), + "typed_methods": len(c.this_type), + "interfaces": {i: {m: {"call_sites": n, "functions": len(per_funcs[i][m])} + for m, n in per[i].most_common()} for i in sorted(per)}, + "sites": sites, + } + OUT_JSON.write_text(json.dumps(data, indent=1) + "\n") + + used = [m for m in dev if per["IDirect3DDevice9"][m]] + nd = sum(per["IDirect3DDevice9"].values()) + L = [ + "# Direct3D 9 call-site census", + "", + "Generated by `tools/modern/d3d9_device_census.py` from the retail `Fable.exe`. Do not edit by", + "hand; every site is in `D3D9_DEVICE_CENSUS.json`. Sizes the RHI for", + "[MODERNIZATION](MODERNIZATION.md) step 4.", + "", + "Method: capstone over every non-thunk function, with registers typed from the PDB layouts.", + "Seeds: the render-manager pointers `[0x1436E18]` / `[0x13CB508]`, the static `CSystemManager`", + "`0x13CA618`, the live RTTI sweep of globals (`ghidra_out/rtti_live_globals.tsv`) and `this` of", + "every method in a retail RTTI vtable. Types spread into callees (unanimous or >= 80% of >= 3 typed", + "callers), globals, getters, embedded structs and member pointers. A site counts when the same", + "load supplies the vtable and the COM `this` (pushed, or stored to `[esp+4]` for a tail jump).", + "Still a lower bound (stack-spilled pointers are not followed), but against the live frontend", + "trace it finds 54 of 55 call sites ([D3D9_LIVE_CENSUS](D3D9_LIVE_CENSUS.md)). Method order is", + "the DirectX SDK `d3d9.h`.", + "", + f"Scanned {len(c.funcs)} functions; typed {len(c.this_type)} methods and {len(c.globals)} globals.", + f"**IDirect3DDevice9: {nd} call sites in {len(dev_funcs)} functions, {len(used)} of 119 methods.**", + f"Tier A (PDB-typed): {tiers[(True, 'A')]} device sites; tier B (retail device offset: the unique", + f"+0x3CF0, or a class-only method name from Ego_r.pdb): {tiers[(True, 'B')]}. Tier B covers", + "`CShaderRenderManager`, whose retail device member is +0x1D4 although Ego_r.pdb says +0x1E4.", + "", + "## IDirect3DDevice9 methods in use", + "", + "| Slot | Method | Call sites | Functions |", + "| ---: | --- | ---: | ---: |", + ] + for m in sorted(used, key=lambda m: (-per["IDirect3DDevice9"][m], dev.index(m))): + L.append(f"| {dev.index(m)} | `{m}` | {per['IDirect3DDevice9'][m]} | " + f"{len(per_funcs['IDirect3DDevice9'][m])} |") + L += ["", "## IDirect3DDevice9 methods not seen", "", + ", ".join(f"`{m}`" for m in dev if not per["IDirect3DDevice9"][m]), "", + "Not seen is not proof of unused: see the lower-bound note above.", "", + "## Other Direct3D 9 interfaces", "", "| Interface | Methods (call sites) |", "| --- | --- |"] + for i in sorted(per): + if i != "IDirect3DDevice9": + L.append(f"| `{i}` | " + ", ".join(f"`{m}` ({n})" for m, n in per[i].most_common()) + " |") + L += ["", "## Functions with the most device calls", "", "| Function | Name | Device calls |", + "| --- | --- | ---: |"] + for (fva, fname), n in dev_funcs.most_common(40): + L.append(f"| `{fva:08X}` | `{fname}` | {n} |") + OUT_MD.write_text("\n".join(L) + "\n", encoding="utf-8") + print(f"device: {nd} sites, {len(used)} methods, {len(dev_funcs)} functions; all D3D9: {len(sites)}") + + +def main() -> None: + c = Census() + calls = c.run() + write_report(c, calls) + + +if __name__ == "__main__": + main() diff --git a/tools/modern/d3d9_live_census.py b/tools/modern/d3d9_live_census.py new file mode 100644 index 0000000000..53a6782c44 --- /dev/null +++ b/tools/modern/d3d9_live_census.py @@ -0,0 +1,172 @@ +"""Live IDirect3DDevice9 census: launch retail Fable.exe under dbgeng (pybag) and record every device +method's callers. Complements the static lower bound in d3d9_device_census.py. + +How: breakpoint on the instruction in CRenderManagerCore::Init that stores the new device +(0xA0A955 `mov [esi+0x3CF0], edi`). There, read the device vtable and set a breakpoint on each of +the 119 methods (names from d3d9.h). Each hit records the return address ([esp]); a method's +breakpoint is removed after BUDGET hits so the game keeps running at speed. Results are written to +--out every few seconds. + +Safety: DEBUG_PROCESS_DETACH_ON_EXIT is set, so killing this tracer leaves the game running. It +does not touch game files. Launch only when the shared install is free. +""" +from __future__ import annotations + +import argparse +import collections +import json +import os +import struct +import sys +import time +from pathlib import Path + +os.environ["WINDBG_DIR"] = r"C:\Windows\System32" +os.environ["_NT_SYMBOL_PATH"] = "" +sys.stdout.reconfigure(line_buffering=True) + +from pybag import UserDbg # noqa: E402 +from pybag.dbgeng import core as DbgEng # noqa: E402 + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from d3d9_device_census import d3d9_interfaces # noqa: E402 + +INSTALL = Path(r"C:\Programs\Steam\steamapps\common\Fable The Lost Chapters") +DEVICE_STORE = 0xA0A955 # CRenderManagerCore::Init: mov [esi+0x3CF0], edi +DEVICE_STORE_BYTES = bytes.fromhex("89bef03c0000") +IMAGE_FILE_MACHINE_I386 = 0x14C +RENDER_CORE_GLOBAL = 0x13CB508 # CRenderManagerCore* (live-verified 2026-10-16: [+4] == [+0x3CF0] == device) +BUDGET = 3000 + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--seconds", type=int, default=150) + ap.add_argument("--out", required=True) + ap.add_argument("--attach", action="store_true", + help="attach to the running Fable.exe (Steam relaunches a debugger-created one) and " + "read the device from [[0x1436E18]+0x3CF0]; load-time Create* calls are missed") + args = ap.parse_args() + names = d3d9_interfaces()["IDirect3DDevice9"] + + dbg = UserDbg() + if args.attach: + pids = dbg.pids_by_name("Fable.exe") + if len(pids) != 1: + sys.exit(f"expected one Fable.exe, found {pids}") + dbg.attach(pids[0][0]) + else: + os.chdir(INSTALL) + dbg.create(str(INSTALL / "Fable.exe")) + dbg._client._cli.AddProcessOptions(0x00000001) # DEBUG_PROCESS_DETACH_ON_EXIT + dbg._control.SetEffectiveProcessorType(IMAGE_FILE_MACHINE_I386) + try: + trace(dbg, args, names) + finally: + # A crash without detach leaves every game thread suspended (seen 2026-10-16). + try: + dbg.detach() + print("detached; game left running") + except Exception as e: + print("detach failed:", e) + + +def trace(dbg, args, names) -> None: + got = dbg.read(DEVICE_STORE, len(DEVICE_STORE_BYTES)) + if bytes(got) != DEVICE_STORE_BYTES: + sys.exit(f"unexpected bytes at {DEVICE_STORE:#x}: {bytes(got).hex()}") + + state = {"device": None, "vtable": None, "methods": {}, "t_device": None} + hits = collections.Counter() + sites = collections.defaultdict(collections.Counter) + bp_ids = {} + + def u32(addr): + return struct.unpack("= BUDGET: + try: + d.bd(bp_ids[addr]) + except Exception: + pass + return DbgEng.DEBUG_STATUS_GO + + def on_store(bp, d): + regs = d.reg + dev = regs.edi & 0xFFFFFFFF + vt = u32(dev) + state.update(device=dev, vtable=vt, t_device=time.time()) + slots = [u32(vt + 4 * i) for i in range(len(names))] + by_addr = collections.defaultdict(list) + for i, a in enumerate(slots): + by_addr[a].append(names[i]) + state["methods"] = dict(by_addr) + for a in by_addr: + bp_ids[a] = d.bp(a, handler=lambda b, dd, a=a: on_method(b, dd, a)) + print(f"device {dev:#x} vtable {vt:#x}: {len(by_addr)} distinct method bodies hooked") + return DbgEng.DEBUG_STATUS_GO + + if args.attach: + class _R: # minimal stand-in so on_store can read the live device + pass + core = u32(RENDER_CORE_GLOBAL) + if not core or u32(core + 4) != u32(core + 0x3CF0): + sys.exit(f"render core {core:#x} does not hold one device at +4/+0x3CF0") + fake = _R(); fake.edi = u32(core + 0x3CF0) + class _D: + reg = fake + bp = dbg.bp + on_store(None, _D) + else: + dbg.bp(DEVICE_STORE, handler=on_store, oneshot=True) + + def dump(final=False): + data = { + "device": state["device"] and f"{state['device']:08X}", + "vtable": state["vtable"] and f"{state['vtable']:08X}", + "budget": BUDGET, + "final": final, + "methods": {n: {"hits": hits[n], "budget_exhausted": hits[n] >= BUDGET, + "call_sites": {f"{r:08X}": c for r, c in sites[n].most_common()}} + for n in names if hits[n]}, + "shared_bodies": {f"{a:08X}": v for a, v in state["methods"].items() if len(v) > 1}, + } + Path(args.out).write_text(json.dumps(data, indent=1)) + + t0 = time.time() + last = 0.0 + while time.time() - t0 < args.seconds: + try: + dbg.go(timeout=2) # pybag wait() takes SECONDS (it multiplies by 1000) + except Exception as e: # wait timeout or a non-breakpoint event + msg = str(e) + if "not accessible" in msg: + print("target gone:", msg[:120]) + break + if "timeout" not in msg.lower() and "E_PENDING" not in msg: + print("event:", msg[:200]) + try: + dbg.go_nothandled(timeout=2) + except Exception: + pass + if time.time() - last > 5: + last = time.time() + dump() + print(f"t={time.time() - t0:5.0f}s methods={sum(1 for n in names if hits[n])} " + f"hits={sum(hits.values())}") + dump(final=True) + for a, i in bp_ids.items(): + try: + dbg.bc(i) + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/tools/modern/d3d9_live_report.py b/tools/modern/d3d9_live_report.py new file mode 100644 index 0000000000..755f89d4cf --- /dev/null +++ b/tools/modern/d3d9_live_report.py @@ -0,0 +1,113 @@ +"""Report a live IDirect3DDevice9 trace (d3d9_live_census.py) against the static census. + +Usage: python tools/modern/d3d9_live_report.py [--label frontend] +Writes docs/modern/D3D9_LIVE_CENSUS.md and copies the trace to docs/modern/D3D9_LIVE_CENSUS_