From 05a6909da9c41d2b38f7944d24f75d34695d6925 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 08:56:24 -0600 Subject: [PATCH 01/75] Add guarded GUI definition donor cloning and packaged acceptance --- CMakeLists.txt | 2 +- docs/AUTOMATION.md | 10 ++ docs/CLI.md | 2 +- docs/EDITOR.md | 15 +- docs/HANDOFF_GUI_CLONE_2026-10-10.md | 82 +++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 25 +++- gui/app.cpp | 1 + gui/app.hpp | 5 + gui/automation.cpp | 6 +- gui/definitionbrowser.cpp | 77 ++++++++++- gui/editor.cpp | 2 +- tools/check_all.py | 2 + tools/test_definition_clone_retail.py | 41 +++++- tools/test_definition_clone_ui.py | 192 ++++++++++++++++++++++++++ tools/test_package.py | 2 + 16 files changed, 445 insertions(+), 21 deletions(-) create mode 100644 docs/HANDOFF_GUI_CLONE_2026-10-10.md create mode 100644 tools/test_definition_clone_ui.py diff --git a/CMakeLists.txt b/CMakeLists.txt index e086d745..9c6e83fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,7 +55,7 @@ target_include_directories(forgecore_slice PUBLIC ) # "-dev" until the release is tagged (docs/RELEASE.md step 5 clears it) -set(FORGE_VERSION_SUFFIX "-rc.2") +set(FORGE_VERSION_SUFFIX "-dev") add_compile_definitions(ALBION_VERSION="${PROJECT_VERSION}${FORGE_VERSION_SUFFIX}") add_library(fableforge STATIC src/terrainexport.cpp src/foliageexport.cpp src/thingsexport.cpp src/stbterrain.cpp src/effects.cpp src/leveledit.cpp src/vanilla_props.cpp src/worldedit.cpp src/overworld.cpp src/stbrelocate.cpp src/lodbake.cpp src/stitch.cpp src/livelink.cpp src/backups.cpp src/navlines.cpp src/worldtiles.cpp src/stbcompact.cpp src/presets.cpp src/gtg.cpp src/texturebrowse.cpp src/meshimport.cpp src/modpack.cpp) diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md index 5b936fe7..63ec5c2d 100644 --- a/docs/AUTOMATION.md +++ b/docs/AUTOMATION.md @@ -1081,6 +1081,16 @@ Definitions inspection uses `assets_tab 6`, `wait_definitions`, `definition_bank Definition scalar drafts use `definition_edit `, `definition_value `, `definition_save`, and `definition_cancel`. The edit/value/save `_refused` variants check refusals. `definition_editing` and `definition_edit_failed` report draft state. Saving reselects the explicit native index, even when another record has a numeric name. +Definition clone drafts use `definition_clone` (or the real +`btn_definition_clone` control), `definition_value `, `definition_save` +and `definition_cancel`. `definition_clone_refused` checks draft/load refusal; +`definition_cloning` distinguishes a clone from a field draft. Real name entry +uses `input_definition_edit_value`; `btn_definition_save` and +`btn_definition_save_confirm` exercise the Create confirmation. The +`definition_clone_ui` package workflow covers game/frontend/script banks, +numeric/unnamed/duplicate-name donors, folder/pack outputs, stale sources, +invalid names, cancellation, Close/Install guards and paired Restore at both sizes. + `definition_group` reports the validated game-bank group entry index, or `-`. `btn_definition_group` follows it by index after the current frame, protecting numeric names and the current decoded view. Definition vector drafts expose `input_definition_vector_0`, `_1` and (for C3DVector) `_2` for real coordinate input. `definition_value` accepts the existing comma-separated writer syntax for these drafts. The dedicated `definition_vectors` package workflow independently verifies every other payload byte, exact no-ops and malformed input/layout refusals at both sizes. diff --git a/docs/CLI.md b/docs/CLI.md index ddb70be2..2cfa46af 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -1,6 +1,6 @@ # Command line reference -`FableForge 0.18.1-rc.2 -- Fable: The Lost Chapters level editor (maps -> .glb / .obj, levels, terrain, textures)` +`FableForge 0.18.1-dev -- Fable: The Lost Chapters level editor (maps -> .glb / .obj, levels, terrain, textures)` Generated by `tools/gen_cli_reference.py` from `FableForge --help`; edit `src/cli/common.cpp`'s usage text, not this file. Every command takes `--install ` (default: the Steam / GOG install found on this machine). diff --git a/docs/EDITOR.md b/docs/EDITOR.md index d889fd66..e4b23e20 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1465,4 +1465,17 @@ Definition CDefString fields show a readable name when their byte offset identif Four-byte definition name references can now be edited with an existing-name chooser. Search finds names or byte offsets; duplicate names appear with distinct offsets. No name stores -1. Saving keeps the existing name pool and refuses stale references. Original unsupported offsets can remain unchanged. -The CLI can clone a definition: `forge-tools defs clone --out `. It copies the whole donor payload and saves the bank/name-pool pair with recovery records. Existing outputs must match the loaded bank and pool. GUI clone controls are still pending. +Select a record in Assets > Definitions and choose **Clone selected definition**. +Enter a unique name of up to 127 ASCII characters, then choose **Create definition** +and confirm. The copy keeps every field and reference target from its donor; +it does not copy the records those references point to. Named and unnamed records +can be donors, including records whose fields cannot be decoded by the inspector. +The new record is selected after saving. Save folders and valid pack destinations +use the existing destination chooser; Cancel and Close/Install protection retain +the draft until you save or discard it. Changed sources or destinations refuse. +Restore recovers the bank and its names together, or removes a newly created pair. + +The CLI provides the same operation with `forge-tools defs clone + --out `. Existing outputs must +match the loaded bank and pool. GUI cloning is part of the development continuation +after rc.2; the published rc.2 candidate contains the CLI operation. diff --git a/docs/HANDOFF_GUI_CLONE_2026-10-10.md b/docs/HANDOFF_GUI_CLONE_2026-10-10.md new file mode 100644 index 00000000..3503a60c --- /dev/null +++ b/docs/HANDOFF_GUI_CLONE_2026-10-10.md @@ -0,0 +1,82 @@ +# GUI definition clone continuation, 2026-10-10 + +Owned development checkout: +`C:/Users/Cornelio/AppData/Local/FableForgeDevelopment/definition-clone-gui-20261010`, +branch `feat/definition-clone-gui-20261010`, starting commit `c3f87a7`. +Its 971 tracked baseline files were compared byte-for-byte with the clean release +checkout: zero differences. The slow partial checkout from D was stopped by +terminating only its owned process tree; tracked files and the index were copied +from the clean C release checkout. Git subsequently confirmed a clean baseline. +The separate dirty D lane and its dialogue-audio test remain untouched. + +## Implemented + +Assets > Definitions now offers Clone selected definition, a new-name draft and +Create confirmation. It uses the existing exact donor API and paired publisher; +all prior records and pool offsets remain intact. References retain their original +targets. Unknown/opaque payloads can be cloned without inspector decoding. +Game/frontend/script banks, direct writes, save folders and valid packs are supported. +Whole source bank/name bytes are captured and checked for stale drafts. Changed +destinations and incompatible existing outputs refuse. Existing Close/Install +protection uses the shared definition draft state. New records are selected by +explicit index after saving; game-bank contexts are refreshed. + +## Verification completed before the development-version rebuild + +- First isolated build of GUI, both CLIs and clone/writer tests passed. +- Core clone and BIN writer CTests pass (two targets). +- New GUI cases: 38 at desktop/compact sizes, across 19 modes. Initial save/Restore + evidence: `C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261010/definition-clone-ui-ndvtht_0` + (four cases). Remaining modes: `definition-clone-ui-3cazxxz9` (34 cases). +- Existing scalar draft suite: 34 cases pass; evidence `definition-edit-ui-z8kpwk0s`. +- Actual retail read-only inputs with new GUI controls: all 14,764 records and raw + identity rows preserved, exact donor/name and ordinal appended to owned output, + created bank/name files removed by explicit-root CLI Restore, native compiled-bank + hashes unchanged. Evidence: `retail-definition-clone-k8nu1mql`. Its created-record + screenshot was inspected. The output has no texture bank; context warnings after + the successful copy are expected and do not claim a full asset export. +- `definition_clone_ui` is registered in source and extracted-package gates. + `test_definition_clone_retail.py --gui` makes the retail UI proof repeatable. + +The first feature binaries inherited rc.2's version suffix. The source now uses +`-dev` to distinguish this unpublished continuation. Its isolated rebuild completed; +a subsequent Ninja check reports no work. CLI documentation was regenerated. +The development archive is `dist/development/20261010-def-clone/FableForge-0.18.1-dev-win64.zip`, +SHA256 `73fb4dae506f4a387bc26bcf64e464a1276c29253e77f0aae6ad6cb92c5a9349`. +Its packaging metadata correctly records baseline `c3f87a7` with tracked changes; +it is a development checkpoint, not a clean-commit release. Extracted acceptance +passes under `package-smoke-9hds0ft1`: 112 file identities, 132 local documentation +links and seven selected workflows (retail, GUI smoke, CLI clone, GUI clone, +existing-name picker, scalar drafts and text close prompt); 120 not run, zero skips. +The packaged GUI also passes the independent 14,764-record retail preservation +and Restore check at `retail-definition-clone-ulonnzlv`, with native hashes unchanged. +The first invocation used the extraction parent rather than its versioned folder +and failed before launching an executable; the corrected explicit path passed. +Both core targets pass again after the version rebuild; Python compilation and +`git diff --check` pass. No full suite or live-game test was repeated. +No GUI-clone GitHub publication has occurred yet. + +## RC2 release work in parallel + +PR 1 merged at `f96692dbb6d69b1f3daef6014a092bbcd72a190a` after final-head CI +38059467407 passed. Remote tag `v0.18.1-rc.2` points to that merge. Tag CI +38060259144 and main CI 38060238810 both passed, including tagged packaging. +The unchanged RC2 archive remains in the separate release checkout, SHA256 +`d488737601d00b054bd27d5d0052d1c9994ba248a6136a35e3d5c69afd0c5e91`. +It passed 111 file identities, 131 links and 57 selected extracted workflows; +66 not run, two optional corpus skips. It is now published as a +[GitHub prerelease](https://github.com/BuffJesus/FableForge/releases/tag/v0.18.1-rc.2). +GitHub's application asset digest matches the validated hash exactly; its size is +36,896,482 bytes. The illustrated guide is also attached, SHA256 +`bcdbdb7b33ae616c765bc00437cb3a42d9fe2311e2b28e45f9547c996f211721`. +Stable 0.18.0 remains latest; rc.1 is retained. The release checkout stays clean +at `c3f87a7`. The new GUI-clone development build is not part of that release. + +## Next + +Commit the scoped GUI continuation, then choose the next roadmap task. The +development rebuild, compact confirmation inspection and exact extracted archive +acceptance are complete. RC2 publication is completed; keep its validated archive +unchanged and separate from the new development checkpoint. +Continue roadmap work until the user asks to stop. Live gameplay and a clean-machine +walkthrough remain separate; no native install write or checkpoint deletion is needed. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 76e6bcd6..f9af8d7e 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Latest session checkpoint: [2026-10-10 continuation](HANDOFF_2026-10-10.md). RC2 passes all 54 CTests and 57 selected extracted workflows; GitHub release integration is next. +> Active continuation: [GUI definition cloning](HANDOFF_GUI_CLONE_2026-10-10.md) in its owned C development worktree. RC2 is published with a verified asset hash; stable 0.18.0 stays latest. GUI clone source/retail and extracted development-package checks pass. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index a13a2047..94a0bf61 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Latest session checkpoint: [2026-10-10 continuation](HANDOFF_2026-10-10.md). RC2 passes all 54 CTests and 57 selected extracted workflows; GitHub release integration is next. +> Active continuation: [GUI definition cloning](HANDOFF_GUI_CLONE_2026-10-10.md) in its owned C development worktree. RC2 is published with a verified asset hash; stable 0.18.0 stays latest. GUI clone source/retail and extracted development-package checks pass. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there @@ -2456,3 +2456,26 @@ forge-tools defs clone --out [--nam Thirty-five CLI cases pass: exact/numeric/unnamed/duplicate-index donors, invalid selectors/names, fresh and in-place publication, existing outputs, incompatible/missing pools, orphan recovery, physical aliases, rollback after a locked name-file rename, Restore and running-owner/junction refusal using an owned dummy helper. Original source files remain exact. Both clone/BIN-writer CTest targets pass. A full retail-bank export preserves all 14,764 prior records and identity-table rows, appends the exact donor and one correctly keyed name, and restores creation outputs without changing native hashes. Nineteen field-publication and fourteen selector neighbors pass. C evidence: definition-clone-038ldrlw, retail-definition-clone-export-rdt8kq3r, definition-publication-lz8k6ia0 and definition-selector-a95ictqb. GUI cloning remains pending. The user explicitly requested GitHub milestone releases/updates on 2026-10-09. Prepare 0.18.1-rc.2 as a prerelease in the isolated C release checkout, preserving the working lane and excluding the separately owned test_dialogueaudio.cpp edits. Existing GitHub rc.1 is retained; do not overwrite it. Source publication, CI and exact RC2 package validation are in progress. + +## GUI definition clone continuation, 2026-10-10 + +The isolated development branch adds Clone selected definition, a bounded new-name +draft and Create confirmation around the proven paired publisher. Prior records, +identity slots and name offsets remain intact; references retain their original +targets. Opaque, numeric-named, unnamed and indexed duplicate-name donors are supported. +Whole captured source bank/name bytes, destination identity, pack validity and +compatible outputs are checked before publication. Shared definition draft state +protects Close and Install; the created record is reselected by explicit index. + +Thirty-eight real-control GUI cases pass at both sizes across game/frontend/script +banks, folder/pack output, refusals, cancellation, stale sources and Restore. +All 34 neighboring scalar-draft cases and both core clone/BIN-writer targets pass. +Actual retail inputs through the new GUI preserve all 14,764 prior records and raw +identity rows; independent donor/key/ordinal checks and explicit-output Restore pass, +with native compiled-bank hashes unchanged. The development-version rebuild and +extracted package pass: 112 identities, 132 links, seven selected workflows, +120 not run and zero skips. Archive SHA256: +`73fb4dae506f4a387bc26bcf64e464a1276c29253e77f0aae6ad6cb92c5a9349`. +The packaged GUI also passes the independent retail clone/Restore check. +This is separate from RC2's unchanged archive; no full-suite or live-game claim. +Evidence and resume: [GUI clone handoff](HANDOFF_GUI_CLONE_2026-10-10.md). diff --git a/gui/app.cpp b/gui/app.cpp index d8893161..13a77a54 100644 --- a/gui/app.cpp +++ b/gui/app.cpp @@ -1931,6 +1931,7 @@ std::vector App::stateDump() const { } v.push_back("definition_name_refs="+std::to_string(definitionNameRefs)); v.push_back("definition_editing="+std::to_string(definitionEditing_)); + v.push_back("definition_cloning="+std::to_string(definitionCloning_)); v.push_back("definition_edit_failed="+std::to_string(!definitionEditError_.empty())); v.push_back("definition_ready="+std::to_string(bool(definitionDecoded_))); v.push_back("definition_failed="+std::to_string(!definitionError_.empty() || !definitionSelectionError_.empty() || !definitionDecodeError_.empty())); diff --git a/gui/app.hpp b/gui/app.hpp index eb464824..502cdf0e 100644 --- a/gui/app.hpp +++ b/gui/app.hpp @@ -589,6 +589,8 @@ class App { bool setDefinitionBank(const std::string& bank); void browseDefinition(const std::string& name); bool beginDefinitionEdit(size_t field); + bool beginDefinitionClone(); + bool saveDefinitionClone(); bool setDefinitionEditValue(const std::string& value); bool saveDefinitionEdit(); bool definitionsBusy() const {return definitionFuture_.valid();} @@ -619,6 +621,9 @@ class App { char definitionSearch_[256]={}; bool definitionLoaded_=false,definitionFilterDirty_=true; bool definitionEditing_=false,definitionEditConfirm_=false; + bool definitionCloning_=false; + forge::bin::File definitionCloneFile_; + std::vector definitionCloneNamesBytes_,definitionCloneBankBytes_; size_t definitionEditIndex_=0; forge::bin::Entry definitionEditExpected_; std::vector> definitionEditExpectedNames_; diff --git a/gui/automation.cpp b/gui/automation.cpp index 27d06b9c..478aced2 100644 --- a/gui/automation.cpp +++ b/gui/automation.cpp @@ -697,6 +697,9 @@ bool Automation::tick(App& app) { } else if (cmd == "model_select") { if (!app.selectModel(rest)) fail("model_select: " + rest); else note("ok " + line); ++pc_; } else if (cmd == "model_search") { std::snprintf(app.modelSearch_, sizeof app.modelSearch_, "%s", rest == "-" ? "" : rest.c_str()); note("ok " + line); ++pc_; } + else if(cmd=="definition_clone" || cmd=="definition_clone_refused") { + if(app.beginDefinitionClone()!=(cmd=="definition_clone")) fail("definition clone returned an unexpected result");else note("ok "+line);++pc_; + } else if(cmd=="definition_edit" || cmd=="definition_edit_refused") { size_t field=0;const auto parsed=std::from_chars(rest.data(),rest.data()+rest.size(),field); const bool ok=parsed.ec==std::errc{} && parsed.ptr==rest.data()+rest.size() && app.beginDefinitionEdit(field); @@ -709,7 +712,8 @@ bool Automation::tick(App& app) { if(app.saveDefinitionEdit()!=(cmd=="definition_save")) fail("definition save returned an unexpected result");else note("ok "+line);++pc_; } else if(cmd=="definition_cancel") { - app.definitionEditing_=false;app.definitionEditConfirm_=false;note("ok "+line);++pc_; + app.definitionEditing_=false;app.definitionCloning_=false;app.definitionEditConfirm_=false; + app.definitionCloneFile_={};app.definitionCloneNamesBytes_.clear();app.definitionCloneBankBytes_.clear();note("ok "+line);++pc_; } else if(cmd=="wait_definitions") { app.pollDefinitions();waitOn(!app.definitionsBusy() && app.definitionLoaded_,"definitions"); diff --git a/gui/definitionbrowser.cpp b/gui/definitionbrowser.cpp index 0742bb02..16f2d6f4 100644 --- a/gui/definitionbrowser.cpp +++ b/gui/definitionbrowser.cpp @@ -2,6 +2,7 @@ #include "theme.hpp" #include "windowsprocess.hpp" #include "definitionfield.hpp" +#include "definitionclone.hpp" #include "definitionread.hpp" #include "texturebrowse.hpp" #include "modpack.hpp" @@ -137,11 +138,65 @@ bool App::beginDefinitionEdit(size_t field) { definitionEditIndex_=*definitionSelected_;definitionEditExpected_=definitionFile_.entries()[definitionEditIndex_]; definitionEditExpectedNames_=definitionFile_.originalNames();definitionEditNames_=definitionLoadedNames_;definitionEditBank_=definitionLoadedBank_; definitionEditDestination_=fs::u8path(packDest_.empty()?saveRoot():packDest_)/"data"/"CompiledDefs"/definitionBank_; - definitionEditField_=selected.name;definitionEditType_=selected.type;definitionEditing_=true; + definitionEditField_=selected.name;definitionEditType_=selected.type;definitionEditing_=true;definitionCloning_=false; definitionNameSearch_[0]=0;setDefinitionEditValue(definitionwrite::fieldValue(selected));return true; } catch(const std::exception& error) {definitionEditError_=error.what();return false;} } +bool App::beginDefinitionClone() { + definitionEditError_.clear();definitionEditSuccess_.clear(); + try { + if(definitionEditing_) throw std::runtime_error("Save or cancel the current definition draft first."); + if(definitionsBusy() || modFilesBusy() || !definitionSelected_ || + definitionPaths()!=std::pair{definitionLoadedNames_,definitionLoadedBank_}) + throw std::runtime_error("Choose a record from the current loaded definition bank."); + if(!packDest_.empty()) {if(!modpack::isPack(fs::u8path(packDest_))) throw std::runtime_error("Choose a valid FableForge pack.");(void)modpack::load(fs::u8path(packDest_));} + auto current=forge::bin::File::open(definitionLoadedNames_,definitionLoadedBank_); + definitionwrite::verifyEntries(definitionFile_,current); + definitionCloneNamesBytes_=definitionwrite::cloneBytes(definitionLoadedNames_); + definitionCloneBankBytes_=definitionwrite::cloneBytes(definitionLoadedBank_); + definitionCloneFile_=std::move(current); + definitionEditIndex_=*definitionSelected_; + definitionEditNames_=definitionLoadedNames_;definitionEditBank_=definitionLoadedBank_; + definitionEditDestination_=fs::u8path(packDest_.empty()?saveRoot():packDest_)/"data"/"CompiledDefs"/definitionBank_; + const auto& donor=definitionFile_.entries()[definitionEditIndex_]; + definitionEditField_=donor.name.empty()?"(unnamed record)":donor.name; + definitionEditType_="clone";definitionEditing_=true;definitionCloning_=true; + definitionEditConfirm_=false;definitionEditValue_[0]=0;return true; + } catch(const std::exception& error) {definitionEditError_=error.what();return false;} +} +bool App::saveDefinitionClone() { + definitionEditError_.clear();definitionEditSuccess_.clear(); + try { + if(!definitionEditing_ || !definitionCloning_) throw std::runtime_error("Choose a record to clone first."); + if(definitionsBusy() || modReadersBusy() || gameWriteBlocked("clone definition")) throw std::runtime_error("Wait for loads and writes, and close the destination game."); + const auto destination=fs::u8path(packDest_.empty()?saveRoot():packDest_)/"data"/"CompiledDefs"/definitionBank_; + if(definitionPaths()!=std::pair{definitionEditNames_,definitionEditBank_} || destination!=definitionEditDestination_) + throw std::runtime_error("Definition source or destination changed; start the clone again."); + if(definitionEditBank_!=destination && fs::exists(destination)) throw std::runtime_error("The destination bank appeared; refresh before cloning."); + if(!packDest_.empty()) {if(!modpack::isPack(fs::u8path(packDest_))) throw std::runtime_error("Choose a valid FableForge pack.");(void)modpack::load(fs::u8path(packDest_));} + if(definitionwrite::cloneBytes(definitionEditNames_)!=definitionCloneNamesBytes_ || + definitionwrite::cloneBytes(definitionEditBank_)!=definitionCloneBankBytes_) + throw std::runtime_error("The source bank or name pool changed; start the clone again."); + auto cloned=definitionCloneFile_; + const auto added=forge::defedit::cloneEntry(cloned,definitionEditIndex_,definitionEditValue_); + definitionwrite::publishClone(cloned,definitionEditNames_,definitionEditBank_,destination,true); + definitionEditSuccess_="Created "+std::string(definitionEditValue_)+"."; + definitionEditing_=false;definitionCloning_=false;definitionEditConfirm_=false; + definitionCloneFile_={};definitionCloneNamesBytes_.clear();definitionCloneBankBytes_.clear(); + definitionLoaded_=false;definitionBrowseRequest_.reset();definitionBrowseIndex_=added; + if(definitionBank_=="game.bin") { + const auto targetRoot=destination.parent_path().parent_path().parent_path(); + if(packDest_.empty() || (!ctx_.gameRoot().empty() && paths::resolvedPath(targetRoot)==paths::resolvedPath(ctx_.gameRoot()))) { + defThumbs_.clear();defList_.clear();themeGroupOf_.clear();envDefs_.clear();soundDefs_.clear();familyNames_.clear();issuesRev_=~0ull; + ctxError_.clear();reloadWhenContextReady_=true; + startContextLoad(packDest_.empty()?saveRoot():packDest_); + } + } + rescanBackups();pushLog(definitionEditSuccess_,3);return true; + } catch(const std::exception& error) {definitionEditError_=error.what();pushLog("clone definition: "+definitionEditError_,1);return false;} +} bool App::saveDefinitionEdit() { + if(definitionCloning_) return saveDefinitionClone(); definitionEditError_.clear();definitionEditSuccess_.clear(); try { if(!definitionEditing_) throw std::runtime_error("Choose a field to edit first."); @@ -179,8 +234,16 @@ void App::drawDefinitionBrowser(float pad,float inner,float cardInner) { } ImGui::SetCursorPosX(pad);theme::beginCard("##definitionbrowser",inner,true);theme::label("Definitions"); drawPackDestination(cardInner,true); + if(!definitionEditing_) { + ImGui::BeginDisabled(definitionsBusy() || modFilesBusy() || !definitionSelected_); + if(theme::ghostButton("Clone selected definition",ImVec2(cardInner,0))) beginDefinitionClone(); + auto_.registerWidget("btn_definition_clone");ImGui::EndDisabled(); + } if(definitionEditing_) { - ImGui::TextWrapped("Edit %s (%s)",definitionEditField_.c_str(),definitionEditType_.c_str()); + if(definitionCloning_) { + ImGui::TextWrapped("Clone %s",definitionEditField_.c_str()); + theme::hint("Copies this record unchanged. References keep their existing targets."); + } else ImGui::TextWrapped("Edit %s (%s)",definitionEditField_.c_str(),definitionEditType_.c_str()); ImGui::TextWrapped("Destination: %s",definitionEditDestination_.string().c_str()); if(definitionEditType_=="CDefString") { std::string preview=definitionEditValue_; @@ -267,19 +330,19 @@ void App::drawDefinitionBrowser(float pad,float inner,float cardInner) { if(changed) setDefinitionEditValue(definitionwrite::vectorValue(values,count)); } else ImGui::InputText("##definitionvalue",definitionEditValue_,sizeof definitionEditValue_); } else { - const bool text=definitionEditType_=="CCharString" || definitionEditType_=="CWideString"; - if(text) ImGui::TextDisabled("Text (up to 127 ASCII characters)"); + const bool text=definitionCloning_ || definitionEditType_=="CCharString" || definitionEditType_=="CWideString"; + if(text) ImGui::TextDisabled("%s",definitionCloning_?"New name (up to 127 ASCII characters)":"Text (up to 127 ASCII characters)"); ImGui::SetNextItemWidth(cardInner);ImGui::InputText("##definitionvalue",definitionEditValue_,sizeof definitionEditValue_, text?ImGuiInputTextFlags_CallbackCharFilter:0,text?definitionASCIIChar:nullptr); } auto_.registerWidget("input_definition_edit_value"); if(!definitionEditConfirm_) { - if(theme::ghostButton("Save field",ImVec2(cardInner,0))) definitionEditConfirm_=true; + if(theme::ghostButton(definitionCloning_?"Create definition":"Save field",ImVec2(cardInner,0))) definitionEditConfirm_=true; auto_.registerWidget("btn_definition_save"); - if(theme::ghostButton("Cancel edit",ImVec2(cardInner,0))) {definitionEditing_=false;definitionEditConfirm_=false;} + if(theme::ghostButton("Cancel edit",ImVec2(cardInner,0))) {definitionEditing_=false;definitionCloning_=false;definitionEditConfirm_=false;definitionCloneFile_={};definitionCloneNamesBytes_.clear();definitionCloneBankBytes_.clear();} auto_.registerWidget("btn_definition_edit_cancel"); } else { - const int answer=confirmRow("Write this field to the destination bank?","Save",cardInner,S(28),"btn_definition_save_confirm"); + const int answer=confirmRow(definitionCloning_?"Create this definition in the destination bank?":"Write this field to the destination bank?",definitionCloning_?"Create":"Save",cardInner,S(28),"btn_definition_save_confirm"); if(answer>0) saveDefinitionEdit();if(answer) definitionEditConfirm_=false; } } diff --git a/gui/editor.cpp b/gui/editor.cpp index 838fa5a0..2d855e50 100644 --- a/gui/editor.cpp +++ b/gui/editor.cpp @@ -4198,7 +4198,7 @@ void App::drawUnsavedPrompt() { if (worldPendingCount()) ImGui::TextColored(theme::vec(theme::Muted), "%zu pending world change%s not written.", worldPendingCount(), worldPendingCount() == 1 ? "" : "s"); if(definitionEditing_) - ImGui::TextColored(theme::vec(theme::Muted),"%s has an open definition field draft. Review and save it from Assets > Definitions before closing.",definitionEditField_.c_str()); + ImGui::TextColored(theme::vec(theme::Muted),"%s has an open definition %s draft. Review and save it from Assets > Definitions before closing.",definitionEditField_.c_str(),definitionCloning_?"clone":"field"); if (textEditing_) ImGui::TextColored(theme::vec(theme::Muted), "%s has an open text draft. Review and save it from Assets > Text before closing.",textEditName_.c_str()); diff --git a/tools/check_all.py b/tools/check_all.py index 25ecc155..6a2c9381 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -322,6 +322,8 @@ def main(): [sys.executable, "tools/test_definition_vector_edit_ui.py"], capture_output=True) ok &= run("Definition donor cloning, paired publication, rollback and exact Restore", [sys.executable, "tools/test_definition_clone.py"], capture_output=True) + ok &= run("Definition clone controls, exact donors, stale drafts and paired Restore", + [sys.executable, "tools/test_definition_clone_ui.py"], capture_output=True) ok &= run("Definition existing-name picker, exact field deltas, invalid offsets and Restore", [sys.executable, "tools/test_definition_name_picker_ui.py"], capture_output=True) ok &= run("Definition owning names offsets, duplicate names, refresh and source preservation", diff --git a/tools/test_definition_clone_retail.py b/tools/test_definition_clone_retail.py index 5e09ea1b..6ab77e04 100644 --- a/tools/test_definition_clone_retail.py +++ b/tools/test_definition_clone_retail.py @@ -3,6 +3,7 @@ import argparse import hashlib import json +import os from pathlib import Path import shutil import struct @@ -23,6 +24,7 @@ def main(): parser.add_argument('--install', type=Path, required=True) parser.add_argument('--tool', type=Path, default=repo/'build/forge-tools.exe') parser.add_argument('--forge', type=Path, default=repo/'build/forge.exe') + parser.add_argument('--gui', type=Path, help='Drive the GUI clone controls instead of the CLI') parser.add_argument('--work-root', type=Path, default=repo/'build') parser.add_argument('--donor', default='OBJECT_QUEST_CARD_TEMPLATE') args = parser.parse_args() @@ -50,18 +52,42 @@ def main(): bank = output/'data/CompiledDefs/game.bin' names = bank.parent/'names.bin' - def run(label, command): + def run(label, command, env=None): result = subprocess.run(list(map(str, command)), cwd=work, capture_output=True, - text=True, encoding='utf-8', errors='replace', timeout=120) + text=True, encoding='utf-8', errors='replace', timeout=120, env=env) (work/(label+'.log')).write_text(result.stdout+result.stderr, encoding='utf-8') assert result.returncode == 0, result.stdout+result.stderr return result try: - result = run('clone', [args.tool.resolve(), 'defs', 'clone', folder/'game.bin', - args.donor, new_name, '--out', bank, '--json']) - report = json.loads(result.stdout) - assert report['donor_index'] == donor and report['entry_index'] == len(before) + if args.gui: + from test_definition_edit_ui import click + # The selected native install supplies read-only map/asset inputs; + # every authored file goes to this new owned output directory. + script = work/'clone.txt' + lines = ['wait_maps', 'wait_ready', f'set saveroot {output.as_posix()}', + 'assets_tab 6', 'wait_definitions', 'frames 3', + 'definition_select '+args.donor, 'frames 3'] + lines += click('btn_definition_clone')+['assert_state definition_cloning 1'] + lines += click('input_definition_edit_value')+['input_text '+new_name, 'frames 3'] + lines += click('btn_definition_save')+click('btn_definition_save_confirm') + lines += ['wait_ready', 'wait_definitions', 'frames 3', + 'assert_state definition_cloning 0', + f'assert_state definition_selected {len(before)}', + 'definition_select '+new_name, + f'screenshot {(work/"created.png").as_posix()}', 'quit'] + script.write_text('\n'.join(lines)+'\n', encoding='utf-8') + run('clone-ui', [args.gui.resolve(), '--install', args.install.resolve(), + '--size', '1440x900', '--auto', script], env=dict(os.environ, + FABLEFORGE_AUTOMATION_HIDDEN='1', APPDATA=str(work/'profile'), + LOCALAPPDATA=str(work/'local'))) + trace = Path(str(script)+'.log').read_text(encoding='utf-8') + assert 'RESULT PASS' in trace and 'FAIL ' not in trace, trace[-7000:] + else: + result = run('clone', [args.tool.resolve(), 'defs', 'clone', folder/'game.bin', + args.donor, new_name, '--out', bank, '--json']) + report = json.loads(result.stdout) + assert report['donor_index'] == donor and report['entry_index'] == len(before) after = read_bank(output) assert after[:-1] == before and len(after) == len(before)+1 assert after[-1] == dict(before[donor], name=new_name) @@ -88,7 +114,8 @@ def run(label, command): (work/'result.json').write_text(json.dumps({ 'passed': True, 'preserved_records': len(before), 'donor_index': donor, 'new_index': len(before), 'native_hashes_unchanged': hashes, - 'tool_sha256': digest(args.tool.resolve()), 'forge_sha256': digest(args.forge.resolve()) + 'tool_sha256': digest(args.tool.resolve()), 'forge_sha256': digest(args.forge.resolve()), + 'gui_sha256': digest(args.gui.resolve()) if args.gui else None }, indent=2)+'\n', encoding='utf-8') print('PASS:', len(before), 'records and identity rows preserved; exact donor/name appended; Restore passed', flush=True) diff --git a/tools/test_definition_clone_ui.py b/tools/test_definition_clone_ui.py new file mode 100644 index 00000000..08c00830 --- /dev/null +++ b/tools/test_definition_clone_ui.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Exercise clone drafts, exact donor preservation, stale guards and paired Restore.""" +import argparse +import json +import os +from pathlib import Path +import shutil +import struct +import subprocess +import tempfile +import time + +from test_definition_clone import fixture +from test_definition_edit_ui import click +from test_fmp_links import read_bank, crc0 +from test_texture_lookup import texture_bank + +MODES = ('save', 'restore', 'numeric', 'unnamed', 'indexed-duplicate', 'folder', 'pack', + 'empty', 'duplicate', 'cancel', 'confirmation-cancel', 'destination', + 'stale-bank', 'stale-names', 'pool-conflict', 'bounds', 'install-switch', 'frontend', 'script') + + +def main(): + repo = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--gui', type=Path, default=repo/'build/FableForge.exe') + parser.add_argument('--forge', type=Path, default=repo/'build/forge.exe') + parser.add_argument('--cwd', type=Path, default=repo) + parser.add_argument('--work-root', type=Path, default=repo/'build') + parser.add_argument('--only', nargs='+', choices=MODES) + args = parser.parse_args() + args.work_root.resolve().mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(prefix='definition-clone-ui-', dir=args.work_root.resolve())) + print('Evidence retained at', work, flush=True) + cases = 0 + for size, scale in (('1440x900', 1), ('800x600', 1.5)): + for mode in args.only or MODES: + case = work/f'{size}-{mode}' + root = case/'install \u03a9' + folder = fixture(root) + levels = root/'data/Levels/FinalAlbion' + levels.mkdir(parents=True) + (levels/'unused.lev').write_bytes(b'') + textures = root/'data/graphics/pc/textures.big' + texture_bank(textures, [(1, 'Existing', 20)]) + texture_bytes = textures.read_bytes() + bank_name = mode+'.bin' if mode in ('frontend', 'script') else 'game.bin' + game_bytes = (folder/'game.bin').read_bytes() + if bank_name != 'game.bin': + shutil.copyfile(folder/'game.bin', folder/bank_name) + source_bank, source_names = folder/bank_name, folder/'names.bin' + original_bank, original_names = source_bank.read_bytes(), source_names.read_bytes() + original_records = read_bank(root, bank_name) + destination = case/'output \u6f22' + output_root = destination if mode in ('folder', 'pack', 'pool-conflict') else root + bank = output_root/'data/CompiledDefs'/bank_name + names = bank.parent/'names.bin' + pack_bytes = None + if mode == 'pack': + destination.mkdir(parents=True) + pack_bytes = b'{"version":1,"name":"Cloned"}\n' + (destination/'forge_pack.json').write_bytes(pack_bytes) + if mode == 'pool-conflict': + names.parent.mkdir(parents=True) + names.write_bytes(b'INCOMPATIBLE POOL') + donor = 1 if mode == 'numeric' else 2 if mode == 'unnamed' else 3 if mode == 'indexed-duplicate' else 0 + selector = '1' if mode == 'numeric' else str(donor) + new_name = 'GUI_CLONE_CHECK' + accepted = mode in ('save', 'restore', 'numeric', 'unnamed', 'indexed-duplicate', 'folder', 'pack', 'frontend', 'script') + lines = ['wait_maps', 'wait_ready', f'set uiscale {scale}'] + if mode in ('folder', 'pool-conflict'): + lines += [f'set saveroot {destination.as_posix()}'] + elif mode == 'pack': + lines += [f'pack_dest {destination.as_posix()}'] + lines += ['assets_tab 6', 'wait_definitions', 'frames 3'] + if bank_name != 'game.bin': + lines += ['definition_bank '+bank_name, 'wait_definitions', 'frames 3'] + lines += [f'definition_select {selector}', 'frames 3'] + lines += click('btn_definition_clone') + ['assert_state definition_editing 1', + 'assert_state definition_cloning 1', 'definition_bank_refused '+('script.bin' if bank_name == 'frontend.bin' else 'frontend.bin'), + 'definition_select_refused 0', 'definition_clone_refused'] + if mode != 'empty': + value = 'DONOR' if mode == 'duplicate' else new_name + lines += click('input_definition_edit_value') + ['input_text '+value, 'frames 3'] + barrier = case/'draft.png' + if mode in ('stale-bank', 'stale-names'): + lines += [f'screenshot {barrier.as_posix()}', 'frames 300'] + if mode == 'destination': + lines += [f'set saveroot {destination.as_posix()}', 'wait_definitions', 'frames 3'] + if mode == 'bounds': + lines += ['definition_value_refused '+('X'*128), 'definition_value_refused BAD\u00e9'] + if mode == 'install-switch': + fixture(case/'other-install') + lines += [f'scan_install {(case/"other-install").as_posix()}', 'frames 3', + 'assert_log install: save or discard the current edits before changing folders', + 'assert_state definition_cloning 1'] + if mode in ('cancel', 'bounds', 'install-switch'): + lines += click('btn_definition_edit_cancel') + ['assert_state definition_editing 0', + 'assert_state definition_cloning 0'] + elif mode == 'confirmation-cancel': + lines += click('btn_definition_save') + ['key_down Escape', 'frames 2', + 'key_up Escape', 'frames 3', 'assert_state definition_editing 1', + 'assert_state definition_cloning 1', 'assert_widget btn_definition_save', 'definition_cancel'] + elif accepted: + if mode == 'save': + lines += ['close', 'frames 3', 'assert_state close_prompt 1', + 'click btn_unsaved_cancel', 'frames 3', 'assert_state definition_cloning 1', + 'close', 'frames 3', 'click btn_unsaved_save', 'frames 3', + 'assert_state close_prompt 0', 'assert_state assets_tab 6'] + lines += click('btn_definition_save') + [f'screenshot {(case/"confirmation.png").as_posix()}'] + lines += click('btn_definition_save_confirm') + ['wait_ready', 'wait_definitions', 'frames 3', + 'assert_state definition_editing 0', 'assert_state definition_cloning 0', + 'assert_state definition_selected 5', f'definition_select {new_name}', + f'screenshot {(case/"created.png").as_posix()}'] + if mode == 'restore': + lines += ['restore_all', 'wait_ready', 'wait_definitions', 'frames 3', + 'definition_select_refused '+new_name, 'definition_select 0'] + else: + lines += click('btn_definition_save') + click('btn_definition_save_confirm') + [ + 'assert_state definition_edit_failed 1', 'assert_state definition_editing 1', + 'assert_state definition_cloning 1', 'definition_cancel'] + lines += ['quit'] + script = case/'clone.txt' + script.write_text('\n'.join(lines)+'\n', encoding='utf-8') + env = dict(os.environ, FABLEFORGE_AUTOMATION_HIDDEN='1', + APPDATA=str(case/'profile'), LOCALAPPDATA=str(case/'local')) + expected_bank, expected_names = original_bank, original_names + with (case/'output.log').open('w', encoding='utf-8') as output: + process = subprocess.Popen([str(args.gui.resolve()), '--install', str(root), + '--size', size, '--auto', str(script)], cwd=args.cwd.resolve(), env=env, + stdout=output, stderr=subprocess.STDOUT) + try: + if mode in ('stale-bank', 'stale-names'): + deadline = time.monotonic()+60 + while not barrier.exists() and process.poll() is None and time.monotonic() Date: Sat, 10 Oct 2026 09:02:00 -0600 Subject: [PATCH 02/75] Retain texture context after definition-only folder saves --- docs/EDITOR.md | 3 +++ docs/HANDOFF_GUI_CLONE_2026-10-10.md | 33 ++++++++++++++++++++++++++- docs/ROADMAP_1.0.md | 15 ++++++++++++ gui/app.cpp | 11 +++++++-- tools/test_definition_clone_retail.py | 1 + tools/test_definition_clone_ui.py | 32 ++++++++++++++++++++++---- tools/test_definition_edit_ui.py | 1 + 7 files changed, 89 insertions(+), 7 deletions(-) diff --git a/docs/EDITOR.md b/docs/EDITOR.md index e4b23e20..03f84e42 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1474,6 +1474,9 @@ The new record is selected after saving. Save folders and valid pack destination use the existing destination chooser; Cancel and Close/Install protection retain the draft until you save or discard it. Changed sources or destinations refuse. Restore recovers the bank and its names together, or removes a newly created pair. +For definition-only save folders, preview uses the edited definitions with textures +from the original install when the folder has no texture bank. An existing invalid +output texture bank reports an error; preview fallback does not copy any assets. The CLI provides the same operation with `forge-tools defs clone --out `. Existing outputs must diff --git a/docs/HANDOFF_GUI_CLONE_2026-10-10.md b/docs/HANDOFF_GUI_CLONE_2026-10-10.md index 3503a60c..7c73430f 100644 --- a/docs/HANDOFF_GUI_CLONE_2026-10-10.md +++ b/docs/HANDOFF_GUI_CLONE_2026-10-10.md @@ -74,9 +74,40 @@ at `c3f87a7`. The new GUI-clone development build is not part of that release. ## Next -Commit the scoped GUI continuation, then choose the next roadmap task. The +The scoped GUI continuation is committed at `05a6909` and pushed as +[draft PR 2](https://github.com/BuffJesus/FableForge/pull/2). The development rebuild, compact confirmation inspection and exact extracted archive acceptance are complete. RC2 publication is completed; keep its validated archive unchanged and separate from the new development checkpoint. Continue roadmap work until the user asks to stop. Live gameplay and a clean-machine walkthrough remain separate; no native install write or checkpoint deletion is needed. + +## Partial definition-output texture context follow-up + +A new assertion reproduced a real context regression: after a successful folder +clone, `context_ready=0` despite valid native textures (`definition-clone-ui-b13wujhj`). +`startContextLoad` previously looked only in the definition output for textures, +then replaced the full context with a definitions-only one. It now retains output +definitions but reads native textures when the output texture path is absent. +Existing corrupt banks, directories and filesystem-status errors do not trigger +fallback. No textures are copied or authored by this change. Existing model lookup +already falls back from `modelRoot_` to the native install. + +Eight focused GUI clone cases pass at both sizes for absent/valid/corrupt/directory +output textures (`definition-clone-ui-lovh7klk`); both folder scalar cases pass +(`definition-edit-ui-ewppfr4s`). The retail GUI check now requires a ready context +and passes all 14,764 records/identities and Restore with native hashes unchanged +(`retail-definition-clone-2tz9ubk0`). The earlier expected partial-output warning +is historical; this follow-up removes it when source textures are available. +The GUI rebuild, Python compilation and diff whitespace check pass. A separate +follow-up archive is retained at `dist/development/20261010-def-context/FableForge-0.18.1-dev-win64.zip`, +SHA256 `9be422f1097e0cee629a5450de6e4a8e230217b93da711438b5d66d668d26e3b`. +Packaging metadata records `05a6909` with tracked changes. Exact extracted evidence +`package-smoke-knjfbzm7` passes 112 identities, 132 links and four selected workflows +(retail, GUI smoke, all 44 GUI clone cases and all 34 scalar cases), 123 not run, +zero skipped. Packaged retail GUI evidence `retail-definition-clone-rw7m968u` +passes ready-context, 14,764-record preservation, Restore and unchanged native hashes. +The editor help and final acceptance notes were updated after packaging; the +validated archive remains unchanged. The initial clone ZIP stays intact. +Next: Unicode GUI wide-string authoring using the proven UTF-8/UTF-16 core contract. +Broader font coverage remains a separate presentation limitation. diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 94a0bf61..9596f13a 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -2479,3 +2479,18 @@ extracted package pass: 112 identities, 132 links, seven selected workflows, The packaged GUI also passes the independent retail clone/Restore check. This is separate from RC2's unchanged archive; no full-suite or live-game claim. Evidence and resume: [GUI clone handoff](HANDOFF_GUI_CLONE_2026-10-10.md). + +### Partial-output texture context follow-up + +A folder-clone regression reproduced `context_ready=0` after writing only compiled +definitions. Context loading now combines authored definitions with native textures +when output textures are absent. Existing corrupt banks, directories or status +errors do not fall back. Eight focused clone cases at both sizes, two scalar-folder +cases and the 14,764-record retail clone/Restore check pass, including a ready-context +assertion and unchanged input hashes. No texture files are copied or authored. +Exact extracted follow-up acceptance passes 112 identities, 132 links and four +selected workflows (including 44 clone/34 scalar cases), 123 not run, zero skips. +Archive SHA256: `9be422f1097e0cee629a5450de6e4a8e230217b93da711438b5d66d668d26e3b`. +The packaged retail GUI check also passes the ready-context requirement and Restore. +The initial clone checkpoint and RC2 archive remain intact. See the GUI clone +handoff for evidence; Unicode GUI wide-string authoring is next. diff --git a/gui/app.cpp b/gui/app.cpp index 13a77a54..003aad06 100644 --- a/gui/app.cpp +++ b/gui/app.cpp @@ -894,10 +894,17 @@ void App::startContextLoad(const std::string& fromRoot) { modelRoot_ = fromRoot.empty() ? installPath_ : fromRoot; ctxPending_ = std::make_shared(); const fs::path root = fromRoot.empty() ? fs::u8path(installPath_) : fs::u8path(fromRoot); + fs::path textures = root / "data" / "graphics" / "pc" / "textures.big"; + // Definition-only save folders keep their authored defs but borrow the + // original install's textures. An existing bad output must still fail. + std::error_code textureStatus; + const bool outputTexturesExist = fs::exists(textures, textureStatus); + if (!fromRoot.empty() && !outputTexturesExist && !textureStatus) + textures = fs::u8path(installPath_) / "data" / "graphics" / "pc" / "textures.big"; auto ctx = ctxPending_; - ctxFuture_ = std::async(std::launch::async, [ctx, root]() { + ctxFuture_ = std::async(std::launch::async, [ctx, root, textures]() { std::string err; - const bool ok = ctx->load(root, root / "data" / "graphics" / "pc" / "textures.big", err); + const bool ok = ctx->load(root, textures, err); // no textures.big (a trimmed install): keep the ENGINE_THEME defs so theme // paint, navigation and blank levels still work without the albedo bake if (!ok) { std::string derr; ctx->loadDefs(root, derr); } diff --git a/tools/test_definition_clone_retail.py b/tools/test_definition_clone_retail.py index 6ab77e04..950ad86e 100644 --- a/tools/test_definition_clone_retail.py +++ b/tools/test_definition_clone_retail.py @@ -73,6 +73,7 @@ def run(label, command, env=None): lines += click('btn_definition_save')+click('btn_definition_save_confirm') lines += ['wait_ready', 'wait_definitions', 'frames 3', 'assert_state definition_cloning 0', + 'assert_state context_ready 1', f'assert_state definition_selected {len(before)}', 'definition_select '+new_name, f'screenshot {(work/"created.png").as_posix()}', 'quit'] diff --git a/tools/test_definition_clone_ui.py b/tools/test_definition_clone_ui.py index 08c00830..c6bc85ed 100644 --- a/tools/test_definition_clone_ui.py +++ b/tools/test_definition_clone_ui.py @@ -15,7 +15,8 @@ from test_fmp_links import read_bank, crc0 from test_texture_lookup import texture_bank -MODES = ('save', 'restore', 'numeric', 'unnamed', 'indexed-duplicate', 'folder', 'pack', +MODES = ('save', 'restore', 'numeric', 'unnamed', 'indexed-duplicate', 'folder', + 'folder-textures', 'folder-bad-textures', 'folder-texture-directory', 'pack', 'empty', 'duplicate', 'cancel', 'confirmation-cancel', 'destination', 'stale-bank', 'stale-names', 'pool-conflict', 'bounds', 'install-switch', 'frontend', 'script') @@ -52,7 +53,8 @@ def main(): original_bank, original_names = source_bank.read_bytes(), source_names.read_bytes() original_records = read_bank(root, bank_name) destination = case/'output \u6f22' - output_root = destination if mode in ('folder', 'pack', 'pool-conflict') else root + folder_mode = mode.startswith('folder') + output_root = destination if folder_mode or mode in ('pack', 'pool-conflict') else root bank = output_root/'data/CompiledDefs'/bank_name names = bank.parent/'names.bin' pack_bytes = None @@ -63,12 +65,23 @@ def main(): if mode == 'pool-conflict': names.parent.mkdir(parents=True) names.write_bytes(b'INCOMPATIBLE POOL') + output_textures = destination/'data/graphics/pc/textures.big' + output_texture_bytes = None + if mode == 'folder-textures': + texture_bank(output_textures, [(2, 'OutputOnly', 20)]) + output_texture_bytes = output_textures.read_bytes() + elif mode == 'folder-bad-textures': + output_textures.parent.mkdir(parents=True) + output_texture_bytes = b'INVALID OUTPUT TEXTURE BANK' + output_textures.write_bytes(output_texture_bytes) + elif mode == 'folder-texture-directory': + output_textures.mkdir(parents=True) donor = 1 if mode == 'numeric' else 2 if mode == 'unnamed' else 3 if mode == 'indexed-duplicate' else 0 selector = '1' if mode == 'numeric' else str(donor) new_name = 'GUI_CLONE_CHECK' - accepted = mode in ('save', 'restore', 'numeric', 'unnamed', 'indexed-duplicate', 'folder', 'pack', 'frontend', 'script') + accepted = folder_mode or mode in ('save', 'restore', 'numeric', 'unnamed', 'indexed-duplicate', 'pack', 'frontend', 'script') lines = ['wait_maps', 'wait_ready', f'set uiscale {scale}'] - if mode in ('folder', 'pool-conflict'): + if folder_mode or mode == 'pool-conflict': lines += [f'set saveroot {destination.as_posix()}'] elif mode == 'pack': lines += [f'pack_dest {destination.as_posix()}'] @@ -112,6 +125,11 @@ def main(): 'assert_state definition_editing 0', 'assert_state definition_cloning 0', 'assert_state definition_selected 5', f'definition_select {new_name}', f'screenshot {(case/"created.png").as_posix()}'] + if folder_mode: + ready = int(mode not in ('folder-bad-textures', 'folder-texture-directory')) + lines += [f'assert_state context_ready {ready}'] + if not ready: + lines += ['assert_log Textures unavailable:'] if mode == 'restore': lines += ['restore_all', 'wait_ready', 'wait_definitions', 'frames 3', 'definition_select_refused '+new_name, 'definition_select 0'] @@ -149,6 +167,12 @@ def main(): trace = Path(str(script)+'.log').read_text(encoding='utf-8') assert result == 0 and 'RESULT PASS' in trace and 'FAIL ' not in trace, trace[-7000:] assert textures.read_bytes() == texture_bytes + if output_texture_bytes is not None: + assert output_textures.read_bytes() == output_texture_bytes + elif mode == 'folder-texture-directory': + assert output_textures.is_dir() + elif folder_mode: + assert not output_textures.exists(), 'preview fallback must not copy textures' if accepted and mode != 'restore': after = read_bank(output_root, bank_name) assert len(after) == 6 and after[:5] == original_records diff --git a/tools/test_definition_edit_ui.py b/tools/test_definition_edit_ui.py index e0ccd464..5845050f 100644 --- a/tools/test_definition_edit_ui.py +++ b/tools/test_definition_edit_ui.py @@ -109,6 +109,7 @@ def main(): else: lines+=['definition_value '+('1' if mode=='noop' else '99'),'definition_save','wait_ready','wait_file_job','wait_definitions','frames 3', 'assert_state definition_editing 0','assert_definition_field RandomVillagerMax '+('1' if mode=='noop' else '99')] + if mode=='folder':lines+=['assert_state context_ready 1'] lines+=['quit'];script=case/'edit.txt';script.write_text('\n'.join(lines)+'\n',encoding='utf-8') env=dict(os.environ,FABLEFORGE_AUTOMATION_HIDDEN='1',APPDATA=str(case/'profile'),LOCALAPPDATA=str(case/'local')) with (case/'output.log').open('w',encoding='utf-8') as output: From 2d8abff713031ddfa07a3bfff68f8b0a6ecea07a Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:14:20 -0600 Subject: [PATCH 03/75] Enable guarded Unicode wide-string definition drafts --- CMakeLists.txt | 5 ++ docs/AUTOMATION.md | 8 ++ docs/EDITOR.md | 6 +- docs/HANDOFF_GUI_UNICODE_2026-10-10.md | 74 +++++++++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/PARITY_BACKLOG.md | 4 +- docs/ROADMAP_1.0.md | 20 ++++- gui/definitionbrowser.cpp | 19 +++-- src/definitionfield.hpp | 25 ++++-- tests/test_definitionfield.cpp | 38 +++++++++ tools/check_all.py | 4 +- tools/test_definition_wide_edit_ui.py | 26 ++++-- tools/test_definition_wide_retail.py | 108 +++++++++++++++++++++++++ tools/test_package.py | 3 + 14 files changed, 316 insertions(+), 26 deletions(-) create mode 100644 docs/HANDOFF_GUI_UNICODE_2026-10-10.md create mode 100644 tests/test_definitionfield.cpp create mode 100644 tools/test_definition_wide_retail.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c6e83fb..098ef53e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -200,6 +200,9 @@ add_test(NAME fableforge_lut_tests COMMAND fableforge_lut_tests) add_executable(fableforge_unicode_tests tests/test_unicode.cpp) target_link_libraries(fableforge_unicode_tests PRIVATE forgecore_slice) add_test(NAME fableforge_unicode_tests COMMAND fableforge_unicode_tests) +add_executable(fableforge_definitionfield_tests tests/test_definitionfield.cpp) +target_link_libraries(fableforge_definitionfield_tests PRIVATE fableforge) +add_test(NAME fableforge_definitionfield_tests COMMAND fableforge_definitionfield_tests) add_executable(fableforge_dialoguetext_tests tests/test_dialoguetext.cpp) target_link_libraries(fableforge_dialoguetext_tests PRIVATE forgecore_slice) @@ -234,6 +237,8 @@ if(WIN32) ${IMGUI_DIR}/imgui_impl_dx11.cpp ) target_include_directories(albion_imgui PUBLIC ${IMGUI_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/vendor/ImGuizmo) + # Keep supplementary Unicode input intact in the GUI and all ImGui consumers. + target_compile_definitions(albion_imgui PUBLIC IMGUI_USE_WCHAR32) target_link_libraries(albion_imgui PUBLIC d3d11 dxgi d3dcompiler dwmapi ole32 shell32 uuid) add_executable(FableForge WIN32 diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md index 63ec5c2d..a59de8aa 100644 --- a/docs/AUTOMATION.md +++ b/docs/AUTOMATION.md @@ -1081,6 +1081,14 @@ Definitions inspection uses `assets_tab 6`, `wait_definitions`, `definition_bank Definition scalar drafts use `definition_edit `, `definition_value `, `definition_save`, and `definition_cancel`. The edit/value/save `_refused` variants check refusals. `definition_editing` and `definition_edit_failed` report draft state. Saving reselects the explicit native index, even when another record has a numeric name. +CWideString drafts accept strict UTF-8 up to 127 bytes (not characters). Real +`input_text` includes supplementary input through 32-bit ImGui characters. C0/C1 +controls refuse at Save; malformed UTF-8 and byte-overflow values refuse in the +automation setter. Narrow strings and clone names retain ASCII-only input. +`definition_wide` independently checks native UTF-16LE bytes and malformed-original +refusal. `definition_wide_retail` drives both window sizes against read-only retail +inputs, compares every record/table/name byte and restores only owned output. + Definition clone drafts use `definition_clone` (or the real `btn_definition_clone` control), `definition_value `, `definition_save` and `definition_cancel`. `definition_clone_refused` checks draft/load refusal; diff --git a/docs/EDITOR.md b/docs/EDITOR.md index 03f84e42..4d434b0f 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1442,15 +1442,15 @@ Game-bank records with a confirmed `GroupDef` reference offer **Open group**. It Definition `CRGBColour` CLI input uses red, green, blue, alpha (0-255; omitted alpha defaults to 255). The writer converts to the native blue, green, red, alpha byte order. Existing unexpected lengths refuse. -Inline CCharString and CWideString drafts accept up to 127 printable ASCII characters, including empty text. Unicode input is filtered. Existing non-ASCII, long, unterminated or opaque-tailed strings remain read-only and exact. The shared writer requires one complete NUL-terminated original string before replacement. Unsupported wide strings and name-table references remain inspectable. +Inline CCharString drafts accept up to 127 printable ASCII characters, including empty text. Non-ASCII narrow input is filtered; legacy byte strings remain read-only and exact. CWideString drafts accept valid Unicode up to 127 UTF-8 bytes, including supplementary characters and empty text. Existing Unicode values within that limit can be edited or saved unchanged. C0/C1 controls, malformed encoding, overlong originals, missing terminators and opaque trailers remain unsupported. The shared writer requires one complete NUL-terminated original string before replacement. -CWideString uses UTF-16LE with a final zero code unit, without a count prefix. The CLI writer currently accepts ASCII text for this type and refuses non-ASCII input; existing Unicode values remain intact until an explicit supported replacement. Odd lengths, missing or embedded terminators and opaque trailers refuse before writes. +CWideString uses UTF-16LE with a final zero code unit, without a count prefix. CLI and GUI edits use strict UTF-8 conversion. Odd lengths, malformed surrogates, missing or embedded terminators and opaque trailers refuse before writes. Unchanged supported drafts retain their exact original bytes. Definitions are grouped by type and confirmed game-bank group. Search includes group names and native indices. Opening a definition expands its tree and reveals the selected row. Confirmed object Graphic fields offer Open model by native ID. Missing or ambiguous IDs preserve the current preview. Definition drafts protect this navigation. -Wide-string CLI edits accept valid UTF-8 and save native UTF-16LE. The GUI currently edits short printable ASCII wide strings; other originals remain inspectable. +Wide-string CLI edits accept valid UTF-8 and save native UTF-16LE. GUI Unicode authoring is part of the development continuation after rc.2. Glyph availability depends on the installed UI font: accented Latin and Greek were inspected, while CJK and emoji currently show replacement glyphs even though input and saved bytes remain intact. Unsupported originals remain inspectable. Terrain-theme texture fields offer Open texture for confirmed positive main-bank IDs. Unsupported and ambiguous targets preserve the current preview; definition drafts protect navigation. diff --git a/docs/HANDOFF_GUI_UNICODE_2026-10-10.md b/docs/HANDOFF_GUI_UNICODE_2026-10-10.md new file mode 100644 index 00000000..50819036 --- /dev/null +++ b/docs/HANDOFF_GUI_UNICODE_2026-10-10.md @@ -0,0 +1,74 @@ +# GUI Unicode definition authoring, 2026-10-10 + +Owned checkout: `C:/Users/Cornelio/AppData/Local/FableForgeDevelopment/definition-clone-gui-20261010`, +branch `feat/definition-clone-gui-20261010`, baseline `a041e20`. The previous GUI +clone and partial-output texture-context work is committed/pushed in draft PR 2. +The release and separately owned dirty D lanes remain unchanged. + +## Implementation + +`CWideString` GUI drafts use the existing strict UTF-8/native NUL-terminated +UTF-16LE converter. Complete valid original strings fitting 127 UTF-8 bytes can +be edited; ASCII and Unicode exact no-ops preserve all bytes and create no backup. +Malformed surrogates, odd lengths, missing/embedded terminators, trailers and +overlong originals remain read-only. C0/C1 controls are refused. Clone names and +narrow strings keep their existing ASCII contract. Only the selected field changes; +all other payloads, identities and the name pool remain exact. Existing transactional +publication, stale-source/destination, running-game and Restore guards are reused. + +ImGui and every linked consumer now share `IMGUI_USE_WCHAR32` so supplementary +characters survive input. GUI renderer tests were rebuilt for that ABI. Draft +initialization now explicitly refuses unsupported values instead of ignoring a +failed setter and retaining an earlier buffer. No core wire-format change. + +The native layout is unchanged: FableWin TransferBinaryOut `01895C50` calls +WriteNullTerminatedWString `030830D0`; `(GetLength+1)*2` bytes, without a count. +Canonical native/retail evidence is recorded in PARITY_BACKLOG/ROADMAP's +2026-10-09 wide-string correction. UI field 3 of entry 5983 is TextValue in retail. + +## Evidence so far + +- Before implementation, the new Unicode raw-byte expectation fails against the + previous GUI: `definition-wide-ui-qddetteh`. +- Rebuilt GUI, ImGui and both renderer tests. Built the new definition-draft unit + target and existing Unicode/definition-decode targets. +- Five targeted CTests pass: definition drafts, Unicode, definition decode, + geometry and picking. Independent expected bytes and malformed/control originals + are covered. The complete 55-target inventory was not run locally. +- Eight focused GUI cases pass at both sizes, including Unicode, supplementary, + unchanged Unicode originals and the 127-byte UTF-8 boundary: + `definition-wide-ui-o6m1ouyb`. +- All 42 expanded wide GUI cases pass: `definition-wide-ui-cu0x_s23`. +- All 24 narrow ASCII GUI cases pass: `definition-string-ui-uvuwofa5`. +- The new repeatable `test_definition_wide_retail.py` passes both sizes at + `retail-definition-wide-ui-qk3gnb0x`. It reads the native install, writes only a + new output, independently compares all 14,764 identities and every unrelated + payload, exact table/name bytes and UTF-16 field delta, then explicit-output + Restore removes the created pair/markers. Native compiled-bank hashes stay exact. +- The compact draft screenshot was inspected. Accented Latin/Greek render; + CJK/emoji show replacement glyphs. Their input and saved bytes are correct. + Wider font coverage remains a separate presentation limitation, not a passed claim. + +The final C1-control tightening was rebuilt; the five targeted CTests pass again +and both invalid GUI cases pass at `definition-wide-ui-iuv25izf`. Python compilation +and diff whitespace checks pass. The new retail workflow is registered in source +and package gates. The archive is retained at +`dist/development/20261010-def-gui-unicode/FableForge-0.18.1-dev-win64.zip`, SHA256 +`ed71aa9bf5ee94ad2d155b24de6fdc71c7af4e457531a4950ec92c53294cd74a`. +Its metadata records baseline `a041e20` with tracked changes at packaging time. +Extracted acceptance passes at `package-smoke-h67gpth1`: 113 file identities, +132 local links and nine selected workflows (retail, GUI smoke, text content, +all 44 clone cases, all 34 scalar cases, all 42 wide cases, both retail wide cases, +all 24 narrow cases and text close prompt), 119 not run, zero skips. +The packaged no-install check separately passes with exit code zero and RESULT PASS +using private profiles; its screenshot is `build/ui/n1_noinstall.png`. The first +PowerShell launch returned before its GUI log existed; a hidden synchronous launch +verified process completion. Calling gh in that same temporary APPDATA could not +find its normal credentials; only that process environment changed, no login/settings. +The updated editor help is in the ZIP; final acceptance/remaining-scope notes are +recorded after packaging without altering the tested bytes. + +The previous head `a041e20` passes Windows CI run 38061993142; the Unicode source +still needs its own pushed-head CI. Next: scoped commit/push, then broader font fallback. +Keep prior archives intact; do not alter published RC2. Broader font fallback +is the next observed presentation issue. Continue Forge work until asked to stop. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index f9af8d7e..8fde22a8 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [GUI definition cloning](HANDOFF_GUI_CLONE_2026-10-10.md) in its owned C development worktree. RC2 is published with a verified asset hash; stable 0.18.0 stays latest. GUI clone source/retail and extracted development-package checks pass. +> Active continuation: [GUI Unicode definition drafts](HANDOFF_GUI_UNICODE_2026-10-10.md) in the owned C development worktree. Source and exact extracted Unicode checks pass; the clone/context head has green Windows CI. RC2 remains published unchanged. Broader UI font fallback is next. # Navigation lane, 2026-09-27 diff --git a/docs/PARITY_BACKLOG.md b/docs/PARITY_BACKLOG.md index f908abb7..c2cd710c 100644 --- a/docs/PARITY_BACKLOG.md +++ b/docs/PARITY_BACKLOG.md @@ -2349,9 +2349,9 @@ Out of scope: live in-engine ReloadSoundBanks, because FableForge does not host **Vanilla:** Dev defs are text: 198 .def and .tpl files in Data/Defs of the dev tree. The editor build can recompile them live: CDefinitionManager::RecompileDefType 0x0305b560, RecompileAllDefs 0x0305b5f0, ClearDefsForRecompile 0x0305f440, ConsoleResetDefinitions 0x018cdb70, ConsoleResetScriptDefinitions 0x018cdc30, CMainGameComponent::ValidateDefinitions 0x018a3ffa. Thing Properties can jump to a thing's def. Retail ships only CompiledDefs (game.bin, names.bin, script.bin, frontend.bin). -**Current:** Assets > Definitions browses all three banks with a searchable type/group tree, confirmed fields and raw fallback. Native indices identify records and groups. It edits scalar, boolean, vector, colour and short printable ASCII string drafts with recovery and stale-source guards. CWideString CLI edits accept valid UTF-8. CDefString drafts choose existing owning-pool byte offsets or -1, with search and duplicate-name choices. Object properties and group/member links navigate definitions; confirmed OBJECT Graphic and ENGINE_THEME texture fields open native-ID previews. Drafts protect navigation and close/install changes. +**Current:** Assets > Definitions browses all three banks with a searchable type/group tree, confirmed fields and raw fallback. Native indices identify records and groups. It edits scalar, boolean, vector, colour and short printable ASCII narrow-string drafts with recovery and stale-source guards. CWideString CLI and development GUI edits accept valid UTF-8; GUI drafts are bounded to 127 UTF-8 bytes and refuse C0/C1 controls or malformed originals. Selected-record GUI cloning preserves exact donor bytes and original reference targets. CDefString drafts choose existing owning-pool byte offsets or -1, with search and duplicate-name choices. Object properties and group/member links navigate definitions; confirmed OBJECT Graphic and ENGINE_THEME texture fields open native-ID previews. Drafts protect navigation and close/install changes. -**Remaining:** Complete the additional field, reference and clone workflows below; GUI Unicode authoring and unsupported inline strings remain pending. +**Remaining:** Complete additional field/reference workflows and multi-record clone retargeting below. Unsupported inline strings and broader UI font coverage remain open; the inspected current font renders Latin/Greek but shows replacement glyphs for CJK/emoji despite exact stored Unicode bytes. (1) Left pane: a tree of definitions grouped by type (OBJECT, BUILDING, CREATURE, and so on), with GroupDef (THING_GROUP) subgroups, from ctx_.groupedDefinitions. Reuse drawDefPalette's search box. Include a bank switch for game.bin, frontend.bin and script.bin. diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 9596f13a..2ef6ea33 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [GUI definition cloning](HANDOFF_GUI_CLONE_2026-10-10.md) in its owned C development worktree. RC2 is published with a verified asset hash; stable 0.18.0 stays latest. GUI clone source/retail and extracted development-package checks pass. +> Active continuation: [GUI Unicode definition drafts](HANDOFF_GUI_UNICODE_2026-10-10.md) in the owned C development worktree. Source and exact extracted Unicode checks pass; the clone/context head has green Windows CI. RC2 remains published unchanged. Broader UI font fallback is next. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there @@ -2494,3 +2494,21 @@ Archive SHA256: `9be422f1097e0cee629a5450de6e4a8e230217b93da711438b5d66d668d26e3 The packaged retail GUI check also passes the ready-context requirement and Restore. The initial clone checkpoint and RC2 archive remain intact. See the GUI clone handoff for evidence; Unicode GUI wide-string authoring is next. + +### GUI Unicode wide-string authoring, 2026-10-10 + +The development GUI edits complete valid CWideString originals and drafts up to +127 UTF-8 bytes using the proven strict native UTF-16LE converter. Supplementary +input is preserved through a shared 32-bit ImGui character ABI. C0/C1 controls, +malformed surrogates/UTF-8, overlong originals and incomplete/opaque-tailed strings +refuse. Narrow strings and clone names remain ASCII-only. Exact no-ops keep original +bytes; transactional publication and all existing draft/write guards remain. +Five targeted CTests, 42 GUI wide cases, 24 narrow neighbors and the both-size +14,764-record retail Unicode delta/Restore check pass. The final control tightening +passes targeted reruns. Exact extracted acceptance passes 113 identities, 132 links, +nine selected workflows, 119 not run and zero skips. Archive SHA256: +`ed71aa9bf5ee94ad2d155b24de6fdc71c7af4e457531a4950ec92c53294cd74a`. +The packaged retail GUI check preserves all identities, unrelated payloads and names, +and passes Restore with native hashes unchanged. Broader font +coverage remains open: Latin/Greek render, but inspected CJK/emoji use replacement +glyphs despite correct stored bytes. See [Unicode handoff](HANDOFF_GUI_UNICODE_2026-10-10.md). diff --git a/gui/definitionbrowser.cpp b/gui/definitionbrowser.cpp index 16f2d6f4..38a3ff49 100644 --- a/gui/definitionbrowser.cpp +++ b/gui/definitionbrowser.cpp @@ -19,6 +19,7 @@ namespace albion::gui { namespace fs=std::filesystem; namespace { int definitionASCIIChar(ImGuiInputTextCallbackData* data) {return data->EventChar<32 || data->EventChar>126;} +int definitionUnicodeChar(ImGuiInputTextCallbackData* data) {return data->EventChar<32 || (data->EventChar>=127 && data->EventChar<=159);} std::string definitionLower(std::string value) { for(auto& c:value) c=char(std::tolower(static_cast(c)));return value; } @@ -121,8 +122,11 @@ void App::browseDefinition(const std::string& name) { setDefinitionBank("game.bin");definitionSearch_[0]=0;definitionType_.clear();definitionFilterDirty_=true;definitionBrowseRequest_=name; } bool App::setDefinitionEditValue(const std::string& value) { - if(!definitionEditing_ || value.size()>=sizeof definitionEditValue_ || - std::any_of(value.begin(),value.end(),[](unsigned char c){return c==0 || c>=128;})) return false; + if(!definitionEditing_ || value.size()>=sizeof definitionEditValue_) return false; + if(definitionEditType_=="CWideString") { + try {std::vector validated;forge::unicode::appendUtf16z(validated,value);} + catch(const std::exception&) {return false;} + } else if(std::any_of(value.begin(),value.end(),[](unsigned char c){return c==0 || c>=128;})) return false; std::snprintf(definitionEditValue_,sizeof definitionEditValue_,"%s",value.c_str());definitionEditConfirm_=false;return true; } bool App::beginDefinitionEdit(size_t field) { @@ -139,7 +143,11 @@ bool App::beginDefinitionEdit(size_t field) { definitionEditExpectedNames_=definitionFile_.originalNames();definitionEditNames_=definitionLoadedNames_;definitionEditBank_=definitionLoadedBank_; definitionEditDestination_=fs::u8path(packDest_.empty()?saveRoot():packDest_)/"data"/"CompiledDefs"/definitionBank_; definitionEditField_=selected.name;definitionEditType_=selected.type;definitionEditing_=true;definitionCloning_=false; - definitionNameSearch_[0]=0;setDefinitionEditValue(definitionwrite::fieldValue(selected));return true; + definitionNameSearch_[0]=0; + if(!setDefinitionEditValue(definitionwrite::fieldValue(selected))) { + definitionEditing_=false;throw std::runtime_error("The field value does not fit a supported definition draft."); + } + return true; } catch(const std::exception& error) {definitionEditError_=error.what();return false;} } bool App::beginDefinitionClone() { @@ -331,9 +339,10 @@ void App::drawDefinitionBrowser(float pad,float inner,float cardInner) { } else ImGui::InputText("##definitionvalue",definitionEditValue_,sizeof definitionEditValue_); } else { const bool text=definitionCloning_ || definitionEditType_=="CCharString" || definitionEditType_=="CWideString"; - if(text) ImGui::TextDisabled("%s",definitionCloning_?"New name (up to 127 ASCII characters)":"Text (up to 127 ASCII characters)"); + const bool wide=definitionEditType_=="CWideString"; + if(text) ImGui::TextDisabled("%s",definitionCloning_?"New name (up to 127 ASCII characters)":wide?"Unicode text (up to 127 UTF-8 bytes)":"Text (up to 127 ASCII characters)"); ImGui::SetNextItemWidth(cardInner);ImGui::InputText("##definitionvalue",definitionEditValue_,sizeof definitionEditValue_, - text?ImGuiInputTextFlags_CallbackCharFilter:0,text?definitionASCIIChar:nullptr); + text?ImGuiInputTextFlags_CallbackCharFilter:0,text?(wide?definitionUnicodeChar:definitionASCIIChar):nullptr); } auto_.registerWidget("input_definition_edit_value"); if(!definitionEditConfirm_) { diff --git a/src/definitionfield.hpp b/src/definitionfield.hpp index 63d37662..e7f8a35e 100644 --- a/src/definitionfield.hpp +++ b/src/definitionfield.hpp @@ -2,6 +2,7 @@ #include "definitionwrite.hpp" #include "forge/defdecode.hpp" #include "forge/defedit.hpp" +#include "forge/unicode.hpp" #include #include #include @@ -10,12 +11,22 @@ inline bool printableString(const forge::defdecode::DecodedField& field) { return field.type=="CCharString" && !field.value.empty() && field.value.size()<=128 && field.value.back()==0 && std::all_of(field.value.begin(),field.value.end()-1,[](unsigned char c){return c>=32 && c<=126;}); } +inline bool printableWideText(const std::string& text) { + if(text.size()>127) return false; + try { + std::vector encoded;forge::unicode::appendUtf16z(encoded,text); + for(size_t i=0;i+2=127 && encoded[i]<=159))) return false; + return true; + } catch(const std::exception&) {return false;} +} inline bool printableWideString(const forge::defdecode::DecodedField& field) { if(field.type!="CWideString" || field.value.size()<2 || field.value.size()>256 || field.value.size()%2 || field.value[field.value.size()-2]!=0 || field.value.back()!=0) return false; - for(size_t i=0;i+2126 || field.value[i+1]!=0) return false; - return true; + try { + size_t cursor=0;const auto text=forge::unicode::readUtf16z(field.value,cursor); + return cursor==field.value.size() && printableWideText(text); + } catch(const std::exception&) {return false;} } inline bool fieldEditable(const forge::defdecode::DecodedField& field) { return !field.name.empty() && ((field.type=="bool" && field.value.size()==1) || @@ -44,7 +55,7 @@ inline std::string fieldValue(const forge::defdecode::DecodedField& field) { const int rgba[]={field.value[2],field.value[1],field.value[0],field.value[3]};return colourValue(rgba); } if(printableWideString(field)) { - std::string text;for(size_t i=0;i+2type=="CCharString" || target->type=="CWideString") && (value.size()>127 || + if(target->type=="CCharString" && (value.size()>127 || std::any_of(value.begin(),value.end(),[](unsigned char c){return c<32 || c>126;}))) throw std::runtime_error("Definition text accepts up to 127 printable ASCII characters."); + if(target->type=="CWideString") { + if(!printableWideText(value)) + throw std::runtime_error("Wide definition text accepts up to 127 UTF-8 bytes without control characters."); + } // Preserve the exact original representation when the draft is unchanged, // including noncanonical true bytes and existing NaN payload bits. if(fieldValue(*target)==value) return false; diff --git a/tests/test_definitionfield.cpp b/tests/test_definitionfield.cpp new file mode 100644 index 00000000..16a30d87 --- /dev/null +++ b/tests/test_definitionfield.cpp @@ -0,0 +1,38 @@ +#include "definitionfield.hpp" +#include + +int main() try { + const auto require=[](bool ok,const char* message) {if(!ok) throw std::runtime_error(message);}; + forge::defdecode::DecodedField field;field.name="TextValue";field.type="CWideString"; + const std::vector>> valid={ + {"",{0,0}}, {"A",{65,0,0,0}}, {"caf\xc3\xa9",{99,0,97,0,102,0,0xe9,0,0,0}}, + {"\xce\xa9",{0xa9,3,0,0}}, {"\xe6\xbc\xa2",{0x22,0x6f,0,0}}, + {"\xf0\x9f\x99\x82",{0x3d,0xd8,0x42,0xde,0,0}} + }; + for(const auto& [text,bytes]:valid) { + field.value=bytes; + require(albion::definitionwrite::fieldEditable(field),"Valid bounded Unicode original refused"); + require(albion::definitionwrite::fieldValue(field)==text,"Draft conversion changed original text"); + } + const std::vector> invalid={ + {},{65},{65,0},{0,0xd8,0,0},{0,0xdc,0,0},{0,0xd8,65,0,0,0}, + {65,0,0,0,66,0,0,0},{0,0,0xff,0xff},{10,0,0,0},{127,0,0,0},{128,0,0,0},{159,0,0,0} + }; + for(const auto& bytes:invalid) { + field.value=bytes; + require(!albion::definitionwrite::fieldEditable(field),"Malformed/control/trailing wide original accepted"); + } + const std::string suffix="\xe6\xbc\xa2\xf0\x9f\x99\x82"; + for(size_t prefix:{size_t(120),size_t(121)}) { + field.value.clear();forge::unicode::appendUtf16z(field.value,std::string(prefix,'X')+suffix); + require(albion::definitionwrite::fieldEditable(field)==(prefix==120),"UTF-8 byte draft limit differs"); + } + field.type="CCharString";field.value={0xe9,0}; + require(!albion::definitionwrite::fieldEditable(field),"Narrow legacy bytes became Unicode editable"); + field.value={65,0};require(albion::definitionwrite::fieldEditable(field),"Narrow ASCII regression"); + const std::vector invalidText={std::string("A\0B",3),"\x80","\xc0\x80","\xc2","\xed\xa0\x80","\xf4\x90\x80\x80","\xc2\x80","\xc2\x9f"}; + for(const auto& text:invalidText) + require(!albion::definitionwrite::printableWideText(text),"Malformed/control UTF-8 draft accepted"); + std::cout<<"Definition draft Unicode, exact originals, malformed data and byte limits: PASS\n"; + return 0; +} catch(const std::exception& error) {std::cerr< Date: Sat, 10 Oct 2026 09:27:10 -0600 Subject: [PATCH 04/75] Add shared system glyph fallback and missing-font UI checks --- docs/AUTOMATION.md | 10 +++ docs/EDITOR.md | 2 +- docs/HANDOFF_GUI_FONTS_2026-10-10.md | 63 +++++++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/PARITY_BACKLOG.md | 2 +- docs/ROADMAP_1.0.md | 20 ++++- gui/app.cpp | 44 +++++++++-- gui/app.hpp | 3 + gui/automation.cpp | 10 +++ tools/check_all.py | 2 + tools/test_definition_wide_retail.py | 4 + tools/test_font_fallback_ui.py | 114 +++++++++++++++++++++++++++ tools/test_package.py | 2 + 13 files changed, 268 insertions(+), 10 deletions(-) create mode 100644 docs/HANDOFF_GUI_FONTS_2026-10-10.md create mode 100644 tools/test_font_fallback_ui.py diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md index a59de8aa..bdbb11a5 100644 --- a/docs/AUTOMATION.md +++ b/docs/AUTOMATION.md @@ -1089,6 +1089,16 @@ automation setter. Narrow strings and clone names retain ASCII-only input. refusal. `definition_wide_retail` drives both window sizes against read-only retail inputs, compares every record/table/name byte and restores only owned output. +`assert_font_glyph ` checks actual source +glyph availability without accepting the replacement glyph; `_missing` checks +absence. Malformed code points, surrogates and unknown roles fail the assertion. +`FABLEFORGE_AUTOMATION_FONT_DIR` overrides the font directory only when automation +is active. `font_fallback` checks all four roles across repeated scale/atlas rebuilds +and system/no-font/primary-only fixtures. Optional absent language/emoji groups are +explicitly reported in its `skipped_glyph_groups`; `--require-wide` requires the +installed CJK and emoji sources for a stronger local proof. No system fonts are +modified or bundled; only primary-font copies in owned scratch fixtures are used. + Definition clone drafts use `definition_clone` (or the real `btn_definition_clone` control), `definition_value `, `definition_save` and `definition_cancel`. `definition_clone_refused` checks draft/load refusal; diff --git a/docs/EDITOR.md b/docs/EDITOR.md index 4d434b0f..91a5d4d5 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1450,7 +1450,7 @@ Definitions are grouped by type and confirmed game-bank group. Search includes g Confirmed object Graphic fields offer Open model by native ID. Missing or ambiguous IDs preserve the current preview. Definition drafts protect this navigation. -Wide-string CLI edits accept valid UTF-8 and save native UTF-16LE. GUI Unicode authoring is part of the development continuation after rc.2. Glyph availability depends on the installed UI font: accented Latin and Greek were inspected, while CJK and emoji currently show replacement glyphs even though input and saved bytes remain intact. Unsupported originals remain inspectable. +Wide-string CLI edits accept valid UTF-8 and save native UTF-16LE. GUI Unicode authoring is part of the development continuation after rc.2. The GUI keeps Segoe UI as its primary font and merges missing glyphs from available Windows language and emoji fonts, sharing their data across roles and scale rebuilds. Accented Latin, Greek, CJK and monochrome emoji were inspected at both sizes. Optional fonts vary by installation; absent glyphs still use replacements without changing input/storage. This is glyph fallback, not a guarantee of every script, text shaping or colour emoji. Unsupported originals remain inspectable. Terrain-theme texture fields offer Open texture for confirmed positive main-bank IDs. Unsupported and ambiguous targets preserve the current preview; definition drafts protect navigation. diff --git a/docs/HANDOFF_GUI_FONTS_2026-10-10.md b/docs/HANDOFF_GUI_FONTS_2026-10-10.md new file mode 100644 index 00000000..8be1801f --- /dev/null +++ b/docs/HANDOFF_GUI_FONTS_2026-10-10.md @@ -0,0 +1,63 @@ +# UI glyph fallback continuation, 2026-10-10 + +Owned development checkout/branch are unchanged from +[Unicode handoff](HANDOFF_GUI_UNICODE_2026-10-10.md), starting HEAD `2d8abff`. +That Unicode commit is pushed in draft PR 2 and its Windows CI run 38062819381 +passes. The preceding clone/context head also passed Windows CI. RC2 remains +published unchanged; stable 0.18.0 remains latest. All prior archives are retained. + +## Implemented + +The compact Unicode screenshot had replacement glyphs for CJK and emoji despite +correct input/storage. `buildFonts` now locates Windows Fonts through the native +Windows-directory API rather than assuming C:/Windows. Segoe UI remains primary. +The first readable available CJK face among Microsoft YaHei, MS Gothic, Yu Gothic +and Malgun Gothic is merged; Segoe UI Emoji (or Symbol if absent) adds missing +glyphs. This applies to body/bold/title/small roles. Primary glyphs keep priority. + +Font bytes are cached once per file in App-owned immutable buffers, shared among +all roles and repeated scale rebuilds. ImGui does not own these bytes; main.cpp +destroys the atlas before App, and rebuild clears the atlas while buffers persist. +Sources over 64 MiB, missing/unreadable files and unavailable Windows paths refuse +loading and retain existing default-font behavior. No font is downloaded, installed, +modified or bundled. The installed system fonts remain external dependencies. + +Automation glyph assertions inspect source availability, not the replacement glyph. +A font-directory override is available only in active automation so missing-font +paths can be tested without touching Windows Fonts or normal user settings. + +## Evidence + +- GUI build passes. Initial full-font role/three-scale cases pass at both sizes: + `font-fallback-ui-k9vti0e7`. Initial retail Unicode GUI checks with actual glyph + assertions pass both sizes: `retail-definition-wide-ui-ueq2zd8h`. Native compiled + hashes, all 14,764 identities/unrelated records, exact names and Restore pass. +- Final source checks include six system/no-font/primary-only cases at both sizes, + all four roles and three scale passes per case: `font-fallback-ui-bgvjh5ba`. + Required CJK and emoji fonts were present; zero skipped glyph groups. Default + and primary-only fixtures explicitly assert missing CJK/emoji while retaining ASCII. +- Inspected compact native and synthetic drafts: 漢 and monochrome smiling/rocket + glyphs are now visible. The previous replacement-glyph finding is historical. +- Python compilation and diff whitespace checks pass. The font workflow is + registered in source and extracted-package gates. Core serializer/ABI behavior + is unchanged from the validated Unicode checkpoint; no new full CTest run. + +The extracted archive is retained at +`dist/development/20261010-ui-fonts/FableForge-0.18.1-dev-win64.zip`, SHA256 +`581cc3201f1382c21bf4c0895b09ae0c8df16d6c4e2217f04d2bbc77517bfe3c`. +Packaging metadata records `2d8abff` with tracked changes. Exact evidence +`package-smoke-5jlsn3sm` passes 114 file identities, 134 local links and eight +selected workflows (retail, GUI smoke, all six font environments, text content, +all 42 wide cases, both retail wide cases, text close prompt and Unicode paths), +121 not run, zero skipped workflows. The font result separately records zero +skipped glyph groups on this machine, with YaHei and Segoe UI Emoji present. +Final notes are updated after packaging; the archive remains unchanged. + +Coverage depends on available font files; fallback is not universal Unicode coverage, +complex-script shaping or colour emoji support. Missing optional glyph groups are +reported in the font test's result JSON. Real clean-machine walkthrough and gameplay +remain separate, unclaimed. Next: commit/push the scoped font continuation, then +the documented definition-reference authoring gap. Start with confirmed GroupDef +indices used by existing browser/group tests; inspect target-family constraints, +preserve unresolved originals on no-op, and refuse invalid changed targets rather +than treating every integer as a definition reference. No new wire-format inference. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 8fde22a8..90dc4cae 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [GUI Unicode definition drafts](HANDOFF_GUI_UNICODE_2026-10-10.md) in the owned C development worktree. Source and exact extracted Unicode checks pass; the clone/context head has green Windows CI. RC2 remains published unchanged. Broader UI font fallback is next. +> Active continuation: [UI glyph fallback](HANDOFF_GUI_FONTS_2026-10-10.md) in the owned C development worktree. Unicode pushed-head CI and exact extracted font acceptance pass. Confirmed definition-reference controls are the next authoring gap. RC2 remains published unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/PARITY_BACKLOG.md b/docs/PARITY_BACKLOG.md index c2cd710c..6136fb70 100644 --- a/docs/PARITY_BACKLOG.md +++ b/docs/PARITY_BACKLOG.md @@ -2351,7 +2351,7 @@ Out of scope: live in-engine ReloadSoundBanks, because FableForge does not host **Current:** Assets > Definitions browses all three banks with a searchable type/group tree, confirmed fields and raw fallback. Native indices identify records and groups. It edits scalar, boolean, vector, colour and short printable ASCII narrow-string drafts with recovery and stale-source guards. CWideString CLI and development GUI edits accept valid UTF-8; GUI drafts are bounded to 127 UTF-8 bytes and refuse C0/C1 controls or malformed originals. Selected-record GUI cloning preserves exact donor bytes and original reference targets. CDefString drafts choose existing owning-pool byte offsets or -1, with search and duplicate-name choices. Object properties and group/member links navigate definitions; confirmed OBJECT Graphic and ENGINE_THEME texture fields open native-ID previews. Drafts protect navigation and close/install changes. -**Remaining:** Complete additional field/reference workflows and multi-record clone retargeting below. Unsupported inline strings and broader UI font coverage remain open; the inspected current font renders Latin/Greek but shows replacement glyphs for CJK/emoji despite exact stored Unicode bytes. +**Remaining:** Complete additional field/reference workflows and multi-record clone retargeting below. Unsupported inline strings remain open. Development UI font fallback now renders inspected CJK and monochrome emoji through installed Windows fonts at both sizes; unavailable optional fonts still use replacements. Universal coverage, complex-script shaping and colour emoji remain separate limitations. (1) Left pane: a tree of definitions grouped by type (OBJECT, BUILDING, CREATURE, and so on), with GroupDef (THING_GROUP) subgroups, from ctx_.groupedDefinitions. Reuse drawDefPalette's search box. Include a bank switch for game.bin, frontend.bin and script.bin. diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 2ef6ea33..63cf21e4 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [GUI Unicode definition drafts](HANDOFF_GUI_UNICODE_2026-10-10.md) in the owned C development worktree. Source and exact extracted Unicode checks pass; the clone/context head has green Windows CI. RC2 remains published unchanged. Broader UI font fallback is next. +> Active continuation: [UI glyph fallback](HANDOFF_GUI_FONTS_2026-10-10.md) in the owned C development worktree. Unicode pushed-head CI and exact extracted font acceptance pass. Confirmed definition-reference controls are the next authoring gap. RC2 remains published unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there @@ -2512,3 +2512,21 @@ The packaged retail GUI check preserves all identities, unrelated payloads and n and passes Restore with native hashes unchanged. Broader font coverage remains open: Latin/Greek render, but inspected CJK/emoji use replacement glyphs despite correct stored bytes. See [Unicode handoff](HANDOFF_GUI_UNICODE_2026-10-10.md). + +### System glyph fallback, 2026-10-10 + +Windows Fonts are located through the native directory API. Body/bold/title/small +keep their primary Segoe faces and merge available CJK/emoji fallback glyphs. +Immutable file bytes are shared across roles and scale rebuilds; no font is bundled +or installed. Six system/no-font/primary-only cases pass at both sizes across three +scale passes and all roles. CJK and monochrome emoji are now visible in inspected +native/synthetic drafts, superseding the prior replacement-glyph limitation when +the relevant system fonts are installed. Optional absence stays usable and explicit. +Unicode head `2d8abff` passes Windows CI run 38062819381. + +Exact extracted archive passes 114 identities, 134 links and eight selected +workflows, 121 not run and zero skips; the font report has zero skipped glyph groups +on this machine. Archive SHA256: +`581cc3201f1382c21bf4c0895b09ae0c8df16d6c4e2217f04d2bbc77517bfe3c`. +This does not prove universal font coverage, text shaping, colour emoji, gameplay or +another-machine behavior. See [font handoff](HANDOFF_GUI_FONTS_2026-10-10.md). diff --git a/gui/app.cpp b/gui/app.cpp index 003aad06..d544eb98 100644 --- a/gui/app.cpp +++ b/gui/app.cpp @@ -143,17 +143,49 @@ const char* kModeNames[] = {"Textured", "Wireframe", "Walkable", "Height"}; void App::buildFonts(float scale) { ImGuiIO& io = ImGui::GetIO(); io.Fonts->Clear(); - const char* fontsDir = "C:\\Windows\\Fonts\\"; - auto tryFont = [&](const char* file, float size) -> ImFont* { - const std::string p = std::string(fontsDir) + file; - if (!fs::exists(p)) return nullptr; - return io.Fonts->AddFontFromFileTTF(p.c_str(), std::round(size * scale)); + wchar_t windowsDir[MAX_PATH]{}; + const UINT windowsLength=GetWindowsDirectoryW(windowsDir,MAX_PATH); + fs::path fontsDir=windowsLength && windowsLength ImFont* { + if(fontsDir.empty()) return nullptr; + auto found=fontFileData_.find(file); + if(found==fontFileData_.end()) { + const auto path=fontsDir/file;std::error_code error; + const auto bytes=fs::file_size(path,error); + if(error || !bytes || bytes>64*1024*1024) return nullptr; + std::vector data(size_t(bytes),0); + std::ifstream stream(path,std::ios::binary); + if(!stream.read(reinterpret_cast(data.data()),std::streamsize(data.size()))) return nullptr; + found=fontFileData_.emplace(file,std::move(data)).first; + } + ImFontConfig config; + config.FontDataOwnedByAtlas=false; + config.MergeMode=mergeInto!=nullptr;config.DstFont=mergeInto; + std::snprintf(config.Name,sizeof config.Name,"%s",file); + auto& data=found->second; + return io.Fonts->AddFontFromMemoryTTF(data.data(),int(data.size()),std::round(size * scale),&config); + }; + auto addFallbacks = [&](ImFont* font,float size) { + // Preserve primary glyphs; merge only missing glyphs. Optional language + // fonts vary by Windows installation. One CJK face avoids redundant banks. + for(const char* file:{"msyh.ttc","msgothic.ttc","YuGothR.ttc","malgun.ttf"}) + if(tryFont(file,size,font)) break; + if(!tryFont("seguiemj.ttf",size,font)) tryFont("seguisym.ttf",size,font); }; fontBody_ = tryFont("segoeui.ttf", 17.0f); + if (!fontBody_) fontBody_ = io.Fonts->AddFontDefault(); + addFallbacks(fontBody_,17.0f); fontBold_ = tryFont("seguisb.ttf", 17.0f); + if(fontBold_) addFallbacks(fontBold_,17.0f); fontTitle_ = tryFont("seguisb.ttf", 24.0f); + if(fontTitle_) addFallbacks(fontTitle_,24.0f); fontSmall_ = tryFont("segoeui.ttf", 14.0f); - if (!fontBody_) fontBody_ = io.Fonts->AddFontDefault(); + if(fontSmall_) addFallbacks(fontSmall_,14.0f); if (!fontBold_) fontBold_ = fontBody_; if (!fontTitle_) fontTitle_ = fontBody_; if (!fontSmall_) fontSmall_ = fontBody_; diff --git a/gui/app.hpp b/gui/app.hpp index 502cdf0e..36477968 100644 --- a/gui/app.hpp +++ b/gui/app.hpp @@ -1576,6 +1576,9 @@ class App { void setDpiScale(float s) { dpiScale_ = s; } private: ImFont* fontBody_ = nullptr; + // Shared immutable bytes outlive ImGui's sources and avoid one file copy per + // role/scale. main.cpp destroys the atlas before this App is destroyed. + std::map> fontFileData_; ImFont* fontBold_ = nullptr; ImFont* fontTitle_ = nullptr; ImFont* fontSmall_ = nullptr; diff --git a/gui/automation.cpp b/gui/automation.cpp index 478aced2..38336c02 100644 --- a/gui/automation.cpp +++ b/gui/automation.cpp @@ -1672,6 +1672,16 @@ bool Automation::tick(App& app) { else note("ok " + line); ++pc_; } + else if(cmd=="assert_font_glyph" || cmd=="assert_font_glyph_missing") { + std::istringstream args(rest);std::string role,code,extra;args>>role>>code; + uint32_t point=0;const auto parsed=std::from_chars(code.data(),code.data()+code.size(),point,16); + ImFont* font=role=="body"?app.fontBody_:role=="bold"?app.fontBold_:role=="title"?app.fontTitle_:role=="small"?app.fontSmall_:nullptr; + if(!font || parsed.ec!=std::errc{} || parsed.ptr!=code.data()+code.size() || code.empty() || + point>0x10ffff || (point>=0xd800 && point<=0xdfff) || (args>>extra)) fail("Invalid font role or Unicode code point: "+rest); + else if(font->IsGlyphInFont(ImWchar(point))!=(cmd=="assert_font_glyph")) fail("Unexpected font glyph availability: "+rest); + else note("ok "+line); + ++pc_; + } else if (cmd == "assert_log") { // assert_log : some app log line contains the text if (!app.logContains(rest)) fail("assert_log: no log line contains '" + rest + "'"); else note("ok " + line); ++pc_; diff --git a/tools/check_all.py b/tools/check_all.py index c9bf245d..f4c2ebec 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -43,6 +43,8 @@ def main(): if not a.no_build: ok &= run("build", ["cmake", "--build", "build"], capture_output=True) ok &= run("CTest (core, rows, geometry, LZO)", ["ctest", "--test-dir", "build", "--output-on-failure"], capture_output=True) + ok &= run("UI system glyph fallback, missing fonts, roles and atlas rebuilds", + [sys.executable, "tools/test_font_fallback_ui.py"], capture_output=True) for label, script in ( ("read-only world storage CLI and per-entry refusals", "test_worldreview_cli.py"), ("LEV exact-name ambiguity and class findings", "test_levsymbols.py"), diff --git a/tools/test_definition_wide_retail.py b/tools/test_definition_wide_retail.py index 1ebb11da..4b5743b6 100644 --- a/tools/test_definition_wide_retail.py +++ b/tools/test_definition_wide_retail.py @@ -27,6 +27,7 @@ def main(): parser.add_argument('--forge', type=Path, default=repo/'build/forge.exe') parser.add_argument('--cwd', type=Path, default=repo) parser.add_argument('--work-root', type=Path, default=repo/'build') + parser.add_argument('--require-glyphs', action='store_true', help='Require installed CJK/emoji glyph coverage') args = parser.parse_args() native = args.install.resolve() folder = native/'data/CompiledDefs' @@ -60,6 +61,9 @@ def main(): lines = ['wait_maps', 'wait_ready', f'set uiscale {scale}', f'set saveroot {output.as_posix()}', 'assets_tab 6', 'wait_definitions', 'definition_select '+record['name'], 'frames 3'] + if args.require_glyphs: + for role in ('body', 'bold', 'title', 'small'): + lines += [f'assert_font_glyph {role} {point}' for point in ('00e9', '03a9', '6f22', '1f642')] lines += click('btn_definition_edit_3')+click('input_definition_edit_value') lines += ['key_down Ctrl', 'key_down A', 'key_up A', 'key_up Ctrl', 'input_text '+text, 'frames 3', f'screenshot {(case/"draft.png").as_posix()}'] diff --git a/tools/test_font_fallback_ui.py b/tools/test_font_fallback_ui.py new file mode 100644 index 00000000..f9ab743c --- /dev/null +++ b/tools/test_font_fallback_ui.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Check optional system glyphs, font roles and atlas rebuilds without game writes.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import struct +import subprocess +import tempfile + +from test_definition_edit_ui import click +from test_mod_field_details import write_bank +from test_texture_lookup import texture_bank + + +def main(): + repo = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--gui', type=Path, default=repo/'build/FableForge.exe') + parser.add_argument('--cwd', type=Path, default=repo) + parser.add_argument('--work-root', type=Path, default=repo/'build') + parser.add_argument('--require-wide', action='store_true', help='Require both CJK and emoji system fonts') + args = parser.parse_args() + fonts = Path(os.environ.get('WINDIR', 'C:/Windows'))/'Fonts' + cjk = next((fonts/name for name in ('msyh.ttc', 'msgothic.ttc', 'YuGothR.ttc', 'malgun.ttf') + if (fonts/name).is_file()), None) + emoji = fonts/'seguiemj.ttf' + if args.require_wide: + assert cjk and emoji.is_file(), 'required optional language/emoji fonts are absent' + points = ['0041'] + skipped = [] + if (fonts/'segoeui.ttf').is_file() and (fonts/'seguisb.ttf').is_file(): + points += ['00e9', '03a9'] + else: + skipped.append('Latin accent/Greek primary-font coverage') + if cjk: + points += ['6f22'] + else: + skipped.append('CJK: no selected optional language font') + if emoji.is_file(): + points += ['1f642', '1f680'] + else: + skipped.append('supplementary emoji: no Segoe UI Emoji') + args.work_root.resolve().mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(prefix='font-fallback-ui-', dir=args.work_root.resolve())) + print('Evidence retained at', work, flush=True) + fields = [field['name'] for field in json.loads((repo/'docs/re_reference/def_schema.json').read_text())['UI']['fields']] + cases = 0 + for size, scale, mode in ((size, scale, mode) for size, scale in (('1440x900', 1), ('800x600', 1.5)) + for mode in ('system', 'no-fonts', 'primary-only')): + case = work/(size+'-'+mode) + root = case/'install' + values = [bytes(4)]*len(fields) + values[3] = 'caf\u00e9 \u03a9 \u6f22 \U0001f642 \U0001f680'.encode('utf-16-le')+b'\0\0' + write_bank(root, values, definition='UI', record='UNICODE', fields=fields) + levels = root/'data/Levels/FinalAlbion' + levels.mkdir(parents=True) + (levels/'unused.lev').write_bytes(b'') + texture_bank(root/'data/graphics/pc/textures.big', [(1, 'Existing', 20)]) + original = {path: path.read_bytes() for path in root.rglob('*') if path.is_file()} + override = None + if mode != 'system': + override = case/'diagnostic fonts' + override.mkdir() + if mode == 'primary-only': + for name in ('segoeui.ttf', 'seguisb.ttf'): + if (fonts/name).is_file(): + shutil.copyfile(fonts/name, override/name) + current_points = ['0041'] + else: + current_points = points + lines = ['wait_maps', 'wait_ready'] + # Original scale, a second rebuild and return to the intended screenshot + # scale exercise source-buffer lifetime and every font role repeatedly. + for current_scale in (scale, 1.25, scale): + lines += [f'set uiscale {current_scale}', 'frames 5'] + for role in ('body', 'bold', 'title', 'small'): + lines += [f'assert_font_glyph {role} {point}' for point in current_points] + if mode != 'system': + lines += [f'assert_font_glyph_missing {role} {point}' for point in ('6f22', '1f642')] + lines += ['assets_tab 6', 'wait_definitions', 'definition_select UNICODE', 'frames 3'] + lines += click('btn_definition_edit_3') + lines += ['reveal input_definition_edit_value', 'frames 5'] + lines += [f'screenshot {(case/"unicode-draft.png").as_posix()}', 'definition_cancel', 'quit'] + script = case/'fonts.txt' + script.write_text('\n'.join(lines)+'\n', encoding='utf-8') + env = dict(os.environ, FABLEFORGE_AUTOMATION_HIDDEN='1', APPDATA=str(case/'profile'), LOCALAPPDATA=str(case/'local')) + env.pop('FABLEFORGE_AUTOMATION_FONT_DIR', None) + if override: + env['FABLEFORGE_AUTOMATION_FONT_DIR'] = str(override) + result = subprocess.run([str(args.gui.resolve()), '--install', str(root), '--size', size, '--auto', str(script)], + cwd=args.cwd.resolve(), env=env, + capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=120) + (case/'output.log').write_text(result.stdout+result.stderr, encoding='utf-8') + trace = Path(str(script)+'.log').read_text(encoding='utf-8') + assert result.returncode == 0 and 'RESULT PASS' in trace and 'FAIL ' not in trace, trace[-8000:] + assert all(path.read_bytes() == data for path, data in original.items()) + assert not list(root.rglob('*.forge-orig')) and not list(root.rglob('*.forge-created')) + cases += 1 + print(size, mode, 'font roles, Unicode glyphs and repeated atlas rebuilds: PASS', flush=True) + for item in skipped: + print('SKIP:', item, flush=True) + with args.gui.resolve().open('rb') as stream: + gui_hash = hashlib.file_digest(stream, 'sha256').hexdigest() + (work/'result.json').write_text(json.dumps({'passed': True, 'cases': cases, + 'roles': 4, 'atlas_scales_per_case': 3, 'glyphs': points, 'skipped_glyph_groups': skipped, + 'gui_sha256': gui_hash, 'cjk_font': str(cjk) if cjk else None, + 'emoji_font': str(emoji) if emoji.is_file() else None}, indent=2)+'\n', encoding='utf-8') + + +if __name__ == '__main__': + main() diff --git a/tools/test_package.py b/tools/test_package.py index b4824fb6..dd7564f5 100644 --- a/tools/test_package.py +++ b/tools/test_package.py @@ -209,6 +209,8 @@ def captured(value): '--count', '8', '--install', args.root.resolve(), '--out', work / 'exports']) run('gui', [sys.executable, repo / 'tools/ui_smoke.py', '--exe', package / 'FableForge.exe', '--install', args.root.resolve(), '--cwd', cwd]) + run('font_fallback', [sys.executable, repo / 'tools/test_font_fallback_ui.py', + '--gui', package / 'FableForge.exe', '--cwd', cwd, '--work-root', work]) run('text_browser', [sys.executable, repo / 'tools/test_text_browser_ui.py', '--gui', package / 'FableForge.exe', '--cwd', cwd, '--work-root', work]) run('text_backup_restore', [sys.executable, repo / 'tools/test_text_backup_restore.py', From b49dcf2310a3549804673656e621398deb9bd248 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:47:21 -0600 Subject: [PATCH 05/75] Add confirmed GroupDef picker with stale-target publication guards --- docs/AUTOMATION.md | 2 + docs/EDITOR.md | 2 + docs/HANDOFF_GROUP_PICKER_2026-10-10.md | 80 +++++++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 22 ++++- gui/app.hpp | 1 + gui/definitionbrowser.cpp | 35 ++++++- src/definitionfield.hpp | 21 +++- src/definitionread.hpp | 11 +++ tests/test_definitionfield.cpp | 12 ++- tools/check_all.py | 4 + tools/test_definition_group_picker_ui.py | 120 +++++++++++++++++++++++ tools/test_definition_group_retail.py | 78 +++++++++++++++ tools/test_definition_groups_ui.py | 4 +- tools/test_package.py | 5 + 15 files changed, 391 insertions(+), 8 deletions(-) create mode 100644 docs/HANDOFF_GROUP_PICKER_2026-10-10.md create mode 100644 tools/test_definition_group_picker_ui.py create mode 100644 tools/test_definition_group_retail.py diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md index bdbb11a5..f50e8eaf 100644 --- a/docs/AUTOMATION.md +++ b/docs/AUTOMATION.md @@ -1126,3 +1126,5 @@ Definition tree widgets: `definition_type_node_`, `definition_group_ `combo_definition_edit_name` opens the existing-name chooser for four-byte CDefString drafts. `input_definition_name_search` filters pool names or offsets; `definition_name_option_` and `definition_name_option_none` select an existing name or -1. Duplicate pool names have separate offset choices. Saving refuses changed references that are not exact string starts in the captured owning pool. + +`combo_definition_edit_group` opens the confirmed game-bank int32 GroupDef picker. `input_definition_group_search` filters target names or native indices; `definition_group_option_` selects that explicit index, not a name lookup. Only confirmed owner/family combinations participate. Changed wrong-family, missing or stale target identities refuse; unchanged legacy bytes remain a no-op. Synthetic picker and read-only retail-copy workflows cover publication, exact deltas and explicit-output Restore. diff --git a/docs/EDITOR.md b/docs/EDITOR.md index 91a5d4d5..8c9c8d7e 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1440,6 +1440,8 @@ Fixed-size definition fields with unexpected payload lengths remain protected: ` Game-bank records with a confirmed `GroupDef` reference offer **Open group**. It follows the native entry index and requires a group record in the same bank; unsupported or invalid references have no jump. Selecting a group shows its confirmed members; click a member to return to that definition. Membership is read from the current game bank in the background, and unsupported records are omitted. Open drafts keep this navigation disabled. An unchanged fixed-width draft retains the exact original representation, including existing noncanonical boolean bytes and floating-point NaN bits. +Decoded int32 `GroupDef` drafts offer a searched group picker: Object, Building and Creature owners use `THING_GROUP`, and terrain themes use `ENGINE_THEME_GROUP`. Choices show explicit native entry indices, including duplicate or unnamed labels. Changed targets must remain in the same bank and retain their captured type/name/ordinal identity; wrong-family, missing and stale targets refuse before publication. Unresolved original values survive an exact no-op, but there is no invented "no group" sentinel. Unknown schema types remain read-only (including the retail villager template's currently undecoded GroupDef); other integer fields are not inferred to be references. Raw CLI field editing is unchanged. + Definition `CRGBColour` CLI input uses red, green, blue, alpha (0-255; omitted alpha defaults to 255). The writer converts to the native blue, green, red, alpha byte order. Existing unexpected lengths refuse. Inline CCharString drafts accept up to 127 printable ASCII characters, including empty text. Non-ASCII narrow input is filtered; legacy byte strings remain read-only and exact. CWideString drafts accept valid Unicode up to 127 UTF-8 bytes, including supplementary characters and empty text. Existing Unicode values within that limit can be edited or saved unchanged. C0/C1 controls, malformed encoding, overlong originals, missing terminators and opaque trailers remain unsupported. The shared writer requires one complete NUL-terminated original string before replacement. diff --git a/docs/HANDOFF_GROUP_PICKER_2026-10-10.md b/docs/HANDOFF_GROUP_PICKER_2026-10-10.md new file mode 100644 index 00000000..e27a4a87 --- /dev/null +++ b/docs/HANDOFF_GROUP_PICKER_2026-10-10.md @@ -0,0 +1,80 @@ +# Confirmed GroupDef picker continuation, 2026-10-10 + +Owned C development checkout/branch are unchanged from the +[font handoff](HANDOFF_GUI_FONTS_2026-10-10.md), starting clean at `91c9352`. +Font source is pushed in draft PR 2; its CI passed at 38063688200. +RC2 and prior validated development archives remain unchanged. No native writes. + +## Contract and implementation + +The existing reader resolves game-bank four-byte int32 GroupDef entry indices. +A read-only retail inventory found 2,799 Object, 319 Building and 504 Creature +records targeting THING_GROUP; 462 terrain themes target ENGINE_THEME_GROUP. +Another 78 records retain unresolved index-zero values. This is corroborating +retail evidence for the existing placement/theme family contract, not a universal +reference schema. A first inventory assumed a literal ENGINE_THEME schema key and +failed; that category resolves to CEngineThemeDef. Native CLI decoding confirms +GROUND_BLEACHEDLONGGRASS GroupDef is int32, raw `24090000` (entry 2340). + +GUI drafts for confirmed owner categories now offer a searched, virtualized group +chooser with explicit native indices. Numeric, duplicate and unnamed labels do not +reinterpret indices as names. New targets must exist in the same loaded game bank +and have the confirmed family. Changed wrong-type, wrong-family, negative and +out-of-range values refuse before staging/backups. No-op unresolved originals retain +their exact bytes. No invented "no group" sentinel. Other categories/banks keep +their existing raw scalar behavior; no blanket inference from integer fields. +The raw CLI wire-format editor remains unchanged. + +The draft captures eligible target identities (native index, type, name and ordinal, +without target payload copies). Fresh source validation must match the selected +identity before publication. Unrelated target payload edits can compose, while a +renamed/reordered identity refuses. Existing owner/namespace/destination/game/pack +guards and transactional publication/Restore remain in use. Only the four selected +GroupDef bytes change; every other payload, table identity and pool stays exact. + +## Evidence so far + +- GUI and draft unit target build; definition-draft CTest passes. +- Six focused GUI save/family/no-op cases pass at both sizes: + `definition-group-picker-8uwd68nw`. +- The prior exact font archive accepts a wrong-family Object reference; the new + refusal regression fails against it before the guard: + `definition-group-picker-6lrxv_eb`. All writes are owned fixture copies. +- Initial full 28-case picker run passes: `definition-group-picker-tt9gwqh5`. + Existing group/member/edit/Restore neighbors pass: `definition-groups-ui-3cn6o5zr`. +- Four retail picker cases pass for thing/theme owners at both sizes: + `retail-definition-group-picker-ksicjt1p`. Independently compare all 14,764 records, + exact identity tables/names and four-byte field delta; explicit-output Restore + removes created outputs, and native compiled hashes remain exact. +- A newly added renamed-target regression fails against the initial implementation: + `definition-group-picker-0_q_981_`. The final identity-snapshot guard now passes + six focused stale-target/hex-no-op/repeated-save cases at both sizes: + `definition-group-picker-f_7ouafe`; affected CTest also passes. +- Final full 32-case picker run passes: `definition-group-picker-_kr23xgg`. +- Expanded retail probe found CREATURE_VILLAGER_TEMPLATE GroupDef decodes as + unknown type, not int32. It remains read-only; no speculative schema change. + The initial expanded gate stops at that assertion after four successful cases: + `retail-definition-group-picker-knx6_rru`. Final six-case Object/Building/theme + gate passes: `retail-definition-group-picker-xkcf76bo`, preserving all 14,764 + records except each selected four-byte delta, exact identities/names, Restore + and native input hashes. Compact 800x600 picker screenshot inspected. +- New synthetic/retail workflows are registered in source and package gates. + +## Extracted checkpoint + +`dist/development/20261010-def-group-picker/FableForge-0.18.1-dev-win64.zip` +SHA256 `cf6d013ef97fd8fe3fcf128f3dcebf8c156a3d272a32d852bed4750b8cc733dd`. +Packaged from starting HEAD 91c9352 with the scoped feature dirty, after final build +and source checks. Final notes updated after packaging; archive remains unchanged. +`package-smoke-j8vk1zaf` passes 115 file identities, 136 local documentation links +and eight selected workflows: retail eight-map smoke, GUI, existing-name picker, +definition tree, groups, all 34 scalar drafts, all 32 new picker cases and six +retail-copy picker cases. 120 workflows not run; two unrelated corpus skips: +controller/Aeon and F2 inputs absent at the supplied corpus root. These are not +exercised coverage. No full-suite, gameplay or clean-machine claim. +Project dev/format-I/O procedures selected affected tests and required independently +decoded exact deltas, untouched records/identities/names and explicit-output Restore. + +Next: commit/push this scoped milestone. Cross-family rules for other categories, all other reference types, +multi-record clone retargeting, gameplay and clean-machine validation remain separate. +Continue Forge work until asked to stop. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 90dc4cae..59a15b3c 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [UI glyph fallback](HANDOFF_GUI_FONTS_2026-10-10.md) in the owned C development worktree. Unicode pushed-head CI and exact extracted font acceptance pass. Confirmed definition-reference controls are the next authoring gap. RC2 remains published unchanged. +> Active continuation: [confirmed GroupDef picker](HANDOFF_GROUP_PICKER_2026-10-10.md) in the owned C development worktree. Final 32 synthetic and six retail-copy cases plus eight extracted workflows pass. Two unrelated corpus workflows skip explicitly. RC2 remains unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 63cf21e4..6a8442ef 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [UI glyph fallback](HANDOFF_GUI_FONTS_2026-10-10.md) in the owned C development worktree. Unicode pushed-head CI and exact extracted font acceptance pass. Confirmed definition-reference controls are the next authoring gap. RC2 remains published unchanged. +> Active continuation: [confirmed GroupDef picker](HANDOFF_GROUP_PICKER_2026-10-10.md) in the owned C development worktree. Final 32 synthetic and six retail-copy cases plus eight extracted workflows pass. Two unrelated corpus workflows skip explicitly. RC2 remains unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there @@ -2530,3 +2530,23 @@ on this machine. Archive SHA256: `581cc3201f1382c21bf4c0895b09ae0c8df16d6c4e2217f04d2bbc77517bfe3c`. This does not prove universal font coverage, text shaping, colour emoji, gameplay or another-machine behavior. See [font handoff](HANDOFF_GUI_FONTS_2026-10-10.md). + +### Confirmed GroupDef target picker, 2026-10-10 + +Decoded game-bank int32 GroupDef drafts now offer searched native-index choices +for confirmed Thing/terrain-theme families. Wrong-family, missing and stale target +identities refuse before publication; unresolved originals retain exact no-op bytes. +Other integers/banks remain uninterpreted and raw CLI editing is unchanged. +The current retail villager schema does not decode its GroupDef type, so it remains +read-only; no speculative schema change. All 32 synthetic cases and six retail-copy +Object/Building/theme cases pass, independently preserving 14,764 records except +the selected four-byte delta, exact identities/names and Restore; native inputs stay +unchanged. The dev skill selected affected tests; the format-I/O skill required the +independent raw-delta and untouched-byte checks. + +Exact extracted archive passes 115 identities, 136 documentation links and eight +selected workflows (including 32 picker, six retail, 34 scalar and name/tree/group +neighbors), 120 not run and two unrelated missing-corpus skips. SHA256: +`cf6d013ef97fd8fe3fcf128f3dcebf8c156a3d272a32d852bed4750b8cc733dd`. +No full-suite, clean-machine or gameplay claim. See +[GroupDef handoff](HANDOFF_GROUP_PICKER_2026-10-10.md). diff --git a/gui/app.hpp b/gui/app.hpp index 36477968..b6c8bf0b 100644 --- a/gui/app.hpp +++ b/gui/app.hpp @@ -627,6 +627,7 @@ class App { size_t definitionEditIndex_=0; forge::bin::Entry definitionEditExpected_; std::vector> definitionEditExpectedNames_; + std::map definitionEditExpectedGroups_; std::filesystem::path definitionEditNames_,definitionEditBank_,definitionEditDestination_; std::string definitionEditField_,definitionEditType_,definitionEditError_,definitionEditSuccess_; char definitionEditValue_[128]={}; diff --git a/gui/definitionbrowser.cpp b/gui/definitionbrowser.cpp index 38a3ff49..473f5083 100644 --- a/gui/definitionbrowser.cpp +++ b/gui/definitionbrowser.cpp @@ -141,6 +141,13 @@ bool App::beginDefinitionEdit(size_t field) { if(!packDest_.empty()) {if(!modpack::isPack(fs::u8path(packDest_))) throw std::runtime_error("Choose a valid FableForge pack.");(void)modpack::load(fs::u8path(packDest_));} definitionEditIndex_=*definitionSelected_;definitionEditExpected_=definitionFile_.entries()[definitionEditIndex_]; definitionEditExpectedNames_=definitionFile_.originalNames();definitionEditNames_=definitionLoadedNames_;definitionEditBank_=definitionLoadedBank_; + definitionEditExpectedGroups_.clear(); + if(definitionBank_=="game.bin" && selected.name=="GroupDef" && selected.type=="int32" && definitionread::groupFamily(definitionEditExpected_)) + for(size_t i=0;i options;const auto search=definitionLower(definitionNameSearch_); + for(size_t i=0;i>& expectedNames, - const forge::defschema::Schema& schema,const std::string& field,const std::string& value) { + const forge::defschema::Schema& schema,const std::string& field,const std::string& value, + const std::map* expectedGroups=nullptr) { auto file=forge::bin::File::open(sourceNames,sourceBank); if(file.originalNames()!=expectedNames || index>=file.entries().size()) throw std::runtime_error("Definition namespace changed; refresh before editing."); @@ -88,6 +90,22 @@ inline bool replaceField(const std::filesystem::path& sourceNames,const std::fil // including noncanonical true bytes and existing NaN payload bits. if(fieldValue(*target)==value) return false; forge::defedit::setField(file,schema,index,field,value); + // Alternate spellings of an unchanged scalar are still byte-exact no-ops, + // including legacy unresolved references. There is nothing to publish. + if(file.entries()[index].data==expected.data) return false; + if(sourceBank.filename()=="game.bin" && field=="GroupDef" && target->type=="int32" && definitionread::groupFamily(expected)) { + const auto updated=forge::defdecode::decode(file.entries()[index],schema); + const auto found=std::find_if(updated.fields.begin(),updated.fields.end(),[&](const auto& candidate){return candidate.name==field;}); + if(found==updated.fields.end() || found->value.size()!=4) throw std::runtime_error("Group reference has no confirmed four-byte layout."); + const auto& bytes=found->value; + const uint32_t group=uint32_t(bytes[0]) | (uint32_t(bytes[1])<<8) | (uint32_t(bytes[2])<<16) | (uint32_t(bytes[3])<<24); + if(!definitionread::groupChoice(file,expected,group)) throw std::runtime_error("Choose an existing group of the confirmed family in this game bank."); + if(!expectedGroups) throw std::runtime_error("Capture the current group choices before editing."); + const auto captured=expectedGroups->find(group);const auto& current=file.entries()[group]; + if(captured==expectedGroups->end() || captured->second.name!=current.name || + captured->second.definition!=current.definition || captured->second.indexInDefinition!=current.indexInDefinition) + throw std::runtime_error("The selected group identity changed; start the edit again."); + } if(target->type=="CDefString") { const auto updated=forge::defdecode::decode(file.entries()[index],schema); const auto found=std::find_if(updated.fields.begin(),updated.fields.end(),[&](const auto& candidate){return candidate.name==field;}); @@ -101,7 +119,6 @@ inline bool replaceField(const std::filesystem::path& sourceNames,const std::fil } if(!valid) throw std::runtime_error("Choose an existing name from this bank or no name."); } - if(file.entries()[index].data==expected.data) return false; publish(file,sourceNames,destination,true);return true; } } // namespace albion::definitionwrite diff --git a/src/definitionread.hpp b/src/definitionread.hpp index 54a53563..2cbc5439 100644 --- a/src/definitionread.hpp +++ b/src/definitionread.hpp @@ -3,6 +3,17 @@ #include "forge/bin.hpp" #include "forge/defdecode.hpp" namespace albion::definitionread { +// Confirmed placement/theme families only. Other integers or categories do not +// imply a writable reference, even when a browser can display their group. +inline const char* groupFamily(const forge::bin::Entry& owner) { + if(owner.definition=="OBJECT" || owner.definition=="BUILDING" || owner.definition=="CREATURE") return "THING_GROUP"; + if(owner.definition=="ENGINE_THEME") return "ENGINE_THEME_GROUP"; + return nullptr; +} +inline bool groupChoice(const forge::bin::File& file,const forge::bin::Entry& owner,size_t index) { + const auto family=groupFamily(owner); + return family && index groupTarget(const forge::bin::File& file,const forge::defdecode::Decoded& decoded) { diff --git a/tests/test_definitionfield.cpp b/tests/test_definitionfield.cpp index 16a30d87..800d7ab6 100644 --- a/tests/test_definitionfield.cpp +++ b/tests/test_definitionfield.cpp @@ -33,6 +33,16 @@ int main() try { const std::vector invalidText={std::string("A\0B",3),"\x80","\xc0\x80","\xc2","\xed\xa0\x80","\xf4\x90\x80\x80","\xc2\x80","\xc2\x9f"}; for(const auto& text:invalidText) require(!albion::definitionwrite::printableWideText(text),"Malformed/control UTF-8 draft accepted"); - std::cout<<"Definition draft Unicode, exact originals, malformed data and byte limits: PASS\n"; + forge::bin::File groups;groups.addEntry("THING_GROUP","3",{});groups.addEntry("ENGINE_THEME_GROUP","THEMES",{}); + forge::bin::Entry owner;owner.definition="OBJECT"; + for(const auto* category:{"OBJECT","BUILDING","CREATURE"}) { + owner.definition=category; + require(albion::definitionread::groupChoice(groups,owner,0) && !albion::definitionread::groupChoice(groups,owner,1),"Thing group family differs"); + } + owner.definition="ENGINE_THEME"; + require(!albion::definitionread::groupChoice(groups,owner,0) && albion::definitionread::groupChoice(groups,owner,1),"Theme group family differs"); + require(!albion::definitionread::groupChoice(groups,owner,size_t(-1)),"Out-of-range group accepted"); + owner.definition="UNKNOWN";require(!albion::definitionread::groupFamily(owner),"Unknown category inferred"); + std::cout<<"Definition draft Unicode, exact originals, malformed data, byte limits and group families: PASS\n"; return 0; } catch(const std::exception& error) {std::cerr< Date: Sat, 10 Oct 2026 09:55:04 -0600 Subject: [PATCH 06/75] Recover inherited Creature GroupDef type from native evidence --- docs/EDITOR.md | 2 +- docs/HANDOFF_CREATURE_GROUP_2026-10-10.md | 77 ++++++++++++++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 22 ++++++- docs/re_reference/def_schema.json | 2 +- tools/check_all.py | 2 + tools/test_creature_group_schema.py | 79 +++++++++++++++++++++++ tools/test_definition_group_retail.py | 2 +- tools/test_package.py | 3 + 9 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 docs/HANDOFF_CREATURE_GROUP_2026-10-10.md create mode 100644 tools/test_creature_group_schema.py diff --git a/docs/EDITOR.md b/docs/EDITOR.md index 8c9c8d7e..7d32211c 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1440,7 +1440,7 @@ Fixed-size definition fields with unexpected payload lengths remain protected: ` Game-bank records with a confirmed `GroupDef` reference offer **Open group**. It follows the native entry index and requires a group record in the same bank; unsupported or invalid references have no jump. Selecting a group shows its confirmed members; click a member to return to that definition. Membership is read from the current game bank in the background, and unsupported records are omitted. Open drafts keep this navigation disabled. An unchanged fixed-width draft retains the exact original representation, including existing noncanonical boolean bytes and floating-point NaN bits. -Decoded int32 `GroupDef` drafts offer a searched group picker: Object, Building and Creature owners use `THING_GROUP`, and terrain themes use `ENGINE_THEME_GROUP`. Choices show explicit native entry indices, including duplicate or unnamed labels. Changed targets must remain in the same bank and retain their captured type/name/ordinal identity; wrong-family, missing and stale targets refuse before publication. Unresolved original values survive an exact no-op, but there is no invented "no group" sentinel. Unknown schema types remain read-only (including the retail villager template's currently undecoded GroupDef); other integer fields are not inferred to be references. Raw CLI field editing is unchanged. +Decoded int32 `GroupDef` drafts offer a searched group picker: Object, Building and Creature owners use `THING_GROUP`, and terrain themes use `ENGINE_THEME_GROUP`. Choices show explicit native entry indices, including duplicate or unnamed labels. Changed targets must remain in the same bank and retain their captured type/name/ordinal identity; wrong-family, missing and stale targets refuse before publication. Unresolved original values survive an exact no-op, but there is no invented "no group" sentinel. Creature GroupDef's inherited int32 type is confirmed by native transfer evidence and all 517 retail Creature records. Other unknown schema types remain read-only; integer fields are not generally inferred to be references. Raw CLI field editing is unchanged. Definition `CRGBColour` CLI input uses red, green, blue, alpha (0-255; omitted alpha defaults to 255). The writer converts to the native blue, green, red, alpha byte order. Existing unexpected lengths refuse. diff --git a/docs/HANDOFF_CREATURE_GROUP_2026-10-10.md b/docs/HANDOFF_CREATURE_GROUP_2026-10-10.md new file mode 100644 index 00000000..fd428f48 --- /dev/null +++ b/docs/HANDOFF_CREATURE_GROUP_2026-10-10.md @@ -0,0 +1,77 @@ +# Creature GroupDef schema recovery, 2026-10-10 + +Owned development checkout and branch remain as in the +[picker handoff](HANDOFF_GROUP_PICKER_2026-10-10.md), starting clean at b49dcf2. +The preceding turn made concrete progress: committed/pushed guarded GroupDef +choices and completed exact extracted acceptance. CI run 38065042651 is still live. +RC2, previous checkpoints and native game files remain unchanged. + +## Acceptance question and evidence + +Why does retail CREATURE_VILLAGER_TEMPLATE GroupDef decode as unknown, and can +it use the existing confirmed Thing-group picker without guessing other fields? +The literal CREATURE schema is a reconstructed derived-chain alias of +CThingCreatureDef. Its GroupDef was `?`; CThingBaseDef already declares int32. +Do not confuse CThingCreatureDef (placed creature) with CCreatureDef (component). + +Read-only evidence checkout: D:/Documents/FableTLC. No changes there. +FableWin.exe SHA256: +`a9d6d0977d9d7fc8242da4292ae7ed92e926ca80364a95c003d9b9f689e41845`. +Existing exports `ghidra_out/donor_def_transfers.c` lines 9633/9711 and 10156/10326 +show derived CThingCreatureDef transfer calling CThingBaseDef and the latter +passing GroupDef to the signed-long Transfer overload. Retail exports +`ghidra_out/def_transfers.c` show the same base call at CThingCreatureDef 00671600 +and the base implementation at 00676be0. Decompiler offset units differ; these +are not serialized field offsets. + +Bounded Capstone disassembly of original FableWin bytes confirms: +- 01fb3ef8 calls 01844d10, whose jump reaches base transfer 01fd5340. +- 01fd5932 adds 4c to the owner's address; 01fd5935 pushes that address; + 01fd5936 pushes 041536a0, whose bytes are `GroupDef\0`. +- 01fd593e calls 017b28e8, whose jump reaches 018951e0. +- `ghidra_out/fablewin_pdb_names.tsv:926` identifies 018951e0 as + `??$Transfer@J@CPersistContext@@QAEXPBDAAJ@Z` (signed long on Windows). + Its bounded body uses the referenced scalar and returns with two arguments. + +This establishes the inherited type, not every unknown type or runtime rule. +An independent native-bank inventory finds all 517 Creature payloads have a +unique GroupDef tag and exactly four bytes before RenderFadeDistance's tag. +All 517 decoded signed values match those raw bytes, and each resolves to +THING_GROUP or the preserved unresolved zero. Evidence: +`creature-group-schema-d6qh0lxm` under the 20261010 C evidence root. + +## Scoped change and validation + +Only CREATURE.GroupDef changes from `?` to int32. Adjacent unknown fields, +VILLAGE.GroupDef and other categories remain unchanged. No executable writer +or CLI-wire behavior changes; the existing picker and stale/family/no-op guards +now apply to this decoded field. New read-only schema regression checks all +Creature records and keeps adjacent unknown fields unknown. Package invocation +accepts an explicit extracted schema path, not the source checkout schema. +Registered in source and extracted gates. + +Eight retail picker cases now pass for Object, Building, Creature and terrain +theme at both sizes: `retail-definition-group-picker-o9oas4v9`. Each independently +compares all 14,764 records, exact identity rows/names and only the selected +four-byte field delta; explicit-output Restore removes tracked creations and +native hashes remain unchanged. No full-suite, clean-machine or gameplay claim. + +Both affected CTests (defdecode and definitionfield) pass. Compact 800x600 Creature +chooser screenshot inspected. Python syntax and whitespace checks pass. + +Separate checkpoint: +`dist/development/20261010-creature-group-schema/FableForge-0.18.1-dev-win64.zip` +SHA256 `495961b1ab015cc6c6fab6c1508050184be1b9144fa715ad4c93eaad176effb7`. +Packaged at starting HEAD b49dcf2 with this scoped schema/docs/tests dirty; +final evidence notes updated afterward without replacing the archive. +`package-smoke-r0e8438x` passes 116 identities, 138 local documentation links +and five selected workflows: definition tree, groups, all 32 synthetic picker +cases, eight retail-copy edits and the 517-record check using the extracted schema. +124 workflows not run; two unrelated controller/Aeon and F2 missing-corpus skips +remain explicitly unexercised. Previous scalar acceptance is unchanged and retained +in the picker checkpoint, not rerun or claimed as new extracted coverage. + +Next: commit/push. Creature donor +self/appearance/body-reference retargeting remains distinct unfinished work. +The debug-evidence procedure required original-byte/PDB confirmation before +promotion; format-I/O required independent retail boundaries and exact deltas. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 59a15b3c..9c623902 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [confirmed GroupDef picker](HANDOFF_GROUP_PICKER_2026-10-10.md) in the owned C development worktree. Final 32 synthetic and six retail-copy cases plus eight extracted workflows pass. Two unrelated corpus workflows skip explicitly. RC2 remains unchanged. +> Active continuation: [Creature GroupDef schema recovery](HANDOFF_CREATURE_GROUP_2026-10-10.md) in the owned C development worktree. Original-byte/PDB evidence confirms int32; all 517 retail records, eight real picker cases and five extracted workflows pass. Two unrelated corpus skips remain explicit. RC2 remains unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 6a8442ef..67ab6d94 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [confirmed GroupDef picker](HANDOFF_GROUP_PICKER_2026-10-10.md) in the owned C development worktree. Final 32 synthetic and six retail-copy cases plus eight extracted workflows pass. Two unrelated corpus workflows skip explicitly. RC2 remains unchanged. +> Active continuation: [Creature GroupDef schema recovery](HANDOFF_CREATURE_GROUP_2026-10-10.md) in the owned C development worktree. Original-byte/PDB evidence confirms int32; all 517 retail records, eight real picker cases and five extracted workflows pass. Two unrelated corpus skips remain explicit. RC2 remains unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there @@ -2550,3 +2550,23 @@ neighbors), 120 not run and two unrelated missing-corpus skips. SHA256: `cf6d013ef97fd8fe3fcf128f3dcebf8c156a3d272a32d852bed4750b8cc733dd`. No full-suite, clean-machine or gameplay claim. See [GroupDef handoff](HANDOFF_GROUP_PICKER_2026-10-10.md). + +### Creature GroupDef schema recovery, 2026-10-10 + +The prior villager read-only limitation is now resolved for GroupDef only. Original +FableWin bytes, derived/base transfer bodies and PDB names confirm the inherited +signed int32 type. All 517 retail Creature records independently match unique +four-byte boundaries and decoded values. No neighboring unknown types change. +Eight real Object/Building/Creature/theme picker cases pass at both sizes with +exact unrelated records/identities/names, Restore and unchanged native hashes. +Both affected CTests pass; compact Creature picker inspected. + +Separate archive SHA256: +`495961b1ab015cc6c6fab6c1508050184be1b9144fa715ad4c93eaad176effb7`. +Exact extracted acceptance passes 116 identities, 138 links and five selected +workflows (including the extracted-schema 517-record check, eight retail edits, +32 synthetic cases and tree/group neighbors), 124 not run and two unrelated +missing-corpus FMP skips. No full-suite, clean-machine or gameplay claim. +The debug-evidence skill required native/PDB confirmation before promotion; +format-I/O required independent boundaries and unchanged-byte/Restore checks. +See [Creature schema handoff](HANDOFF_CREATURE_GROUP_2026-10-10.md). diff --git a/docs/re_reference/def_schema.json b/docs/re_reference/def_schema.json index 3142e923..ea577c22 100644 --- a/docs/re_reference/def_schema.json +++ b/docs/re_reference/def_schema.json @@ -33257,7 +33257,7 @@ }, { "name": "GroupDef", - "type": "?" + "type": "int32" }, { "name": "RenderFadeDistance", diff --git a/tools/check_all.py b/tools/check_all.py index 2368bb52..d2ab5edb 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -123,6 +123,8 @@ def main(): dialogue = retail_dialogue_big() if dialogue: install_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(dialogue)))) + ok &= run("Creature GroupDef schema, independent boundaries and all retail records", + [sys.executable, "tools/test_creature_group_schema.py", "--install", install_root], capture_output=True) ok &= run("retail group picker families, exact field deltas and Restore", [sys.executable, "tools/test_definition_group_retail.py", "--install", install_root], capture_output=True) ok &= run("retail GUI Unicode definition delta, identity preservation and Restore", diff --git a/tools/test_creature_group_schema.py b/tools/test_creature_group_schema.py new file mode 100644 index 00000000..1e22bb94 --- /dev/null +++ b/tools/test_creature_group_schema.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Check the narrowly recovered Creature GroupDef against read-only retail records.""" +import argparse +import hashlib +import json +from pathlib import Path +import struct +import subprocess +import tempfile +from test_fmp_links import read_bank +from test_mod_field_details import tag + + +def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def main(): + repo = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--install', type=Path, required=True) + parser.add_argument('--tool', type=Path, default=repo/'build/forge-tools.exe') + parser.add_argument('--schema', type=Path, default=repo/'docs/re_reference/def_schema.json') + parser.add_argument('--cwd', type=Path, default=repo) + parser.add_argument('--work-root', type=Path, default=repo/'build') + args = parser.parse_args() + native = args.install.resolve() + inputs = sorted((native/'data/CompiledDefs').glob('*.bin')) + hashes = {str(path): digest(path) for path in inputs} + records = read_bank(native) + schema_path = args.schema.resolve() + schema = json.loads(schema_path.read_text(encoding='utf-8')) + fields = schema['CREATURE']['fields'] + group = [field for field in fields if field['name'] == 'GroupDef'] + assert len(group) == 1 and group[0]['type'] == 'int32' + # Do not silently promote adjacent reconstructed unknown fields. + for name in ('GiveHeroStatChangesOnBeingHit', 'RenderFadeDistance'): + assert next(field for field in fields if field['name'] == name)['type'] == '?' + work = Path(tempfile.mkdtemp(prefix='creature-group-schema-', dir=args.work_root.resolve())) + print('Evidence retained at', work, flush=True) + rows = [] + try: + for index, record in enumerate(records): + if record['definition'] != 'CREATURE': + continue + data = record['data'] + key = struct.pack(' Date: Sat, 10 Oct 2026 10:00:22 -0600 Subject: [PATCH 07/75] Verify Creature CDefs self-reference relocation and audit retail donors --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 74 ++++++++++++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 15 +++- libs/forgecore/include/forge/defrelocate.hpp | 2 +- libs/forgecore/src/defrelocate.cpp | 7 +- tests/test_defrelocate.cpp | 35 ++++++++- tools/audit_creature_self_refs.py | 66 +++++++++++++++++ tools/check_all.py | 2 + 8 files changed, 195 insertions(+), 8 deletions(-) create mode 100644 docs/HANDOFF_CREATURE_CLONE_2026-10-10.md create mode 100644 tools/audit_creature_self_refs.py diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md new file mode 100644 index 00000000..cfd74df3 --- /dev/null +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -0,0 +1,74 @@ +# Creature donor self-reference foundation, 2026-10-10 + +Owned C development checkout/branch unchanged; starting clean at d2678ed. +Previous turn is progress: recovered one Creature schema field, completed exact +source/extracted acceptance and committed/pushed. See +[schema handoff](HANDOFF_CREATURE_GROUP_2026-10-10.md). +RC2 and all existing development archives remain unchanged. No game/native writes. + +## Acceptance question + +The roadmap's distinct creature donor needs its self/owner references retargeted, +while shared component/appearance/body references must retain their existing indices. +Generic defedit::cloneEntry deliberately copies exact bytes and remains unchanged. +This foundation is reference analysis, not a completed creature-clone workflow. + +Canonical core already supplies defrelocate, which rewrites only verified reference +slots and refuses candidates in unknown bytes. Reuse that instead of copying the +older value-wide replacement in FableTLC's DEFS.md. That historical experiment is +evidence for the self-index problem, not permission to rewrite every matching word. +The early APPEND_ALGORITHM.md also has superseded CRC text; use the later DEFS.md +crc0 contract, already implemented by the current paired publisher. + +## Evidence and scope + +Read-only evidence checkout D:/Documents/FableTLC: +`ghidra_out/def_transfers.c` CThingCreatureDef 00671600 calls CThingBaseDef 00676be0 +before derived fields. FableWin donor exports show the same call at 01fb3ef8 via +01844d10 to 01fd5340; original-byte disassembly and executable hash are retained in +the preceding schema handoff. CREATURE is CThingCreatureDef, not the distinct +CCreatureDef component with its appearance/body prefix. + +Current fmplinks::locate already describes CDefs as a five-byte header followed +by count × 12-byte {name CRC, game entry, inherited game entry} triples. +Independent raw retail checks now confirm all 517 Creature records have the +matching count/prefix boundary ending exactly at the Components tag. Only the +two game-index members are reference slots; the CRC/header and all other bytes +stay exact. The existing strict decoded/re-encoded equality check is retained. + +defrelocate now recognizes CREATURE/CThingCreatureDef CDefs prefixes alongside +the already confirmed Object/Building owners. No other unknown field inference. +Malformed counts remain unresolved/refused. Synthetic owner cases exercise both +game-reference members, equal-valued CRC preservation and malformed counts. + +New read-only test-executable mode `--creature-self-audit ` analyzes +each Creature donor mapping only its own native index to the hypothetical append +slot. `tools/audit_creature_self_refs.py` independently reconstructs every CDefs +slot from raw records and checks exact copied payload deltas for accepted analyses. +Native compiled hashes are retained and checked. Registered in source gate only; +the development package does not ship test executables. + +Evidence: `creature-self-refs-tfiziza6` under the 20261010 C evidence root. +517 donors, 1,521 verified self slots; 509 relocation-eligible, eight unresolved. +This is not a claim that 509 clones are published or game-valid. Unresolved matches +fall outside the verified CDefs prefix, often on stride-nine Components rows. +Keep them refused until their domain is established; matching integers alone +do not distinguish a parameter from a definition reference. + +## Next implementation + +Affected relocation/namespace/FMP unit checks all pass in the isolated build; +Python syntax and whitespace checks pass. Picker head b49dcf2 Windows CI +38065042651 completed successfully; schema head d2678ed run 38065569483 is live. +Commit/push this foundation. +Then add a distinct creature-clone API using the existing canonical clone and +paired transaction: precompute verified self-only relocation before mutating; +preserve shared refs, original identities, raw pools and existing name offsets; +refuse unknown collisions/malformed donors before staging. Keep generic exact +clone semantics intact. Prove independent append/delta/Restore on synthetic and +retail copies before GUI promotion. Own appearance/body cloning is still part of +the roadmap, not completed by shared-component self-retargeting. + +The dev/format-I/O/debug-evidence procedures selected the shared relocator, +native inheritance evidence and independent raw-byte audit. No new package, +full-suite, clean-machine or gameplay claim for this foundation. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 9c623902..e1831a08 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [Creature GroupDef schema recovery](HANDOFF_CREATURE_GROUP_2026-10-10.md) in the owned C development worktree. Original-byte/PDB evidence confirms int32; all 517 retail records, eight real picker cases and five extracted workflows pass. Two unrelated corpus skips remain explicit. RC2 remains unchanged. +> Active continuation: [Creature donor self-reference foundation](HANDOFF_CREATURE_CLONE_2026-10-10.md). All 517 donors independently audited: 1,521 verified self slots, 509 relocation-eligible and eight unresolved/refused. Distinct clone publication remains next; generic exact cloning and RC2 remain unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 67ab6d94..e4742b34 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [Creature GroupDef schema recovery](HANDOFF_CREATURE_GROUP_2026-10-10.md) in the owned C development worktree. Original-byte/PDB evidence confirms int32; all 517 retail records, eight real picker cases and five extracted workflows pass. Two unrelated corpus skips remain explicit. RC2 remains unchanged. +> Active continuation: [Creature donor self-reference foundation](HANDOFF_CREATURE_CLONE_2026-10-10.md). All 517 donors independently audited: 1,521 verified self slots, 509 relocation-eligible and eight unresolved/refused. Distinct clone publication remains next; generic exact cloning and RC2 remain unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there @@ -2570,3 +2570,16 @@ missing-corpus FMP skips. No full-suite, clean-machine or gameplay claim. The debug-evidence skill required native/PDB confirmation before promotion; format-I/O required independent boundaries and unchanged-byte/Restore checks. See [Creature schema handoff](HANDOFF_CREATURE_GROUP_2026-10-10.md). + +### Creature donor self-reference foundation, 2026-10-10 + +Canonical defrelocate now recognizes the verified Creature CDefs prefix, retaining +the same strict clean/encode checks and unresolved-candidate refusal. Independent +raw audit covers all 517 retail donors and 1,521 self slots: 509 relocate using only +those slots, eight remain refused for outside-prefix matches. Header/CRC, shared +references and every other payload byte stay exact; native compiled hashes unchanged. +Three affected relocation/namespace/FMP CTests pass. No clone publisher or GUI +feature is claimed for this foundation; generic clone still copies exact bytes. +Distinct self-retargeted cloning and own appearance/body references remain next. +Picker head b49dcf2 passes Windows CI 38065042651. See +[creature donor handoff](HANDOFF_CREATURE_CLONE_2026-10-10.md). diff --git a/libs/forgecore/include/forge/defrelocate.hpp b/libs/forgecore/include/forge/defrelocate.hpp index e19ee2f7..6edc91a1 100644 --- a/libs/forgecore/include/forge/defrelocate.hpp +++ b/libs/forgecore/include/forge/defrelocate.hpp @@ -29,7 +29,7 @@ struct Analysis { std::vector unresolved; }; -// Inspect one decoded-bank payload. Recognizes OBJECT/BUILDING CDefs listings, +// Inspect one decoded-bank payload. Recognizes OBJECT/BUILDING/CREATURE CDefs listings, // verified hero-ability, brain-behaviour and shop-stock reference layouts, and // explicitly typed CDefIndex/CDefString scalar/vector fields. Other integer // fields are not inferred to be references. Every-byte candidate scanning is diff --git a/libs/forgecore/src/defrelocate.cpp b/libs/forgecore/src/defrelocate.cpp index 8b2ca190..a96ed953 100644 --- a/libs/forgecore/src/defrelocate.cpp +++ b/libs/forgecore/src/defrelocate.cpp @@ -68,9 +68,12 @@ Analysis analyze(const bin::Entry& entry, const defschema::Schema& schema, const // Existing fmplinks::locate's native-verified CDefs layout. Discovery is // restricted to verified owners; a similarly sized unknown prefix is not // evidence that it contains a component listing. Retail Building::Transfer - // 00454B8E calls ThingBase::Transfer 00676BE0 (component set before fields). + // 00454B8E and Creature::Transfer 00671600 call ThingBase::Transfer + // 00676BE0 before their fields; the retail Creature prefix uses the same + // counted CDefs triples. This is not CCreatureDef's appearance prefix. if (entry.definition == "OBJECT" || entry.definition == "CObjectDef" || - entry.definition == "BUILDING" || entry.definition == "CThingBuildingDef") { + entry.definition == "BUILDING" || entry.definition == "CThingBuildingDef" || + entry.definition == "CREATURE" || entry.definition == "CThingCreatureDef") { if (decoded.prefixLen >= 5) { const size_t count = size_t(entry.data[3]) | size_t(entry.data[4]) << 8; if (decoded.prefixLen == 5 + count * 12) { diff --git a/tests/test_defrelocate.cpp b/tests/test_defrelocate.cpp index 6dae10fe..363e3f97 100644 --- a/tests/test_defrelocate.cpp +++ b/tests/test_defrelocate.cpp @@ -79,6 +79,33 @@ int audit(char** argv) { } int main(int argc, char** argv) { + if (argc == 4 && std::string_view(argv[1]) == "--creature-self-audit") { + try { + using namespace forge; + const auto folder = std::filesystem::path(argv[2]) / "data" / "CompiledDefs"; + const auto schema = defschema::Schema::load(argv[3]); + const auto source = bin::File::open(folder / "names.bin", folder / "game.bin"); + nlohmann::json rows = nlohmann::json::array(); + for (size_t i = 0; i < source.entries().size(); ++i) { + const auto& entry = source.entries()[i]; + if (entry.definition != "CREATURE") continue; + const defrelocate::Maps maps{{{uint32_t(i),uint32_t(source.entries().size())}}, {}}; + const auto decoded = defdecode::decode(entry,schema); + const auto analysis = defrelocate::analyze(entry,schema,maps); + nlohmann::json row{{"index",i},{"name",entry.name},{"prefix",decoded.prefixLen}, + {"rewrites",nlohmann::json::array()},{"unresolved",nlohmann::json::array()}}; + for (const auto& rewrite : analysis.rewrites) + row["rewrites"].push_back({{"offset",rewrite.offset},{"before",rewrite.before}, + {"after",rewrite.after},{"path",rewrite.path}}); + for (const auto& candidate : analysis.unresolved) + row["unresolved"].push_back({{"offset",candidate.offset},{"value",candidate.value}}); + if (analysis.unresolved.empty()) row["after_hex"] = hex(defrelocate::relocate(entry,schema,maps)); + rows.push_back(std::move(row)); + } + std::cout << nlohmann::json{{"entries",source.entries().size()},{"records",rows}}.dump() << '\n'; + return 0; // Audit reports unresolved donors, never silently accepts them. + } catch (const std::exception& e) { std::fprintf(stderr,"%s\n",e.what()); return 1; } + } if (argc == 7) { try { return audit(argv); } catch (const std::exception& e) { std::fprintf(stderr, "%s\n", e.what()); return 1; } @@ -115,19 +142,21 @@ int main(int argc, char** argv) { const auto ownerSchema = defschema::Schema::loadText(R"({ "BUILDING":{"fields":[{"name":"Value","type":"int32"}]}, "CThingBuildingDef":{"fields":[{"name":"Value","type":"int32"}]}, + "CREATURE":{"fields":[{"name":"Value","type":"int32"}]}, + "CThingCreatureDef":{"fields":[{"name":"Value","type":"int32"}]}, "HERO_ABILITY":{"fields":[{"name":"DummyObject","type":"?"},{"name":"Name","type":"?"}]}, "CHeroAbilityDef":{"fields":[{"name":"DummyObject","type":"?"},{"name":"Name","type":"?"}]}, "UNVERIFIED":{"fields":[{"name":"DummyObject","type":"?"},{"name":"Name","type":"?"}]} })"); - for (const auto* owner : {"BUILDING", "CThingBuildingDef"}) { + for (const auto* owner : {"BUILDING", "CThingBuildingDef", "CREATURE", "CThingCreatureDef"}) { auto bytes = fixture(false); bytes.resize(17); field(bytes, "Value", {12345}); auto expected = fixture(true); expected.resize(17); field(expected, "Value", {12345}); bin::Entry building{owner, "BUILDING_PROBE", 0, bytes}; check(defrelocate::relocate(building, ownerSchema, maps) == expected, - "building component data/inheritance move while CRC/header/trailing scalar stay exact"); + "confirmed owner component data/inheritance move while CRC/header/trailing scalar stay exact"); building.data[3] = 2; rejects([&] { defrelocate::relocate(building, ownerSchema, maps); }, - "building listing with wrong count refuses"); + "confirmed owner listing with wrong count refuses"); } for (const auto* owner : {"HERO_ABILITY", "CHeroAbilityDef"}) { bin::Entry hero{owner, "ABILITY_PROBE", 0, {1,0,1,0,0}}; diff --git a/tools/audit_creature_self_refs.py b/tools/audit_creature_self_refs.py new file mode 100644 index 00000000..14875d35 --- /dev/null +++ b/tools/audit_creature_self_refs.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Read-only donor self-reference audit; unresolved donors are findings, not accepted clones.""" +import argparse +import hashlib +import json +from pathlib import Path +import struct +import subprocess +import tempfile +from test_fmp_links import read_bank +from test_mod_field_details import tag + + +def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream,'sha256').hexdigest() + + +def main(): + repo=Path(__file__).resolve().parents[1] + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--install',type=Path,required=True) + parser.add_argument('--exe',type=Path,default=repo/'build/fableforge_defrelocate_tests.exe') + parser.add_argument('--work-root',type=Path,default=repo/'build') + args=parser.parse_args();native=args.install.resolve() + inputs=sorted((native/'data/CompiledDefs').glob('*.bin')) + hashes={str(path):digest(path) for path in inputs} + records=read_bank(native) + work=Path(tempfile.mkdtemp(prefix='creature-self-refs-',dir=args.work_root.resolve())) + print('Evidence retained at',work,flush=True) + result=subprocess.run([str(args.exe.resolve()),'--creature-self-audit',str(native), + str(repo/'docs/re_reference/def_schema.json')],capture_output=True,text=True,encoding='utf-8',timeout=120) + (work/'audit.log').write_text(result.stderr,encoding='utf-8') + assert result.returncode==0,result.stderr + report=json.loads(result.stdout);assert report['entries']==len(records) + rows=report['records'];assert len(rows)==sum(r['definition']=='CREATURE' for r in records) + accepted=blocked=total_rewrites=0 + for row in rows: + owner=row['index'];record=records[owner];data=record['data'] + count=struct.unpack_from(' Date: Sat, 10 Oct 2026 10:08:04 -0600 Subject: [PATCH 08/75] Add guarded self-retargeted Creature clone API and retail append proof --- CMakeLists.txt | 3 + docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 42 +++++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 12 +++- libs/forgecore/include/forge/defedit.hpp | 7 +++ libs/forgecore/src/defedit.cpp | 23 +++++++ tests/test_creatureclone.cpp | 77 +++++++++++++++++++++++ tools/audit_creature_self_refs.py | 5 ++ tools/check_all.py | 2 +- tools/test_creature_clone_api.py | 71 +++++++++++++++++++++ 10 files changed, 241 insertions(+), 3 deletions(-) create mode 100644 tests/test_creatureclone.cpp create mode 100644 tools/test_creature_clone_api.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 098ef53e..06d600d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,6 +108,9 @@ add_test(NAME fableforge_defnamespace_tests COMMAND fableforge_defnamespace_test add_executable(fableforge_binclone_tests tests/test_binclone.cpp) target_link_libraries(fableforge_binclone_tests PRIVATE forgecore_slice) add_test(NAME fableforge_binclone_tests COMMAND fableforge_binclone_tests) +add_executable(fableforge_creatureclone_tests tests/test_creatureclone.cpp) +target_link_libraries(fableforge_creatureclone_tests PRIVATE forgecore_slice) +add_test(NAME fableforge_creatureclone_tests COMMAND fableforge_creatureclone_tests) add_executable(fableforge_binwrite_tests tests/test_binwrite.cpp) target_link_libraries(fableforge_binwrite_tests PRIVATE forgecore_slice) add_test(NAME fableforge_binwrite_tests COMMAND fableforge_binwrite_tests) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index cfd74df3..3a944009 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -72,3 +72,45 @@ the roadmap, not completed by shared-component self-retargeting. The dev/format-I/O/debug-evidence procedures selected the shared relocator, native inheritance evidence and independent raw-byte audit. No new package, full-suite, clean-machine or gameplay claim for this foundation. + +## Distinct in-memory clone API continuation + +Starting at committed/pushed e5d6e8e, defedit::cloneCreature now requires an explicit +native Creature donor index, clean exact decode/encode and a nonempty counted CDefs +prefix beginning before Components. It precomputes self-only defrelocate output, +so unknown candidates refuse before mutation; canonical clone validates name, +capacity and all existing ordinals before append. Only the appended payload gets +the moved data. Shared component/appearance/body entries, original payloads/slots +and loaded names remain unchanged. Generic exact clone is unchanged. + +New synthetic CTest exercises both self-reference members, equal-valued CRC and +shared references, invalid indices/types/names, wrong counts, unknown matches, +empty listings, refusal non-mutation and unchanged generic cloning. Five targeted +clone/relocation/namespace/FMP CTests pass; final empty-list tightening rerun passes. +CTest inventory is now 56; no full-suite claim. + +Optional test-executable retail mode writes only a fresh directory outside the +native install. It appends each accepted donor, verifies all prior records and +loaded names stay exact, saves and reopens every result. Independent Python gate +compares all 14,764 original records, exact identity rows/header, original raw pool +body, appended native ordinals/new-name crc0 and each self-only four-byte delta. +It verifies refusal counts against the input-hash-bound independent reference audit. +508 clones pass; eight unresolved donors plus empty NULLDEF_CREATURE refuse. +Donors whose CDefs are inherited with no own self slot keep those references exact. +This is a memory/container API check, not transactional publication or Restore. + +Final combined source gate passes at `creature-self-refs-ydr_bbo3` and +`creature-clone-api-enor6zsk`; native input hashes unchanged. First API probe +`creature-clone-api-nia6lhpi` hit a Python assertion KeyError because the independent +reader omits ordinals. Corrected to read the raw identity table; subsequent +`creature-clone-api-weu1kk0z` and final gate pass. No core defect was inferred from +that harness failure. Source gate chains `audit_creature_self_refs.py --clone-api` +with its own verified report, avoiding fixed evidence-path dependencies. + +Schema head d2678ed Windows CI 38065569483 passed; foundation head e5d6e8e +run 38065970643 is live. No new GUI package for this API-only continuation. +Next: commit/push, then expose this distinct mode through the paired definition +publisher/CLI and verify transactional backup/rollback/Restore before GUI promotion. +Own appearance/body cloning remains unfinished roadmap work, not replaced by +shared-component cloning. Project procedures required preservation and independent +raw-delta proof before promoting the in-memory API. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index e1831a08..6fa345bd 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [Creature donor self-reference foundation](HANDOFF_CREATURE_CLONE_2026-10-10.md). All 517 donors independently audited: 1,521 verified self slots, 509 relocation-eligible and eight unresolved/refused. Distinct clone publication remains next; generic exact cloning and RC2 remain unchanged. +> Active continuation: [distinct Creature clone API](HANDOFF_CREATURE_CLONE_2026-10-10.md). Independent retail container gate passes 508 clones and nine refusals with unchanged prior records/pool/native hashes. Paired CLI publication/Restore is next; generic exact cloning and RC2 remain unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index e4742b34..00116522 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [Creature donor self-reference foundation](HANDOFF_CREATURE_CLONE_2026-10-10.md). All 517 donors independently audited: 1,521 verified self slots, 509 relocation-eligible and eight unresolved/refused. Distinct clone publication remains next; generic exact cloning and RC2 remain unchanged. +> Active continuation: [distinct Creature clone API](HANDOFF_CREATURE_CLONE_2026-10-10.md). Independent retail container gate passes 508 clones and nine refusals with unchanged prior records/pool/native hashes. Paired CLI publication/Restore is next; generic exact cloning and RC2 remain unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there @@ -2583,3 +2583,13 @@ feature is claimed for this foundation; generic clone still copies exact bytes. Distinct self-retargeted cloning and own appearance/body references remain next. Picker head b49dcf2 passes Windows CI 38065042651. See [creature donor handoff](HANDOFF_CREATURE_CLONE_2026-10-10.md). + +Distinct defedit::cloneCreature now precomputes strict self-only relocation before +canonical append. Shared entries, loaded pool and all prior slots remain exact; +unknown matches/empty listings refuse before mutation. Generic clone remains an +exact byte copy. Five affected CTests pass (inventory 56, not full suite), and the +independent retail container gate passes 508 appended donors plus nine refusals, +including exact identities, name crc0/ordinals, original raw pool and per-clone +deltas with unchanged native hashes. Paired publication/Restore and GUI promotion +remain next; own appearance/body cloning remains unfinished. No new package or +gameplay claim. Schema head d2678ed CI 38065569483 passes. diff --git a/libs/forgecore/include/forge/defedit.hpp b/libs/forgecore/include/forge/defedit.hpp index 0b2b9f73..6cd41c54 100644 --- a/libs/forgecore/include/forge/defedit.hpp +++ b/libs/forgecore/include/forge/defedit.hpp @@ -71,6 +71,13 @@ SetFieldResult setField(bin::File& file, const defschema::Schema& schema, size_t cloneEntry(bin::File& file, size_t donorIndex, std::string newName); size_t cloneEntry(bin::File& file, std::string_view donorName, std::string newName); +// Distinct Creature clone: relocate only verified self references to the final +// append index, preserving shared components, all prior records and name offsets. +// Requires a clean CREATURE CDefs layout; unresolved matching words refuse before +// mutation. No private appearance/body cloning or publication is performed here. +size_t cloneCreature(bin::File& file, const defschema::Schema& schema, + size_t donorIndex, std::string newName); + // The raw value bytes of one field (everything between its tag and the next // tag), for the types encodeValue does not parse -- maps, vectors, nested // structs. Same entry/field location rules as setField. diff --git a/libs/forgecore/src/defedit.cpp b/libs/forgecore/src/defedit.cpp index d8c37878..6328b036 100644 --- a/libs/forgecore/src/defedit.cpp +++ b/libs/forgecore/src/defedit.cpp @@ -1,4 +1,5 @@ #include "forge/defedit.hpp" +#include "forge/defrelocate.hpp" #include "forge/unicode.hpp" #include @@ -352,6 +353,28 @@ SetFieldResult setField(bin::File& file, const defschema::Schema& schema, size_t cloneEntry(bin::File& file,size_t donorIndex,std::string newName) { return file.cloneEntry(donorIndex,std::move(newName)); } +size_t cloneCreature(bin::File& file,const defschema::Schema& schema,size_t donorIndex,std::string newName) { + if(donorIndex>=file.entries().size() || file.entries()[donorIndex].definition!="CREATURE") + throw std::runtime_error("creature clone: choose a Creature donor by its native index"); + if(file.entries().size()>=UINT32_MAX) + throw std::runtime_error("creature clone: append index exceeds the game reference domain"); + const auto& donor=file.entries()[donorIndex]; + const auto decoded=defdecode::decode(donor,schema); + if(!decoded.clean() || defdecode::encode(decoded)!=donor.data || donor.data.size()<5 || + decoded.fields.empty() || decoded.fields.front().name!="Components" || + std::any_of(decoded.fields.begin(),decoded.fields.end(),[](const auto& field){return !field.tagOk;})) + throw std::runtime_error("creature clone: donor has no verified component layout"); + const size_t count=size_t(donor.data[3]) | size_t(donor.data[4])<<8; + if(!count || decoded.prefixLen!=5+count*12) + throw std::runtime_error("creature clone: donor component listing is empty or malformed"); + const auto landing=uint32_t(file.entries().size()); + // Compute and reject every unsupported candidate before appending. The exact + // clone validates the new name, payload capacity and whole-bank ordinals. + auto payload=defrelocate::relocate(donor,schema,{{{uint32_t(donorIndex),landing}}, {}}); + const auto appended=file.cloneEntry(donorIndex,std::move(newName)); + file.setEntryData(appended,std::move(payload)); + return appended; +} size_t cloneEntry(bin::File& file,std::string_view donorName,std::string newName) { if(donorName.empty()) throw std::runtime_error("defedit clone: use an index for an unnamed donor"); size_t selected=file.entries().size(); diff --git a/tests/test_creatureclone.cpp b/tests/test_creatureclone.cpp new file mode 100644 index 00000000..1121262c --- /dev/null +++ b/tests/test_creatureclone.cpp @@ -0,0 +1,77 @@ +#include "forge/defedit.hpp" +#include "forge/defdecode.hpp" +#include +#include +#include +using Bytes=std::vector; +static void require(bool value,const char* why){if(!value)throw std::runtime_error(why);} +static void word(Bytes& data,uint32_t value){for(int i=0;i<4;++i)data.push_back(uint8_t(value>>(8*i)));} +static Bytes payload(){ + Bytes data{1,0,1,2,0}; + for(auto value:{1u,2u,1u,0x5678u,1u,0u})word(data,value); + word(data,forge::defdecode::fieldTag("Components"));word(data,0);word(data,0); + word(data,forge::defdecode::fieldTag("GroupDef"));word(data,2); + return data; +} +static bool same(const forge::bin::Entry& a,const forge::bin::Entry& b){return a.definition==b.definition && a.name==b.name && a.indexInDefinition==b.indexInDefinition && a.data==b.data;} +static int retail(char** argv){ + namespace fs=std::filesystem; + const auto sourceRoot=fs::weakly_canonical(fs::path(argv[1])); + const auto output=fs::weakly_canonical(fs::path(argv[3])); + require(!fs::exists(output),"retail output must be a fresh owned directory"); + const auto relative=output.lexically_relative(sourceRoot); + require(!relative.empty() && *relative.begin()=="..","retail output must be outside native install"); + const auto source=sourceRoot/"data"/"CompiledDefs"; + auto file=forge::bin::File::open(source/"names.bin",source/"game.bin"); + const auto originals=file.entries();const auto names=file.originalNames(); + const auto schema=forge::defschema::Schema::load(argv[2]); + nlohmann::json rows=nlohmann::json::array(); + for(size_t i=0;i{"","DONOR","BAD\nNAME",std::string(128,'A')}) + refuses([&]{forge::defedit::cloneCreature(file,schema,1,name);}); + auto bad=payload();bad[3]=3;file.setEntryData(1,bad); + refuses([&]{forge::defedit::cloneCreature(file,schema,1,"BAD_COUNT");}); + bad=payload();bad[33]=1;file.setEntryData(1,bad); // Unknown Components count equals donor index. + refuses([&]{forge::defedit::cloneCreature(file,schema,1,"UNKNOWN_MATCH");}); + bad=payload();bad.erase(bad.begin()+5,bad.begin()+29);bad[3]=0;file.setEntryData(1,bad); + refuses([&]{forge::defedit::cloneCreature(file,schema,1,"EMPTY_COMPONENTS");}); + file.setEntryData(1,payload()); + const auto appended=forge::defedit::cloneCreature(file,schema,1,"CREATURE_NEW"); + require(appended==3 && file.entries()[appended].indexInDefinition==1,"creature append identity differs"); + auto expected=payload();expected[13]=3;expected[21]=3; + require(file.entries()[appended].data==expected,"self-only delta differs"); + for(size_t i=0;i Date: Sat, 10 Oct 2026 10:17:47 -0600 Subject: [PATCH 09/75] Publish verified Creature clones through paired CLI transactions --- docs/EDITOR.md | 2 + docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 44 ++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 15 ++- src/definitionclone.hpp | 16 ++- tools/check_all.py | 4 + tools/forge-cli/main.cpp | 26 +++-- tools/test_creature_clone_cli.py | 118 ++++++++++++++++++++++ tools/test_creature_clone_retail.py | 74 ++++++++++++++ tools/test_package.py | 5 + 10 files changed, 294 insertions(+), 12 deletions(-) create mode 100644 tools/test_creature_clone_cli.py create mode 100644 tools/test_creature_clone_retail.py diff --git a/docs/EDITOR.md b/docs/EDITOR.md index 7d32211c..ab697a83 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1442,6 +1442,8 @@ Game-bank records with a confirmed `GroupDef` reference offer **Open group**. It Decoded int32 `GroupDef` drafts offer a searched group picker: Object, Building and Creature owners use `THING_GROUP`, and terrain themes use `ENGINE_THEME_GROUP`. Choices show explicit native entry indices, including duplicate or unnamed labels. Changed targets must remain in the same bank and retain their captured type/name/ordinal identity; wrong-family, missing and stale targets refuse before publication. Unresolved original values survive an exact no-op, but there is no invented "no group" sentinel. Creature GroupDef's inherited int32 type is confirmed by native transfer evidence and all 517 retail Creature records. Other unknown schema types remain read-only; integer fields are not generally inferred to be references. Raw CLI field editing is unchanged. +The separate CLI `forge-tools defs clone-creature --schema --out ` retargets verified self references before paired publication. Shared components/appearance/body definitions remain shared. Empty or unresolved donors refuse before writes; exact `defs clone` and the current GUI donor-clone control still copy exact payloads. Creature-specific GUI controls and private appearance/body cloning remain unfinished. Use owned outputs first; no gameplay validation is claimed. + Definition `CRGBColour` CLI input uses red, green, blue, alpha (0-255; omitted alpha defaults to 255). The writer converts to the native blue, green, red, alpha byte order. Existing unexpected lengths refuse. Inline CCharString drafts accept up to 127 printable ASCII characters, including empty text. Non-ASCII narrow input is filtered; legacy byte strings remain read-only and exact. CWideString drafts accept valid Unicode up to 127 UTF-8 bytes, including supplementary characters and empty text. Existing Unicode values within that limit can be edited or saved unchanged. C0/C1 controls, malformed encoding, overlong originals, missing terminators and opaque trailers remain unsupported. The shared writer requires one complete NUL-terminated original string before replacement. diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 3a944009..5355746a 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -114,3 +114,47 @@ publisher/CLI and verify transactional backup/rollback/Restore before GUI promot Own appearance/body cloning remains unfinished roadmap work, not replaced by shared-component cloning. Project procedures required preservation and independent raw-delta proof before promoting the in-memory API. + +## Paired CLI publication continuation + +Starting clean at committed/pushed 0e4eeaf. `defs clone-creature` requires an +explicit schema, game.bin source/output and the existing exact-name-or-native-index +selector. The shared pair publisher now has an optional explicit Creature donor +contract: recompute a verified clone from fresh source, compare its full appended +payload/type/ordinal, then use unchanged namespace/destination/offline/preparation/ +rollback/recovery guards. Incomplete typed contracts refuse. Generic exact-clone +publication retains its exact-donor requirement. New GUI controls are not added yet. + +Build of forge-tools/forge/FableForge passes (existing u8path warnings retained). +Five affected CTests pass. Final source CLI gate passes 30 selector/name/options, +self-only delta, empty/malformed/unknown donor, bank-scope, aliases, incompatible +pool/orphan, locked-rename rollback, fresh/existing/in-place and explicit Restore +cases: `creature-clone-cli-xfy96ui0`. Initial 22-case gate also passed at +`creature-clone-cli-n4ps4v9r`. Exact generic 35-case neighbor passes: +`definition-clone-7wsyj7g1`, including its owned hidden running-owner/junction helper. + +Six read-only native-input cases pass: `creature-clone-retail-qh1ssvs7`. +Trader clones move exactly three self slots; inherited villager-base clones move +zero and preserve shared references. Fresh/existing paired outputs independently +preserve all 14,764 prior records/identity rows, original raw name-pool body, +new-name crc0 and native ordinal; Restore removes creations or restores exact +originals. Boss-bandit unresolved and empty NULLDEF refuse before output creation. +Native compiled hashes remain unchanged. Registered source and extracted gates; +the retail extracted gate explicitly uses the packaged schema path. + +Exact checkpoint: +`dist/development/20261010-creature-clone-cli/FableForge-0.18.1-dev-win64.zip` +SHA256 `732db7d64d142e7f7f9138d26345aaf25c1a36d651269074bba8c43b4908fc92`. +Packaged at starting HEAD 0e4eeaf with scoped CLI/publisher/docs/tests dirty; +final notes updated afterward without replacing the tested archive. +`package-smoke-ptyqh3vy` passes 117 file identities, 141 local links and six +selected workflows: eight-map retail smoke, GUI smoke, all 35 generic CLI cases, +all 30 Creature CLI cases, six retail-copy Creature cases using packaged schema, +and all 44 existing GUI clone cases. 125 workflows not run; two unrelated missing +controller/Aeon and F2 corpus skips explicitly remain unexercised coverage. +No full-suite, clean-machine or gameplay claim. + +Next: commit/push, then add a distinct guarded Creature GUI clone action using this +paired publisher. Own appearance/body cloning remains unfinished roadmap work; +generic exact GUI/CLI clones remain unchanged. Foundation head e5d6e8e CI +38065970643 passed; API head 0e4eeaf run 38066442857 remains live at this checkpoint. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 6fa345bd..eea29745 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [distinct Creature clone API](HANDOFF_CREATURE_CLONE_2026-10-10.md). Independent retail container gate passes 508 clones and nine refusals with unchanged prior records/pool/native hashes. Paired CLI publication/Restore is next; generic exact cloning and RC2 remain unchanged. +> Active continuation: [paired Creature clone CLI](HANDOFF_CREATURE_CLONE_2026-10-10.md). Thirty CLI cases, six retail-copy cases and six extracted workflows pass, including rollback/Restore and 44 existing GUI clone neighbors. Distinct Creature GUI action is next; own appearance/body cloning remains open. RC2 unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 00116522..2f665f54 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [distinct Creature clone API](HANDOFF_CREATURE_CLONE_2026-10-10.md). Independent retail container gate passes 508 clones and nine refusals with unchanged prior records/pool/native hashes. Paired CLI publication/Restore is next; generic exact cloning and RC2 remain unchanged. +> Active continuation: [paired Creature clone CLI](HANDOFF_CREATURE_CLONE_2026-10-10.md). Thirty CLI cases, six retail-copy cases and six extracted workflows pass, including rollback/Restore and 44 existing GUI clone neighbors. Distinct Creature GUI action is next; own appearance/body cloning remains open. RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there @@ -2593,3 +2593,16 @@ including exact identities, name crc0/ordinals, original raw pool and per-clone deltas with unchanged native hashes. Paired publication/Restore and GUI promotion remain next; own appearance/body cloning remains unfinished. No new package or gameplay claim. Schema head d2678ed CI 38065569483 passes. + +Paired `defs clone-creature` now requires an explicit schema and game-bank paths. +The publisher recomputes the expected self-only clone from fresh source before +using existing namespace, offline, transaction and recovery guards. Generic exact +cloning retains its donor-byte requirement. Thirty synthetic CLI cases, six native +input/owned output cases, five affected CTests and 35 generic CLI neighbors pass. +Native hashes stay unchanged; fresh/existing outputs independently verify all +14,764 prior records, raw identity rows/pool, appended crc0/ordinal and Restore. +Exact extracted checkpoint passes 117 identities, 141 links and six workflows, +including 44 unchanged GUI clone cases; 125 not run and two unrelated missing-corpus +skips. ZIP SHA256 `732db7d64d142e7f7f9138d26345aaf25c1a36d651269074bba8c43b4908fc92`. +Distinct Creature GUI action is next; own appearance/body cloning remains open. +No full-suite, clean-machine or gameplay claim. See the creature donor handoff. diff --git a/src/definitionclone.hpp b/src/definitionclone.hpp index 310555ed..e7b3cb41 100644 --- a/src/definitionclone.hpp +++ b/src/definitionclone.hpp @@ -1,5 +1,6 @@ #pragma once #include "definitionwrite.hpp" +#include "forge/defedit.hpp" #include namespace albion::definitionwrite { inline std::vector cloneBytes(const std::filesystem::path& path) { @@ -20,7 +21,8 @@ inline void verifyCloneRecords(const forge::bin::File& expected,const forge::bin // Existing outputs must still be the exact loaded namespace and record set. inline void publishClone(const forge::bin::File& bank,const std::filesystem::path& sourceNames, const std::filesystem::path& sourceBank,const std::filesystem::path& destination, - bool trackRecovery=true) { + bool trackRecovery=true,const forge::defschema::Schema* creatureSchema=nullptr, + std::optional creatureDonor={}) { namespace fs=std::filesystem; const auto target=fs::absolute(destination).lexically_normal(),names=target.parent_path()/"names.bin"; const auto sameFile=[](const fs::path& a,const fs::path& b) { @@ -39,7 +41,17 @@ inline void publishClone(const forge::bin::File& bank,const std::filesystem::pat const auto& clone=bank.entries().back(); if(clone.name.empty() || std::any_of(source.entries().begin(),source.entries().end(),[&](const auto& e){return e.name==clone.name;})) throw std::runtime_error("definition clone: appended name is not unique"); - if(std::none_of(source.entries().begin(),source.entries().end(),[&](const auto& e){return e.definition==clone.definition && e.data==clone.data;})) + if(bool(creatureSchema)!=creatureDonor.has_value()) + throw std::runtime_error("definition clone: incomplete Creature donor contract"); + if(creatureSchema) { + if(sourceBank.filename()!="game.bin" || target.filename()!="game.bin") + throw std::runtime_error("creature clone: source and destination must be game.bin"); + auto verified=source; + const auto added=forge::defedit::cloneCreature(verified,*creatureSchema,*creatureDonor,clone.name); + const auto& expected=verified.entries()[added]; + if(expected.definition!=clone.definition || expected.indexInDefinition!=clone.indexInDefinition || expected.data!=clone.data) + throw std::runtime_error("creature clone: appended payload differs from verified self-only relocation"); + } else if(std::none_of(source.entries().begin(),source.entries().end(),[&](const auto& e){return e.definition==clone.definition && e.data==clone.data;})) throw std::runtime_error("definition clone: appended payload does not match a donor"); const auto originalNames=cloneBytes(sourceNames); const bool newBank=!fs::exists(target),newNames=!fs::exists(names); diff --git a/tools/check_all.py b/tools/check_all.py index e6cfa9b2..ae5eefb2 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -125,6 +125,8 @@ def main(): install_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(dialogue)))) ok &= run("Creature donor self-reference layout and independent relocation audit", [sys.executable, "tools/audit_creature_self_refs.py", "--install", install_root, "--clone-api"], capture_output=True) + ok &= run("Retail paired Creature clone, self-only deltas and Restore", + [sys.executable, "tools/test_creature_clone_retail.py", "--install", install_root], capture_output=True) ok &= run("Creature GroupDef schema, independent boundaries and all retail records", [sys.executable, "tools/test_creature_group_schema.py", "--install", install_root], capture_output=True) ok &= run("retail group picker families, exact field deltas and Restore", @@ -332,6 +334,8 @@ def main(): [sys.executable, "tools/test_definition_vector_edit_ui.py"], capture_output=True) ok &= run("Definition donor cloning, paired publication, rollback and exact Restore", [sys.executable, "tools/test_definition_clone.py"], capture_output=True) + ok &= run("Creature clone selector, verified self-only payloads, paired rollback and Restore", + [sys.executable, "tools/test_creature_clone_cli.py"], capture_output=True) ok &= run("Definition clone controls, exact donors, stale drafts and paired Restore", [sys.executable, "tools/test_definition_clone_ui.py"], capture_output=True) ok &= run("Definition existing-name picker, exact field deltas, invalid offsets and Restore", diff --git a/tools/forge-cli/main.cpp b/tools/forge-cli/main.cpp index 1584e521..a78fa41c 100644 --- a/tools/forge-cli/main.cpp +++ b/tools/forge-cli/main.cpp @@ -259,6 +259,7 @@ int usage() { " forge ui clone-object-model \n" " [--schema ] --out \n" " forge defs clone --out [--names ] [--json]\n" + " forge defs clone-creature --schema --out [--names ] [--json]\n" " forge defs set-field [--names ] \n" " forge title add --donor \n" " --gui --desc --greet --comment --self \n" @@ -7604,7 +7605,7 @@ int questCard(const std::string& gameRoot, const std::string& schemaPath, // writes the modified bin (+ names.bin) to --out or a ".set-field.bin" // sibling. Untouched entries stay byte-identical (semantic re-serialize). int defsClone(const std::string& binPath,const std::string& namesPath,const std::string& donor, - const std::string& newName,const std::string& outPath,bool asJson) { + const std::string& newName,const std::string& outPath,bool asJson,const std::string& creatureSchemaPath={}) { namespace fs=std::filesystem; const fs::path input=fs::u8path(binPath),names=namesPath.empty()?input.parent_path()/"names.bin":fs::u8path(namesPath); auto file=forge::bin::File::open(names,input);std::optional exact; @@ -7618,8 +7619,15 @@ int defsClone(const std::string& binPath,const std::string& namesPath,const std: if(parsed.ec!=std::errc{} || parsed.ptr!=donor.data()+donor.size()) throw std::runtime_error("defs clone: choose an exact donor name or complete unsigned index"); } - const auto added=forge::defedit::cloneEntry(file,index,newName);const auto output=fs::u8path(outPath); - albion::definitionwrite::publishClone(file,names,input,output,true); + const auto output=fs::u8path(outPath);std::optional creatureSchema; + if(!creatureSchemaPath.empty()) { + if(input.filename()!="game.bin" || output.filename()!="game.bin") + throw std::runtime_error("defs clone-creature: source and destination must be game.bin"); + creatureSchema=forge::defschema::Schema::load(fs::u8path(creatureSchemaPath)); + } + const auto added=creatureSchema?forge::defedit::cloneCreature(file,*creatureSchema,index,newName):forge::defedit::cloneEntry(file,index,newName); + albion::definitionwrite::publishClone(file,names,input,output,true,creatureSchema?&*creatureSchema:nullptr, + creatureSchema?std::optional(index):std::nullopt); const auto& entry=file.entries()[added]; if(asJson) std::puts(json{{"bin",output.string()},{"names",(output.parent_path()/"names.bin").string()}, {"donor_index",index},{"entry_index",added},{"definition",entry.definition},{"entry_name",entry.name}, @@ -14309,19 +14317,21 @@ int main(int argc, char** argv) { return uiSetCardModel(gameRoot, schemaPath, entryName, (uint32_t)meshId, outRoot, inPlace); } - if(args.size()>=2 && args[0]=="defs" && args[1]=="clone") { - std::string names,out;bool asJson=false;std::vector positional; + if(args.size()>=2 && args[0]=="defs" && (args[1]=="clone" || args[1]=="clone-creature")) { + const bool creature=args[1]=="clone-creature"; + std::string names,out,creatureSchema;bool asJson=false;std::vector positional; for(size_t i=2;i=args.size() || args[i+1].rfind("--",0)==0) throw std::runtime_error("defs clone: option requires a path"); - if(argument=="--names") names=args[++i];else out=args[++i]; + if(argument=="--names") names=args[++i];else if(argument=="--schema") creatureSchema=args[++i];else out=args[++i]; } else if(argument=="--json") asJson=true; else if(argument.rfind("--",0)==0) throw std::runtime_error("defs clone: unknown option "+argument); else positional.push_back(argument); } if(positional.size()!=3 || out.empty()) throw std::runtime_error("defs clone: usage: defs clone --out [--names ] [--json]"); - return defsClone(positional[0],names,positional[1],positional[2],out,asJson); + if(creature && creatureSchema.empty()) throw std::runtime_error("defs clone-creature: --schema is required"); + return defsClone(positional[0],names,positional[1],positional[2],out,asJson,creatureSchema); } // forge defs set-field [--names ] // [--schema ] [--out ] [--json] diff --git a/tools/test_creature_clone_cli.py b/tools/test_creature_clone_cli.py new file mode 100644 index 00000000..b86f2be6 --- /dev/null +++ b/tools/test_creature_clone_cli.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Verify self-only Creature clones through paired publication and explicit Restore.""" +import argparse +import json +import os +from pathlib import Path +import shutil +import struct +import subprocess +import tempfile +import zlib +from test_fmp_links import read_bank, crc0 +from test_definition_clone import deny_rename + + +def fixture(root): + folder=root/'data/CompiledDefs';folder.mkdir(parents=True) + def payload(owner,mode='normal'): + data=bytearray(b'\x01\0\x01\x02\0') + for value in (0x12345678,2,owner,0x87654321,owner,0):data+=struct.pack(' Date: Sat, 10 Oct 2026 10:33:57 -0600 Subject: [PATCH 10/75] Expose guarded self-retargeted Creature clone controls --- docs/AUTOMATION.md | 9 ++ docs/EDITOR.md | 4 +- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 41 ++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 15 ++- gui/app.cpp | 1 + gui/app.hpp | 2 +- gui/automation.cpp | 5 +- gui/definitionbrowser.cpp | 36 +++++-- tools/check_all.py | 6 ++ tools/test_creature_clone_cli.py | 7 +- tools/test_creature_clone_retail.py | 30 +++++- tools/test_creature_clone_ui.py | 109 ++++++++++++++++++++++ tools/test_package.py | 6 ++ 14 files changed, 254 insertions(+), 19 deletions(-) create mode 100644 tools/test_creature_clone_ui.py diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md index f50e8eaf..e144adc1 100644 --- a/docs/AUTOMATION.md +++ b/docs/AUTOMATION.md @@ -1109,6 +1109,15 @@ uses `input_definition_edit_value`; `btn_definition_save` and numeric/unnamed/duplicate-name donors, folder/pack outputs, stale sources, invalid names, cancellation, Close/Install guards and paired Restore at both sizes. +The distinct Creature mode uses `definition_clone_creature` or the real +`btn_definition_clone_creature` control, plus the same name/Save/Cancel widgets. +`definition_clone_creature_refused` checks preflight refusal; +`definition_creature_cloning` distinguishes it from an exact clone draft. +`creature_clone_ui` covers 36 synthetic cases across both window sizes. +`creature_clone_gui_retail_1440x900` and `_800x600` independently compare +all prior retail records, verified self-only deltas, paired recovery and explicit +Restore on owned outputs. Appearance/body/components remain shared. + `definition_group` reports the validated game-bank group entry index, or `-`. `btn_definition_group` follows it by index after the current frame, protecting numeric names and the current decoded view. Definition vector drafts expose `input_definition_vector_0`, `_1` and (for C3DVector) `_2` for real coordinate input. `definition_value` accepts the existing comma-separated writer syntax for these drafts. The dedicated `definition_vectors` package workflow independently verifies every other payload byte, exact no-ops and malformed input/layout refusals at both sizes. diff --git a/docs/EDITOR.md b/docs/EDITOR.md index ab697a83..722a0bdf 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1442,7 +1442,9 @@ Game-bank records with a confirmed `GroupDef` reference offer **Open group**. It Decoded int32 `GroupDef` drafts offer a searched group picker: Object, Building and Creature owners use `THING_GROUP`, and terrain themes use `ENGINE_THEME_GROUP`. Choices show explicit native entry indices, including duplicate or unnamed labels. Changed targets must remain in the same bank and retain their captured type/name/ordinal identity; wrong-family, missing and stale targets refuse before publication. Unresolved original values survive an exact no-op, but there is no invented "no group" sentinel. Creature GroupDef's inherited int32 type is confirmed by native transfer evidence and all 517 retail Creature records. Other unknown schema types remain read-only; integer fields are not generally inferred to be references. Raw CLI field editing is unchanged. -The separate CLI `forge-tools defs clone-creature --schema --out ` retargets verified self references before paired publication. Shared components/appearance/body definitions remain shared. Empty or unresolved donors refuse before writes; exact `defs clone` and the current GUI donor-clone control still copy exact payloads. Creature-specific GUI controls and private appearance/body cloning remain unfinished. Use owned outputs first; no gameplay validation is claimed. +The distinct **Clone creature** action appears for game-bank CREATURE records. It retargets only verified self references before paired publication; components, appearance and body definitions stay shared. The draft and Create confirmation make this limitation explicit. Empty, malformed or unresolved donors refuse before a draft opens. Folder/pack destinations, stale-source refusal, Close/Install protection and paired Restore use the existing clone safeguards. **Clone selected definition** still copies the exact donor payload without retargeting. + +The same guarded mode is available through `forge-tools defs clone-creature --schema --out `. Private appearance/body cloning remains unfinished. Use owned outputs first; no gameplay validation is claimed. Definition `CRGBColour` CLI input uses red, green, blue, alpha (0-255; omitted alpha defaults to 255). The writer converts to the native blue, green, red, alpha byte order. Existing unexpected lengths refuse. diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 5355746a..d056a1e3 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -158,3 +158,44 @@ Next: commit/push, then add a distinct guarded Creature GUI clone action using t paired publisher. Own appearance/body cloning remains unfinished roadmap work; generic exact GUI/CLI clones remain unchanged. Foundation head e5d6e8e CI 38065970643 passed; API head 0e4eeaf run 38066442857 remains live at this checkpoint. + +## Distinct Creature GUI continuation + +Starting at 20df28c, the game-bank CREATURE-only Clone creature action preflights +the strict core clone on a disposable copy before opening a draft. Draft mode +captures the donor/source/destination through existing clone guards. Save uses +the distinct core API and paired publisher contract; generic exact clones retain +their original payload contract. Both the draft and compact Create confirmation +state that components stay shared. Appearance/body cloning remains unfinished. + +Source 36-case GUI gate passes at `creature-clone-ui-qrtdjqfc`: both-size actual +controls, numeric/indexed duplicate donors, folders/packs, Cancel/Close/Install, +stale bank/names, invalid names, malformed/unknown/empty donors and bank scope. +Retail GUI gates pass six cases each at `creature-clone-retail-3_4bx33n` +(800x600) and `creature-clone-retail-j06wlilk` (1440x900). Independent raw checks +preserve all prior records/table/pool bytes and only retarget verified self slots; +explicit-output Restore is exact and native compiled hashes remain unchanged. + +After shortening the confirmation so shared-component wording remains visible +at 800x600, the GUI rebuilt successfully. Final compact retail gate passes at +`creature-clone-retail-tjf2twxu`; confirmation screenshot inspected. Focused +save/folder/Close review is retained at `creature-clone-ui-xsg8ss5h`. +Registered synthetic and both-size native GUI source/extracted workflows. +API CI 38066442857 and CLI CI 38067118481 passed. Extracted checkpoint is next; +no full-suite, clean-machine, gameplay or private appearance/body clone claim. + +Focused final GUI gate passes all six cases at `creature-clone-ui-xsg8ss5h`. +Exact checkpoint `dist/development/20261010-creature-clone-gui/FableForge-0.18.1-dev-win64.zip` +SHA256 `51d135c1e95f4fb51cfe4a501f164a7064aa3c1577d4467c137c789a54ffca4e`. +Packaged at starting HEAD 20df28c with scoped GUI/docs/tests dirty; final evidence +notes updated afterward without replacing the tested archive. Extracted gate +`package-smoke-f7lkku3o` passes 117 identities, 141 links and eight workflows: +eight-map retail smoke, GUI smoke, 30 Creature CLI cases, six packaged-schema +retail CLI cases, 36 Creature GUI cases, six retail GUI cases at each size and +all 44 generic exact-clone GUI neighbors. 126 workflows not run; two unrelated +missing controller/Aeon and F2 corpus skips remain unexercised coverage. + +Next: commit/push this GUI milestone, then recover a bounded private appearance/body +clone graph from verified CDefs ownership and native evidence. Do not rewrite all +numeric matches or silently clone shared gameplay components. RC2 and predecessor +archives remain unchanged; no full-suite, clean-machine or gameplay claim. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index eea29745..6c20e294 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [paired Creature clone CLI](HANDOFF_CREATURE_CLONE_2026-10-10.md). Thirty CLI cases, six retail-copy cases and six extracted workflows pass, including rollback/Restore and 44 existing GUI clone neighbors. Distinct Creature GUI action is next; own appearance/body cloning remains open. RC2 unchanged. +> Active continuation: [guarded Creature clone GUI](HANDOFF_CREATURE_CLONE_2026-10-10.md). Distinct shared-component controls pass 36 GUI cases, both-size retail-copy preservation/Restore and eight extracted workflows, including 44 exact-clone GUI neighbors. Own appearance/body cloning remains open. RC2 unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 2f665f54..678b122c 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [paired Creature clone CLI](HANDOFF_CREATURE_CLONE_2026-10-10.md). Thirty CLI cases, six retail-copy cases and six extracted workflows pass, including rollback/Restore and 44 existing GUI clone neighbors. Distinct Creature GUI action is next; own appearance/body cloning remains open. RC2 unchanged. +> Active continuation: [guarded Creature clone GUI](HANDOFF_CREATURE_CLONE_2026-10-10.md). Distinct shared-component controls pass 36 GUI cases, both-size retail-copy preservation/Restore and eight extracted workflows, including 44 exact-clone GUI neighbors. Own appearance/body cloning remains open. RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there @@ -2606,3 +2606,16 @@ including 44 unchanged GUI clone cases; 125 not run and two unrelated missing-co skips. ZIP SHA256 `732db7d64d142e7f7f9138d26345aaf25c1a36d651269074bba8c43b4908fc92`. Distinct Creature GUI action is next; own appearance/body cloning remains open. No full-suite, clean-machine or gameplay claim. See the creature donor handoff. + +Distinct game-bank CREATURE Clone creature controls now use the verified self-only +core API and paired publisher. Preflight rejects unresolved/malformed/empty donors; +draft/confirmation explicitly keep components shared. Thirty-six synthetic source +GUI cases and six read-only native-input/owned-output cases at each window size +pass, including all-record/raw-pool preservation and exact Restore. Final compact +confirmation is visually checked; six focused cases pass after the wording change. +Generic exact clone remains unchanged. Own appearance/body cloning stays open; +exact extracted GUI checkpoint passes 117 identities, 141 links and eight selected +workflows, including all 44 exact-clone GUI neighbors. 126 not run and two unrelated +missing-corpus skips remain explicit. ZIP SHA256 +`51d135c1e95f4fb51cfe4a501f164a7064aa3c1577d4467c137c789a54ffca4e`. +Evidence and remaining scope are recorded in the creature handoff. diff --git a/gui/app.cpp b/gui/app.cpp index d544eb98..403c98d6 100644 --- a/gui/app.cpp +++ b/gui/app.cpp @@ -1971,6 +1971,7 @@ std::vector App::stateDump() const { v.push_back("definition_name_refs="+std::to_string(definitionNameRefs)); v.push_back("definition_editing="+std::to_string(definitionEditing_)); v.push_back("definition_cloning="+std::to_string(definitionCloning_)); + v.push_back("definition_creature_cloning="+std::to_string(definitionCloning_ && definitionEditType_=="clone-creature")); v.push_back("definition_edit_failed="+std::to_string(!definitionEditError_.empty())); v.push_back("definition_ready="+std::to_string(bool(definitionDecoded_))); v.push_back("definition_failed="+std::to_string(!definitionError_.empty() || !definitionSelectionError_.empty() || !definitionDecodeError_.empty())); diff --git a/gui/app.hpp b/gui/app.hpp index b6c8bf0b..e6c26bc5 100644 --- a/gui/app.hpp +++ b/gui/app.hpp @@ -589,7 +589,7 @@ class App { bool setDefinitionBank(const std::string& bank); void browseDefinition(const std::string& name); bool beginDefinitionEdit(size_t field); - bool beginDefinitionClone(); + bool beginDefinitionClone(bool creature=false); bool saveDefinitionClone(); bool setDefinitionEditValue(const std::string& value); bool saveDefinitionEdit(); diff --git a/gui/automation.cpp b/gui/automation.cpp index 38336c02..1e074a6b 100644 --- a/gui/automation.cpp +++ b/gui/automation.cpp @@ -697,8 +697,9 @@ bool Automation::tick(App& app) { } else if (cmd == "model_select") { if (!app.selectModel(rest)) fail("model_select: " + rest); else note("ok " + line); ++pc_; } else if (cmd == "model_search") { std::snprintf(app.modelSearch_, sizeof app.modelSearch_, "%s", rest == "-" ? "" : rest.c_str()); note("ok " + line); ++pc_; } - else if(cmd=="definition_clone" || cmd=="definition_clone_refused") { - if(app.beginDefinitionClone()!=(cmd=="definition_clone")) fail("definition clone returned an unexpected result");else note("ok "+line);++pc_; + else if(cmd=="definition_clone" || cmd=="definition_clone_refused" || cmd=="definition_clone_creature" || cmd=="definition_clone_creature_refused") { + const bool creature=cmd=="definition_clone_creature" || cmd=="definition_clone_creature_refused"; + if(app.beginDefinitionClone(creature)!=(cmd=="definition_clone" || cmd=="definition_clone_creature")) fail("definition clone returned an unexpected result");else note("ok "+line);++pc_; } else if(cmd=="definition_edit" || cmd=="definition_edit_refused") { size_t field=0;const auto parsed=std::from_chars(rest.data(),rest.data()+rest.size(),field); diff --git a/gui/definitionbrowser.cpp b/gui/definitionbrowser.cpp index 473f5083..873fcdda 100644 --- a/gui/definitionbrowser.cpp +++ b/gui/definitionbrowser.cpp @@ -157,7 +157,7 @@ bool App::beginDefinitionEdit(size_t field) { return true; } catch(const std::exception& error) {definitionEditError_=error.what();return false;} } -bool App::beginDefinitionClone() { +bool App::beginDefinitionClone(bool creature) { definitionEditError_.clear();definitionEditSuccess_.clear(); try { if(definitionEditing_) throw std::runtime_error("Save or cancel the current definition draft first."); @@ -167,6 +167,16 @@ bool App::beginDefinitionClone() { if(!packDest_.empty()) {if(!modpack::isPack(fs::u8path(packDest_))) throw std::runtime_error("Choose a valid FableForge pack.");(void)modpack::load(fs::u8path(packDest_));} auto current=forge::bin::File::open(definitionLoadedNames_,definitionLoadedBank_); definitionwrite::verifyEntries(definitionFile_,current); + if(creature) { + if(definitionBank_!="game.bin" || current.entries()[*definitionSelected_].definition!="CREATURE") + throw std::runtime_error("Choose a Creature donor in the game bank."); + // Preflight the strict layout/refusal path without changing the + // captured donor bank or choosing the user's eventual new name. + auto probe=current;size_t suffix=0;std::string probeName; + do {probeName="FORGE_CREATURE_DRAFT_"+std::to_string(suffix++);} + while(std::any_of(current.entries().begin(),current.entries().end(),[&](const auto& entry){return entry.name==probeName;})); + (void)forge::defedit::cloneCreature(probe,definitionSchema_,*definitionSelected_,probeName); + } definitionCloneNamesBytes_=definitionwrite::cloneBytes(definitionLoadedNames_); definitionCloneBankBytes_=definitionwrite::cloneBytes(definitionLoadedBank_); definitionCloneFile_=std::move(current); @@ -175,7 +185,7 @@ bool App::beginDefinitionClone() { definitionEditDestination_=fs::u8path(packDest_.empty()?saveRoot():packDest_)/"data"/"CompiledDefs"/definitionBank_; const auto& donor=definitionFile_.entries()[definitionEditIndex_]; definitionEditField_=donor.name.empty()?"(unnamed record)":donor.name; - definitionEditType_="clone";definitionEditing_=true;definitionCloning_=true; + definitionEditType_=creature?"clone-creature":"clone";definitionEditing_=true;definitionCloning_=true; definitionEditConfirm_=false;definitionEditValue_[0]=0;return true; } catch(const std::exception& error) {definitionEditError_=error.what();return false;} } @@ -193,8 +203,11 @@ bool App::saveDefinitionClone() { definitionwrite::cloneBytes(definitionEditBank_)!=definitionCloneBankBytes_) throw std::runtime_error("The source bank or name pool changed; start the clone again."); auto cloned=definitionCloneFile_; - const auto added=forge::defedit::cloneEntry(cloned,definitionEditIndex_,definitionEditValue_); - definitionwrite::publishClone(cloned,definitionEditNames_,definitionEditBank_,destination,true); + const bool creature=definitionEditType_=="clone-creature"; + const auto added=creature?forge::defedit::cloneCreature(cloned,definitionSchema_,definitionEditIndex_,definitionEditValue_): + forge::defedit::cloneEntry(cloned,definitionEditIndex_,definitionEditValue_); + definitionwrite::publishClone(cloned,definitionEditNames_,definitionEditBank_,destination,true, + creature?&definitionSchema_:nullptr,creature?std::optional(definitionEditIndex_):std::nullopt); definitionEditSuccess_="Created "+std::string(definitionEditValue_)+"."; definitionEditing_=false;definitionCloning_=false;definitionEditConfirm_=false; definitionCloneFile_={};definitionCloneNamesBytes_.clear();definitionCloneBankBytes_.clear(); @@ -253,11 +266,19 @@ void App::drawDefinitionBrowser(float pad,float inner,float cardInner) { ImGui::BeginDisabled(definitionsBusy() || modFilesBusy() || !definitionSelected_); if(theme::ghostButton("Clone selected definition",ImVec2(cardInner,0))) beginDefinitionClone(); auto_.registerWidget("btn_definition_clone");ImGui::EndDisabled(); + if(definitionSelected_ && *definitionSelected_0) saveDefinitionEdit();if(answer) definitionEditConfirm_=false; } } diff --git a/tools/check_all.py b/tools/check_all.py index ae5eefb2..81513055 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -127,6 +127,10 @@ def main(): [sys.executable, "tools/audit_creature_self_refs.py", "--install", install_root, "--clone-api"], capture_output=True) ok &= run("Retail paired Creature clone, self-only deltas and Restore", [sys.executable, "tools/test_creature_clone_retail.py", "--install", install_root], capture_output=True) + for window_size in ("1440x900", "800x600"): + ok &= run("Retail Creature clone controls and Restore " + window_size, + [sys.executable, "tools/test_creature_clone_retail.py", "--install", install_root, + "--gui", "build/FableForge.exe", "--size", window_size], capture_output=True) ok &= run("Creature GroupDef schema, independent boundaries and all retail records", [sys.executable, "tools/test_creature_group_schema.py", "--install", install_root], capture_output=True) ok &= run("retail group picker families, exact field deltas and Restore", @@ -336,6 +340,8 @@ def main(): [sys.executable, "tools/test_definition_clone.py"], capture_output=True) ok &= run("Creature clone selector, verified self-only payloads, paired rollback and Restore", [sys.executable, "tools/test_creature_clone_cli.py"], capture_output=True) + ok &= run("Creature clone controls, draft protection, stale sources and paired Restore", + [sys.executable, "tools/test_creature_clone_ui.py"], capture_output=True) ok &= run("Definition clone controls, exact donors, stale drafts and paired Restore", [sys.executable, "tools/test_definition_clone_ui.py"], capture_output=True) ok &= run("Definition existing-name picker, exact field deltas, invalid offsets and Restore", diff --git a/tools/test_creature_clone_cli.py b/tools/test_creature_clone_cli.py index b86f2be6..a73a962a 100644 --- a/tools/test_creature_clone_cli.py +++ b/tools/test_creature_clone_cli.py @@ -13,13 +13,14 @@ from test_definition_clone import deny_rename -def fixture(root): +def fixture(root,creature_fields=None): folder=root/'data/CompiledDefs';folder.mkdir(parents=True) def payload(owner,mode='normal'): data=bytearray(b'\x01\0\x01\x02\0') for value in (0x12345678,2,owner,0x87654321,owner,0):data+=struct.pack(' Date: Sat, 10 Oct 2026 10:59:39 -0600 Subject: [PATCH 11/75] Audit private Creature visual graph ownership against native evidence --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 51 +++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 13 +- tests/test_defrelocate.cpp | 54 ++++++++ tools/audit_creature_visual_graph.py | 159 ++++++++++++++++++++++ tools/check_all.py | 5 + tools/probe_creature_cdefs_native.py | 91 +++++++++++++ 7 files changed, 373 insertions(+), 2 deletions(-) create mode 100644 tools/audit_creature_visual_graph.py create mode 100644 tools/probe_creature_cdefs_native.py diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index d056a1e3..96906aae 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -199,3 +199,54 @@ Next: commit/push this GUI milestone, then recover a bounded private appearance/ clone graph from verified CDefs ownership and native evidence. Do not rewrite all numeric matches or silently clone shared gameplay components. RC2 and predecessor archives remain unchanged; no full-suite, clean-machine or gameplay claim. + +## Private visual graph evidence continuation + +GUI head a04ba4c Windows CI 38068198733 passed. Current continuation is read-only +graph/relocation observation, not a private clone writer or eligibility claim. +`audit_creature_visual_graph.py` independently decodes native CDefs triples and +checks selected CCreatureDef/CAppearanceDef class targets, original-parent rows, +duplicate/missing rows and sharing. Synthetic malformed/range/type/duplicate, +inherited/shared and empty cases pass. Native `creature-visual-graph-c4db1bp_` +observes 517 Creature roots: 502 have both visual rows, fourteen have no appearance +row and NULLDEF has an empty listing. All observed visual targets match their +class keys; referenced original parents contain matching component rows. +CCreatureDef: 328 own/188 inherited rows, 261 shared rows. CAppearanceDef: +100 own/402 inherited rows, 442 shared rows. There are 428 unique visual components. +Sharing counts are Creature-root use observations, not a complete incoming-edge audit. + +Bounded original-byte evidence `creature-cdefs-native-d4h9ut0j` records FableWin +SHA256 a9d6d0977d9d7fc8242da4292ae7ed92e926ca80364a95c003d9b9f689e41845 +and matching FableWin PDB/layout identity. Native CParentDefClassBase save/load +030633f0/030634d0 writes/reads u16 count plus {u32 class CRC, i32 DefIndex, +i32 OriginalParentDefIndex}; base loader 03063850 consumes three flag bytes. +PDB CDefClassBase::CSubDefInfo is eight bytes with those two signed-long members +at +0/+4. GetSubDef 01ac010e and appearance 01b207b4 call the +class-key lookup and resolve the first member through GetPDefFromGlobalIndex +030620e0. CompileSubDefinition 03060c70 compares OriginalParentDefIndex to the +current parent at 03060cf9..03060cff; a different parent takes instantiate/copy +before editing, equal parent reuses the existing component. This corroborates +copy-on-write ownership; private inherited visual rows need localized parent +reassignment, not remapping that old parent across shared gameplay components. +`probe_creature_cdefs_native.py` reproduces the bounded packet with optional local +pefile/capstone and the existing read-only PDB oracle; not a shipping dependency. +Earlier packet uco3u3aq lacked compiler/PDB capture; final d4h9ut0j includes both. +The preliminary Ego_r layout query was cross-checked with matching FableWin PDB. + +Canonical optional `fableforge_defrelocate_tests --creature-visual-audit` analyzes +root/body/appearance maps in memory only. Combined independent gate passes at +`creature-visual-graph-bbhi1gb8`: all 502 three-node graphs round-trip all payloads; +2,435 verified root rewrites, zero component rewrites. 477 graphs have no unresolved +candidates; 25 have findings (eight root payloads, twenty appearance payloads, +with overlap). Raw checks verify every root CDefs rewrite and full payload deltas; +native bank hashes unchanged. Appearance examples hit long Animation payload +offsets, not necessarily model IDs. Unknown matches remain refused observations; +do not value-wide rewrite them or infer a reference domain from integer equality. +One affected relocation CTest passes; native observation gate registered. +No new GUI archive or native install copies for this analysis continuation. + +Next: establish the unresolved appearance-field asset domains, then implement the +coordinated private root/body/appearance plan with explicit selected-row parent +updates and non-mutating refusal. Keep gameplay components and existing assets +shared until separately requested by the chosen import workflow. Still no private +visual writer, transaction, GUI or gameplay claim. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 6c20e294..6abea4ed 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [guarded Creature clone GUI](HANDOFF_CREATURE_CLONE_2026-10-10.md). Distinct shared-component controls pass 36 GUI cases, both-size retail-copy preservation/Restore and eight extracted workflows, including 44 exact-clone GUI neighbors. Own appearance/body cloning remains open. RC2 unchanged. +> Active continuation: [private Creature visual graph recovery](HANDOFF_CREATURE_CLONE_2026-10-10.md). GUI milestone a04ba4c passes CI. Read-only CDefs/native/PDB evidence establishes visual targets and original-parent ownership; 502 three-node graphs observed, 25 with unresolved candidates. Private writer remains next. RC2 unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 678b122c..72d4b18a 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [guarded Creature clone GUI](HANDOFF_CREATURE_CLONE_2026-10-10.md). Distinct shared-component controls pass 36 GUI cases, both-size retail-copy preservation/Restore and eight extracted workflows, including 44 exact-clone GUI neighbors. Own appearance/body cloning remains open. RC2 unchanged. +> Active continuation: [private Creature visual graph recovery](HANDOFF_CREATURE_CLONE_2026-10-10.md). GUI milestone a04ba4c passes CI. Read-only CDefs/native/PDB evidence establishes visual targets and original-parent ownership; 502 three-node graphs observed, 25 with unresolved candidates. Private writer remains next. RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there @@ -2619,3 +2619,14 @@ workflows, including all 44 exact-clone GUI neighbors. 126 not run and two unrel missing-corpus skips remain explicit. ZIP SHA256 `51d135c1e95f4fb51cfe4a501f164a7064aa3c1577d4467c137c789a54ffca4e`. Evidence and remaining scope are recorded in the creature handoff. + +Private visual ownership recovery now independently observes all 517 Creature +roots: 502 have both CCreatureDef/CAppearanceDef rows, fourteen omit appearance, +NULLDEF is empty. Native original-byte serialization/lookup and matching PDB +confirm DefIndex and OriginalParentDefIndex plus copy-on-write parent handling. +In-memory canonical relocation observations round-trip all 502 three-node graphs; +477 have no unresolved candidates, 25 retain findings (eight roots/twenty +appearance payloads with overlap). No graph clone eligibility/private writer is +claimed. One affected CTest and independent raw-delta/hash checks pass. Next: +recover unresolved appearance asset domains, then coordinated private-clone API. +No copied native install or new GUI archive for this evidence continuation. diff --git a/tests/test_defrelocate.cpp b/tests/test_defrelocate.cpp index 363e3f97..ea21cfd7 100644 --- a/tests/test_defrelocate.cpp +++ b/tests/test_defrelocate.cpp @@ -79,6 +79,60 @@ int audit(char** argv) { } int main(int argc, char** argv) { + if (argc == 4 && std::string_view(argv[1]) == "--creature-visual-audit") { + try { + using namespace forge; + const auto folder=std::filesystem::path(argv[2])/"data"/"CompiledDefs"; + const auto schema=defschema::Schema::load(argv[3]); + const auto source=bin::File::open(folder/"names.bin",folder/"game.bin"); + const auto& entries=source.entries(); + if(entries.size()>UINT32_MAX-3) throw std::runtime_error("visual audit index overflow"); + auto read=[](const auto& bytes,size_t at) { + if(at>bytes.size() || bytes.size()-at<4) throw std::runtime_error("truncated visual audit word"); + return uint32_t(bytes[at]) | uint32_t(bytes[at+1])<<8 | uint32_t(bytes[at+2])<<16 | uint32_t(bytes[at+3])<<24; + }; + nlohmann::json rows=nlohmann::json::array(); + for(size_t i=0;i nodes{i}; + for(const auto* kind:{"CCreatureDef","CAppearanceDef"}) { + std::vector matches; + for(size_t row=0;row=entries.size() || entries[matches.front()].definition!=kind) break; + nodes.push_back(matches.front()); + } + if(nodes.size()!=3) continue; // Missing shapes are reported independently, not accepted. + defrelocate::Maps maps; + for(size_t n=0;nlen(data) or data[end:end+4]!=struct.pack('1:kinds[kind+'.shared_rows']+=1 + selected.append(row) + counts['empty_listings' if not rows else 'nonempty_listings']+=1 + counts['roots_with_observation_issues' if issues else 'roots_without_observation_issues']+=1 + findings.append(dict(index=index,name=records[index]['name'],component_count=len(rows), + visual_rows=selected,issues=issues)) + return dict(entries=len(records),creature_roots=len(roots),counts=dict(counts), + observations=dict(kinds),unique_visual_components=len(users),records=findings, + clone_eligibility_verified=False) + + +def self_test(): + def root(rows): + data=b'\x01\x00\x01'+struct.pack(' Date: Sat, 10 Oct 2026 11:41:06 -0600 Subject: [PATCH 12/75] Recover Creature animation boundaries with read-only native and retail audits --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 29 ++++++ tools/audit_creature_animation.py | 115 ++++++++++++++++++++++ tools/check_all.py | 4 + tools/probe_creature_cdefs_native.py | 67 +++++++++++-- 4 files changed, 206 insertions(+), 9 deletions(-) create mode 100644 tools/audit_creature_animation.py diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 96906aae..3a706d6f 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -250,3 +250,32 @@ coordinated private root/body/appearance plan with explicit selected-row parent updates and non-mutating refusal. Keep gameplay components and existing assets shared until separately requested by the chosen import workflow. Still no private visual writer, transaction, GUI or gameplay claim. + +## Animation boundary recovery + +Bounded packet `creature-animation-native-oo2dfwaa` uses the same hash-guarded +FableWin bytes and matching PDB. `probe_creature_cdefs_native.py --feature animation` +captures set/entry readers and writers, thirteen component reader/writer pairs, +allocation, Add, PostInit and rotation lookup. PDB confirms CAnimationEntry +BankIndex/AnimName/GroupName and CAnimationSet DefaultFlags/TransitionTime/Delay/ +DefaultGroup. Matching PDB enum EAnimComponent is 0..12; queried with llvm-pdbutil +pretty --enums --include-types=^EAnimComponent$ and the DIA SDK on PATH. +Native Add 01fa0f8f takes AnimName.GetCRC for the map key. Rotation lookup +01fa0c78..01fa0c81 passes BankIndex to CMeshDataBank::PeekAnimation, not the +definition manager. Component 8/11 transfer CDefString; component 12 transfers a +u32-length inline byte string. Other component bodies use signed scalars, flags, +floats, booleans or a 2D vector; do not classify the whole field as one domain. + +Independent `audit_creature_animation.py` synthetic truncation/count/unknown-type +cases pass. Native observations `creature-animation-audit-qcn0gzqq` decode all +320 appearance fields, 27,779 animation entries, zero layout failures; each ends +at the following OutlineEffectAlphaRef tag. Source bank hashes remain unchanged. +The 27 root-ID-equal windows across twenty affected root observations comprise +nineteen whole BankIndex fields in the separate animation asset namespace and +eight windows straddling name-offset/count or combo-value/component-type fields. +These are layout observations, not rewritten IDs or a clone eligibility claim. +Registered native/synthetic gates; no full-suite or new shipping archive claim. + +Next: translate these verified primitive/reference boundaries into canonical +relocation coverage, testing name-offset remapping and strict malformed refusal. +Then implement the coordinated private visual plan with localized parent updates. diff --git a/tools/audit_creature_animation.py b/tools/audit_creature_animation.py new file mode 100644 index 00000000..09594533 --- /dev/null +++ b/tools/audit_creature_animation.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Read-only native-layout observations; not relocation or clone eligibility. + +Layout evidence: FableWin CAnimationSet 01fa26b0/01fa27a0, +CAnimationEntry 01f9d270/01f9d590, component readers 01f9b300..01f9d200, +matching PDB EAnimComponent and member layouts. No numeric-match rewriting. +""" +import argparse +from collections import Counter +import hashlib +import json +from pathlib import Path +import struct +import tempfile +from test_fmp_links import read_bank, crc0 +from audit_creature_visual_graph import listing + + +def decode_animation(data): + spans=[];cursor=0 + def take(size,label): + nonlocal cursor + if size<0 or size>len(data)-cursor:raise ValueError('truncated '+label) + offset=cursor;cursor+=size + spans.append(dict(offset=offset,size=size,label=label)) + return data[offset:cursor] + def word(label):return struct.unpack('(len(data)-cursor)//20:raise ValueError('entry count exceeds minimum extent') + for i in range(count): + prefix=f'entries.{i}.' + word(prefix+'key');word(prefix+'BankIndex') + word(prefix+'AnimName');word(prefix+'GroupName') + components=word(prefix+'component_count') + if components>(len(data)-cursor)//8:raise ValueError('component count exceeds minimum extent') + seen=set() + for j in range(components): + kind=word(prefix+f'components.{j}.type') + if kind not in range(13) or kind in seen:raise ValueError('unknown/duplicate component type') + seen.add(kind);label=prefix+f'components.{j}.type{kind}' + if kind==12: + size=word(label+'.string_length');take(size,label+'.string_bytes') + elif kind in (3,4): + word(label+'.first');word(label+'.second') + elif kind==6: + take(1,label+'.bool0');take(1,label+'.bool1') + elif kind==7: + take(1,label+'.bool');take(4,label+'.x');take(4,label+'.y') + else: + word(label+('.name_offset' if kind in (8,11) else '.scalar')) + for label in ('DefaultFlags','DefaultTransitionTime','DefaultDelay','DefaultGroup'): + word(label) + return cursor,spans,count + + +def self_test(): + data=struct.pack('s['offset']] + observations.append(dict(root=index,appearance=component,offset=offset,value=index, + spans=overlaps,whole_primitive=len(overlaps)==1 and overlaps[0]['offset']==relative and overlaps[0]['size']==4)) + assert all(hashlib.sha256(p.read_bytes()).hexdigest()==hashes[str(p)] for p in sources),'native bank changed' + args.work_root.resolve().mkdir(parents=True,exist_ok=True) + work=Path(tempfile.mkdtemp(prefix='creature-animation-audit-',dir=args.work_root.resolve())) + report=dict(source_sha256=hashes,counts=dict(counts),layout_failures=failures, + candidate_observations=observations,clone_eligibility_verified=False) + (work/'result.json').write_text(json.dumps(report,indent=2)+'\n',encoding='utf-8') + print(json.dumps(dict(counts=dict(counts),layout_failures=len(failures),candidates=len(observations)))) + print('Read-only animation observations retained at',work) + assert not failures,'appearance layout failures: inspect retained observations' + + +if __name__=='__main__':main() diff --git a/tools/check_all.py b/tools/check_all.py index d2c674ea..dab85089 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -126,6 +126,8 @@ def main(): ok &= run("Read-only Creature visual ownership and relocation observations (not clone eligibility)", [sys.executable, "tools/audit_creature_visual_graph.py", "--install", install_root, "--payload-exe", "build/fableforge_defrelocate_tests.exe"], capture_output=True) + ok &= run("Read-only Creature animation boundaries and numeric candidate observations", + [sys.executable, "tools/audit_creature_animation.py", "--install", install_root], capture_output=True) ok &= run("Creature donor self-reference layout and independent relocation audit", [sys.executable, "tools/audit_creature_self_refs.py", "--install", install_root, "--clone-api"], capture_output=True) ok &= run("Retail paired Creature clone, self-only deltas and Restore", @@ -347,6 +349,8 @@ def main(): [sys.executable, "tools/test_creature_clone_ui.py"], capture_output=True) ok &= run("Creature visual graph synthetic ownership observations", [sys.executable, "tools/audit_creature_visual_graph.py"], capture_output=True) + ok &= run("Creature animation synthetic boundaries and malformed component refusal", + [sys.executable, "tools/audit_creature_animation.py"], capture_output=True) ok &= run("Definition clone controls, exact donors, stale drafts and paired Restore", [sys.executable, "tools/test_definition_clone_ui.py"], capture_output=True) ok &= run("Definition existing-name picker, exact field deltas, invalid offsets and Restore", diff --git a/tools/probe_creature_cdefs_native.py b/tools/probe_creature_cdefs_native.py index 77903faa..fe13ff05 100644 --- a/tools/probe_creature_cdefs_native.py +++ b/tools/probe_creature_cdefs_native.py @@ -27,6 +27,45 @@ 'get_appearance':0x01b207b4, } EXPECTED_EXE='a9d6d0977d9d7fc8242da4292ae7ed92e926ca80364a95c003d9b9f689e41845' +ANIMATION_FUNCTIONS={ + 'persist_animation':0x01998a2d, + 'animation_traits_in':0x019d4208, + 'animation_traits_out':0x019d4223, + 'animation_set_in':0x01fa26b0, + 'animation_set_out':0x01fa27a0, + 'animation_entry_in':0x01f9d270, + 'animation_entry_out':0x01f9d590, + 'allocate_animation_component':0x01fa0770, + 'animation_rotation':0x01fa0b90, + 'animation_postinit':0x01fa0f50, + 'animation_add':0x01fa0f80, + 'component_flags_in':0x01f9b300, + 'component_flags_out':0x01f9b330, + 'component_transition_in':0x01f9b4c0, + 'component_transition_out':0x01f9b4f0, + 'component_delay_in':0x01f9b680, + 'component_delay_out':0x01f9b6b0, + 'component_handedness_in':0x01f9bae0, + 'component_handedness_out':0x01f9bb20, + 'component_combo_in':0x01f9be90, + 'component_combo_out':0x01f9bed0, + 'component_recoil_in':0x01f9c070, + 'component_recoil_out':0x01f9c0a0, + 'component_misc_in':0x01f9c490, + 'component_misc_out':0x01f9c4d0, + 'component_offset_in':0x01f9c8e0, + 'component_offset_out':0x01f9c930, + 'component_following_in':0x01f9cab0, + 'component_following_out':0x01f9cae0, + 'component_creature_mode_in':0x01f9ccb0, + 'component_creature_mode_out':0x01f9cce0, + 'component_following_filter_in':0x01f9ce40, + 'component_following_filter_out':0x01f9ce70, + 'component_speed_in':0x01f9cfe0, + 'component_speed_out':0x01f9d010, + 'component_response_in':0x01f9d160, + 'component_response_out':0x01f9d200, +} def main(): @@ -34,6 +73,8 @@ def main(): parser.add_argument('--exe',type=Path,required=True) parser.add_argument('--symbols',type=Path,required=True) parser.add_argument('--work-root',type=Path,required=True) + parser.add_argument('--feature',choices=('cdefs','animation'),default='cdefs', + help='Bounded ownership or appearance Animation serialization evidence') parser.add_argument('--pdb',type=Path,help='Matching FableWin PDB for member-name corroboration') parser.add_argument('--pdb-oracle',type=Path,help='Existing read-only query_pdb_oracle.ps1') args=parser.parse_args();raw=args.exe.read_bytes();identity=hashlib.sha256(raw).hexdigest() @@ -48,7 +89,7 @@ def main(): addresses=sorted(symbols);pe=pefile.PE(data=raw);base=pe.OPTIONAL_HEADER.ImageBase dis=capstone.Cs(capstone.CS_ARCH_X86,capstone.CS_MODE_32) packets=[];text=[] - for label,address in FUNCTIONS.items(): + for label,address in (FUNCTIONS if args.feature=='cdefs' else ANIMATION_FUNCTIONS).items(): assert address in symbols,(label,'missing exact symbol') end=addresses[bisect_right(addresses,address)] assert 0 Date: Sat, 10 Oct 2026 11:56:42 -0600 Subject: [PATCH 13/75] Relocate verified Creature animation names without changing asset IDs --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 38 ++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 2 +- libs/forgecore/include/forge/defrelocate.hpp | 3 +- libs/forgecore/src/defrelocate.cpp | 74 +++++++++++ tests/test_defrelocate.cpp | 129 +++++++++++++++++++ tools/audit_creature_animation.py | 37 +++++- tools/check_all.py | 3 +- 8 files changed, 282 insertions(+), 6 deletions(-) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 3a706d6f..3b43b7c7 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -279,3 +279,41 @@ Registered native/synthetic gates; no full-suite or new shipping archive claim. Next: translate these verified primitive/reference boundaries into canonical relocation coverage, testing name-offset remapping and strict malformed refusal. Then implement the coordinated private visual plan with localized parent updates. + +## Canonical animation relocation coverage + +`defrelocate` now recognizes only CAppearanceDef.Animation/CAnimationSet. It +checks entry counts, all thirteen component shapes, duplicate/unknown selectors, +canonical booleans, inline string extents, defaults and exact end-of-value. +AnimName, GroupName, DefaultGroup and component 8/11 CDefStrings use the existing +NameOffset domain; CRC keys, animation-bank indices, flags, timings, enum/float +values and inline bytes stay exact. ComboStage/ComboID and RecoilAnimIndex have +verified primitive boundaries but whole matching integers remain refused until +their runtime domains are established. Nothing is inferred from integer equality. + +Synthetic all-variant, simultaneous-domain, unknown-owner, count/duplicate/type, +boolean/string/trailer and every-truncation checks pass. Two initial fixture +mistakes (partial-word mutation and missing scan candidates in very short values) +were corrected; final affected CTest passes. Five affected/neighbor CTests pass: +defrelocate, defnamespace, binclone, fmplinks and binwrite. Not a full-suite claim. + +Read-only `creature-visual-graph-d3hdld8y` checks all 502 three-node graphs: +494 now have no unresolved candidates; eight root payload findings remain. +All 1,506 payloads round-trip; 2,435 verified root rewrites and zero body or +appearance rewrites. Native bank hashes unchanged. The former twenty appearance +findings are resolved as asset fields or straddled primitive windows, not remapped. + +`creature-animation-audit-nzpfyd8g` independently verifies canonical relocation +on isolated native Animation values from all 320 appearance entries. All 38,351 +non-null native name-offset words relocate to synthetic in-memory destinations; +the full before/after bytes match only those independently located words. +Every actual non-null reference is covered by the audit map (native minimum +106,608; map includes names offsets >=65,536). Small-offset ambiguity is tested +synthetically, not claimed as fully classified. Other appearance fields are not +included in this name-rebase audit. Source banks unchanged; no installed outputs, +new archive, gameplay, transaction or private visual clone eligibility claim. + +Next: implement the coordinated root/body/appearance append plan. Selected visual +rows must receive new component indices and localized new-root original parents; +unselected shared gameplay rows retain their original-parent bytes. Do not use +a value-wide old-parent map or mark borrowed components as privately owned. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 6abea4ed..31ea62a2 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [private Creature visual graph recovery](HANDOFF_CREATURE_CLONE_2026-10-10.md). GUI milestone a04ba4c passes CI. Read-only CDefs/native/PDB evidence establishes visual targets and original-parent ownership; 502 three-node graphs observed, 25 with unresolved candidates. Private writer remains next. RC2 unchanged. +> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Canonical animation boundaries now resolve appearance findings; 494 of 502 observed three-node graphs have no unresolved candidates, eight root findings remain. Private append/ownership plan is next; no eligibility claim. RC2 unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 72d4b18a..930702e8 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [private Creature visual graph recovery](HANDOFF_CREATURE_CLONE_2026-10-10.md). GUI milestone a04ba4c passes CI. Read-only CDefs/native/PDB evidence establishes visual targets and original-parent ownership; 502 three-node graphs observed, 25 with unresolved candidates. Private writer remains next. RC2 unchanged. +> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Canonical animation boundaries now resolve appearance findings; 494 of 502 observed three-node graphs have no unresolved candidates, eight root findings remain. Private append/ownership plan is next; no eligibility claim. RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there diff --git a/libs/forgecore/include/forge/defrelocate.hpp b/libs/forgecore/include/forge/defrelocate.hpp index 6edc91a1..a3166c76 100644 --- a/libs/forgecore/include/forge/defrelocate.hpp +++ b/libs/forgecore/include/forge/defrelocate.hpp @@ -30,7 +30,8 @@ struct Analysis { }; // Inspect one decoded-bank payload. Recognizes OBJECT/BUILDING/CREATURE CDefs listings, -// verified hero-ability, brain-behaviour and shop-stock reference layouts, and +// verified hero-ability, brain-behaviour, shop-stock and appearance animation +// reference layouts, and // explicitly typed CDefIndex/CDefString scalar/vector fields. Other integer // fields are not inferred to be references. Every-byte candidate scanning is // a conservative rejection guard, never a source of rewrite locations. diff --git a/libs/forgecore/src/defrelocate.cpp b/libs/forgecore/src/defrelocate.cpp index a96ed953..9f59924f 100644 --- a/libs/forgecore/src/defrelocate.cpp +++ b/libs/forgecore/src/defrelocate.cpp @@ -165,6 +165,8 @@ Analysis analyze(const bin::Entry& entry, const defschema::Schema& schema, const field.name == "StatLevels" && field.type == "vector"; const bool tavernScoreText = entry.definition == "CTavernGameDef" && field.name == "BestScore" && field.type == "?"; + const bool appearanceAnimation = entry.definition == "CAppearanceDef" && + field.name == "Animation" && field.type == "CAnimationSet"; const bool primitiveVector = field.type == "Vector_J" || field.type == "Vector_K" || field.type == "Vector_int32" || field.type == "Vector_uint32" || field.type == "Vector_float" || heroCosts; const bool primitiveMap = field.type == "Map_JJ" || field.type == "Map_KK" || @@ -207,6 +209,78 @@ Analysis analyze(const bin::Entry& entry, const defschema::Schema& schema, const // 022D72F0, which resolves it through the game text bank. if (field.value.size() != 4) fail(entry, "invalid tavern score text width"); cover(at, 4); + } else if (appearanceAnimation) { + // FableWin set in/out 01FA26B0/01FA27A0 and entry in/out + // 01F9D270/01F9D590: count, {name CRC, animation-bank index, + // AnimName, GroupName, counted discriminated components}, then + // three defaults and DefaultGroup. Matching PDB EAnimComponent + // and the thirteen reader/writer pairs establish each body. + // BankIndex is passed to CMeshDataBank::PeekAnimation at 01FA0C81; + // Add obtains the key from AnimName.GetCRC at 01FA0F8F. + size_t cursor = 0; + auto take = [&](size_t size) { + if (cursor > field.value.size() || size > field.value.size() - cursor) + fail(entry, "truncated appearance animation body"); + const auto offset = cursor; cursor += size; return offset; + }; + auto scalar = [&] { const auto offset = take(4); cover(at + offset, 4); + return word(field.value, offset); }; + auto opaque = [&] { + const auto offset = take(4); + primitiveBoundaries.push_back(at + offset); + primitiveBoundaries.push_back(at + offset + 4); + }; + auto nameOffset = [&](std::string path) { + reference(at + take(4), Domain::NameOffset, std::move(path)); + }; + auto boolean = [&] { + const auto offset = take(1); + if (field.value[offset] > 1) fail(entry, "invalid appearance animation boolean"); + cover(at + offset, 1); + }; + if (field.value.size() < 20) fail(entry, "truncated appearance animation defaults"); + const size_t count = scalar(); + if (count > (field.value.size() - 20) / 20) + fail(entry, "invalid appearance animation entry count"); + for (size_t i = 0; i < count; ++i) { + const auto path = "Animation[" + std::to_string(i) + "]"; + scalar(); // Name CRC, not an offset. + scalar(); // Animation asset bank, never a game entry. + nameOffset(path + ".AnimName"); + nameOffset(path + ".GroupName"); + const size_t components = scalar(); + if (components > 13) fail(entry, "invalid appearance animation component count"); + uint32_t seen = 0; + for (size_t j = 0; j < components; ++j) { + const auto kind = scalar(); + if (kind > 12 || (seen & (1u << kind))) + fail(entry, "unknown or duplicate appearance animation component"); + seen |= 1u << kind; + const auto component = path + ".Components[" + std::to_string(kind) + "]"; + if (kind == 12) { + const size_t length = scalar(); + const auto offset = take(length); cover(at + offset, length); + } else if (kind == 8 || kind == 11) { + nameOffset(component + ".Name"); + } else if (kind == 6 || kind == 7) { + boolean(); + if (kind == 6) boolean(); + else { scalar(); scalar(); } // Two floats, packed after the bool. + } else if (kind == 4 || kind == 5) { + // PDB says ComboStage/ComboID or RecoilAnimIndex, but + // their runtime value domains are not established yet. + // Distinct primitive boundaries exclude straddled matches; + // equal whole integers still refuse rather than guessing. + opaque(); if (kind == 4) opaque(); + } else { + scalar(); // Flags, timing, float multiplier or mode enum. + if (kind == 3) scalar(); // Start/end handedness enums. + } + } + } + scalar(); scalar(); scalar(); // DefaultFlags, TransitionTime, Delay. + nameOffset("Animation.DefaultGroup"); + if (cursor != field.value.size()) fail(entry, "trailing appearance animation bytes"); } else if (statLevels) { // Native transfers 01A0E34C/01A0F440. UI 023B058A passes both // nested Description and MaxedOutDescription to GetInventoryText. diff --git a/tests/test_defrelocate.cpp b/tests/test_defrelocate.cpp index ea21cfd7..fdebe4fb 100644 --- a/tests/test_defrelocate.cpp +++ b/tests/test_defrelocate.cpp @@ -27,6 +27,24 @@ Bytes fixture(bool moved) { field(out, "Other", {12345}); return out; } +Bytes animationFixture(bool moved) { + Bytes out{1,0,1}; + field(out,"Animation",{1,77,77,moved ? 900u : 77u,moved ? 900u : 77u,13}); + for (uint32_t kind=0;kind<=12;++kind) { + word(out,kind); + if (kind==3) { word(out,77);word(out,77); } + else if (kind==4) { word(out,1);word(out,60); } + else if (kind==5) word(out,12345); + else if (kind==6) { out.push_back(1);out.push_back(0); } + else if (kind==7) { out.push_back(0);word(out,77);word(out,77); } + else if (kind==8 || kind==11) word(out,moved ? 900 : 77); + else if (kind==12) { word(out,5);word(out,77);out.push_back('X'); } + else word(out,77); + } + word(out,77);word(out,77);word(out,77);word(out,moved ? 900 : 77); + field(out,"Tail",{});out.push_back(0); + return out; +} std::string hex(const Bytes& data) { static constexpr char digits[] = "0123456789abcdef"; std::string out; @@ -79,6 +97,57 @@ int audit(char** argv) { } int main(int argc, char** argv) { + if (argc == 4 && std::string_view(argv[1]) == "--creature-animation-name-audit") { + try { + using namespace forge; + const auto folder=std::filesystem::path(argv[2])/"data"/"CompiledDefs"; + const auto schema=defschema::Schema::load(argv[3]); + const auto source=bin::File::open(folder/"names.bin",folder/"game.bin"); + const auto isolatedSchema=defschema::Schema::loadText(R"({ + "CAppearanceDef":{"prefix_len":3,"fields":[{"name":"Animation","type":"CAnimationSet"}]} + })"); + // In-memory synthetic destinations only. The independent auditor + // verifies that every non-null native Animation name is in this + // map; low integers stay out so unproven combo/recoil domains are + // not implicitly claimed. Small-name collisions have CTest cases. + defrelocate::Maps maps; + uint64_t offset=4; + for(const auto& name:source.originalNames()) { + if(offset>=65536) { + if(offset+0x10000000>=UINT32_MAX) throw std::runtime_error("animation audit name overflow"); + maps.nameOffsets.emplace(uint32_t(offset),uint32_t(offset+0x10000000)); + } + offset+=name.second.size()+5; + } + nlohmann::json rows=nlohmann::json::array(); + for(size_t i=0;i matches; + for(const auto& field:decoded.fields) if(field.name=="Animation" && field.type=="CAnimationSet" && field.tagOk) + matches.push_back(&field); + if(matches.size()!=1) throw std::runtime_error("animation audit requires one verified field"); + bin::Entry isolated{"CAppearanceDef",entry.name,entry.indexInDefinition,decoded.prefix}; + word(isolated.data,defdecode::fieldTag("Animation")); + isolated.data.insert(isolated.data.end(),matches.front()->value.begin(),matches.front()->value.end()); + const auto analysis=defrelocate::analyze(isolated,isolatedSchema,maps); + nlohmann::json row{{"index",i},{"before_hex",hex(isolated.data)}, + {"rewrites",nlohmann::json::array()},{"unresolved",nlohmann::json::array()}}; + for(const auto& rewrite:analysis.rewrites) + row["rewrites"].push_back({{"offset",rewrite.offset},{"before",rewrite.before}, + {"after",rewrite.after},{"path",rewrite.path}, + {"domain",rewrite.domain==defrelocate::Domain::GameEntry?"game":"name"}}); + for(const auto& candidate:analysis.unresolved) + row["unresolved"].push_back({{"offset",candidate.offset},{"value",candidate.value}}); + if(analysis.unresolved.empty()) row["after_hex"]=hex(defrelocate::relocate(isolated,isolatedSchema,maps)); + rows.push_back(std::move(row)); + } + std::cout<> (8*i)); + rejects([&] {defrelocate::relocate(badAnimation,animationSchema,animationMaps);}, + "unproven whole combo/recoil integers remain refusal candidates"); + } + auto truncationMaps=animationMaps; + // The structural tag is always present even before a truncated value's + // first numeric candidate; force analysis rather than its no-candidate no-op. + truncationMaps.gameEntries.emplace(defdecode::fieldTag("Animation"),1234567); + for(size_t end=7;ends['offset']] observations.append(dict(root=index,appearance=component,offset=offset,value=index, spans=overlaps,whole_primitive=len(overlaps)==1 and overlaps[0]['offset']==relative and overlaps[0]['size']==4)) - assert all(hashlib.sha256(p.read_bytes()).hexdigest()==hashes[str(p)] for p in sources),'native bank changed' args.work_root.resolve().mkdir(parents=True,exist_ok=True) work=Path(tempfile.mkdtemp(prefix='creature-animation-audit-',dir=args.work_root.resolve())) report=dict(source_sha256=hashes,counts=dict(counts),layout_failures=failures, candidate_observations=observations,clone_eligibility_verified=False) + if args.payload_exe: + result=subprocess.run([str(args.payload_exe.resolve()),'--creature-animation-name-audit', + str(args.install.resolve()),str(args.schema.resolve())],capture_output=True,text=True,timeout=120) + assert result.returncode==0,result.stderr + payloads=json.loads(result.stdout) + assert not payloads['clone_eligibility_verified'] + maps=dict(payloads['name_map']);seen=set();rewrites=0 + for row in payloads['records']: + index=row['index'];assert index in decoded and index not in seen;seen.add(index) + end,spans=decoded[index];original=records[index]['data'] + before=original[:3]+struct.pack(' Date: Sat, 10 Oct 2026 12:28:48 -0600 Subject: [PATCH 14/75] Add atomic private Creature visual graph cloning with independent retail audit --- CMakeLists.txt | 3 + docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 38 +++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 2 +- libs/forgecore/include/forge/defedit.hpp | 16 +++ libs/forgecore/include/forge/defrelocate.hpp | 6 +- libs/forgecore/src/defedit.cpp | 97 +++++++++++++ libs/forgecore/src/defrelocate.cpp | 12 +- tests/test_creaturevisualclone.cpp | 140 +++++++++++++++++++ tests/test_defrelocate.cpp | 5 + tools/check_all.py | 2 + tools/test_creature_visual_clone_api.py | 97 +++++++++++++ 12 files changed, 411 insertions(+), 9 deletions(-) create mode 100644 tests/test_creaturevisualclone.cpp create mode 100644 tools/test_creature_visual_clone_api.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 06d600d2..6ccf19b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,9 @@ add_test(NAME fableforge_binclone_tests COMMAND fableforge_binclone_tests) add_executable(fableforge_creatureclone_tests tests/test_creatureclone.cpp) target_link_libraries(fableforge_creatureclone_tests PRIVATE forgecore_slice) add_test(NAME fableforge_creatureclone_tests COMMAND fableforge_creatureclone_tests) +add_executable(fableforge_creaturevisualclone_tests tests/test_creaturevisualclone.cpp) +target_link_libraries(fableforge_creaturevisualclone_tests PRIVATE forgecore_slice) +add_test(NAME fableforge_creaturevisualclone_tests COMMAND fableforge_creaturevisualclone_tests) add_executable(fableforge_binwrite_tests tests/test_binwrite.cpp) target_link_libraries(fableforge_binwrite_tests PRIVATE forgecore_slice) add_test(NAME fableforge_binwrite_tests COMMAND fableforge_binwrite_tests) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 3b43b7c7..9ca5496b 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -317,3 +317,41 @@ Next: implement the coordinated root/body/appearance append plan. Selected visua rows must receive new component indices and localized new-root original parents; unselected shared gameplay rows retain their original-parent bytes. Do not use a value-wide old-parent map or mark borrowed components as privately owned. + +## Coordinated private visual core API + +Animation coverage head cdce763 Windows CI 38073837976 passed. New core API +`defedit::cloneCreatureVisuals` appends root/CCreatureDef/CAppearanceDef with +three explicit bounded ASCII names, native signed index limits and no new compiled +name-CRC ambiguity. Donor and selected original-parent rows must be clean counted +Creature listings with exactly one correctly typed/matching visual component. +Selected rows receive their new component index and new-root original parent. +All twelve bytes of every unselected CDefs row remain exact, including shared +gameplay data/parent references that might otherwise match the relocation map. +Other verified graph references relocate, while models/textures/animations remain +shared. A disposable File holds all payload/metadata appends until a noexcept +commit; any later component-name/capacity/ordinal/refusal leaves the caller exact. + +Known-layout validation can now be requested even without matching moved IDs. +Private cloning uses it, so malformed recognized reference/Animation layouts +cannot bypass checks via a no-candidate no-op. It does not validate opaque fields +or certify all runtime behavior. An initial truncation fixture targeted an opaque +integer, which deliberately has no full layout validation; corrected to the +known reference field. Synthetic ownership/inheritance, clone-of-clone, namespace, +late-name, count/type/owner/layout and pre-mutation refusal cases pass. + +Seven affected/neighbor CTests pass (new private API, prior shared Creature API, +generic clone, relocation, namespaces, FMP links and BIN serialization). Inventory +is 57 CTests; not a full-suite claim. Native `creature-private-api-asz7xrrz` accepts +494 donors and makes 23 expected non-mutating refusals: fifteen missing visual +shapes and eight unresolved root payloads. Output has 16,246 records (14,764 prior +plus 1,482 private additions). Independent raw audit checks exact selected-owner +deltas, all prior payloads, full original numeric-table prefix, original name-pool +records, new ordinals/name CRCs and reload. Source SHA256 unchanged; only paired +definition-bank evidence is emitted, not copied full installs or assets. Earlier +kk14wvqv packet lacked the integrated numeric-table/ordinal/CRC checks; final +asz7xrrz includes them. No private publisher, CLI, GUI, archive or gameplay claim. + +Next: a typed paired publisher contract that recomputes the whole three-record +plan against the fresh source, followed by the private CLI and GUI controls. +Keep prior shared/exact modes and their tested checkpoint intact. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 31ea62a2..f3017be4 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Canonical animation boundaries now resolve appearance findings; 494 of 502 observed three-node graphs have no unresolved candidates, eight root findings remain. Private append/ownership plan is next; no eligibility claim. RC2 unchanged. +> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Coordinated core API passes 494 native donors and 23 expected refusals with exact prior-bank preservation. Typed paired publication, CLI and GUI are next; no runtime/full-feature claim. RC2 unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 930702e8..9aa9b62c 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Canonical animation boundaries now resolve appearance findings; 494 of 502 observed three-node graphs have no unresolved candidates, eight root findings remain. Private append/ownership plan is next; no eligibility claim. RC2 unchanged. +> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Coordinated core API passes 494 native donors and 23 expected refusals with exact prior-bank preservation. Typed paired publication, CLI and GUI are next; no runtime/full-feature claim. RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there diff --git a/libs/forgecore/include/forge/defedit.hpp b/libs/forgecore/include/forge/defedit.hpp index 6cd41c54..d4486c64 100644 --- a/libs/forgecore/include/forge/defedit.hpp +++ b/libs/forgecore/include/forge/defedit.hpp @@ -78,6 +78,22 @@ size_t cloneEntry(bin::File& file, std::string_view donorName, std::string newNa size_t cloneCreature(bin::File& file, const defschema::Schema& schema, size_t donorIndex, std::string newName); +struct CreatureVisualCloneResult { + size_t rootIndex, bodyIndex, appearanceIndex; +}; +// Append a private root/CCreatureDef/CAppearanceDef graph in memory. Explicit +// names must be unique printable ASCII (127-byte maximum) and introduce no +// compiled-name CRC ambiguity. Only the two selected visual CDefs rows receive +// new component indices/new-root original parents; every other CDefs row stays +// byte-identical, retaining shared gameplay ownership. Verified references in +// all three payloads relocate; unresolved candidates refuse the whole plan. +// All prior slots and loaded name records stay exact. A disposable File holds +// every append until validation succeeds, so refusal never mutates the caller. +// Models, textures and animations remain shared. No files/publication here. +CreatureVisualCloneResult cloneCreatureVisuals(bin::File& file, + const defschema::Schema& schema, size_t donorIndex, + std::string rootName, std::string bodyName, std::string appearanceName); + // The raw value bytes of one field (everything between its tag and the next // tag), for the types encodeValue does not parse -- maps, vectors, nested // structs. Same entry/field location rules as setField. diff --git a/libs/forgecore/include/forge/defrelocate.hpp b/libs/forgecore/include/forge/defrelocate.hpp index a3166c76..b7df078c 100644 --- a/libs/forgecore/include/forge/defrelocate.hpp +++ b/libs/forgecore/include/forge/defrelocate.hpp @@ -37,11 +37,13 @@ struct Analysis { // a conservative rejection guard, never a source of rewrite locations. // This is not an audit of references in other containers or runtime code. Analysis analyze(const bin::Entry& entry, const defschema::Schema& schema, - const Maps& maps); + const Maps& maps, bool validateKnownLayouts = false); // Returns a copy with only verified reference words changed. Any unresolved // candidate refuses the entire operation; entry and maps always stay const. +// validateKnownLayouts also parses recognized layouts when there is no moved-ID +// candidate. It does not validate opaque fields or certify runtime eligibility. std::vector relocate(const bin::Entry& entry, const defschema::Schema& schema, - const Maps& maps); + const Maps& maps, bool validateKnownLayouts = false); } // namespace forge::defrelocate diff --git a/libs/forgecore/src/defedit.cpp b/libs/forgecore/src/defedit.cpp index 6328b036..0e9e38bc 100644 --- a/libs/forgecore/src/defedit.cpp +++ b/libs/forgecore/src/defedit.cpp @@ -3,6 +3,7 @@ #include "forge/unicode.hpp" #include +#include #include #include #include @@ -12,6 +13,7 @@ #include #include #include +#include #include #include "forge/defdecode.hpp" @@ -386,4 +388,99 @@ size_t cloneEntry(bin::File& file,std::string_view donorName,std::string newName return file.cloneEntry(selected,std::move(newName)); } +CreatureVisualCloneResult cloneCreatureVisuals(bin::File& file,const defschema::Schema& schema, + size_t donorIndex,std::string rootName,std::string bodyName,std::string appearanceName) { + const auto& entries=file.entries(); + // CSubDefInfo persists signed-long definition/original-parent indices. + if(entries.size()>size_t(INT32_MAX)-2u) + throw std::runtime_error("private creature clone: append indices exceed the signed reference domain"); + auto read=[](const auto& data,size_t offset) { + if(offset>data.size() || data.size()-offset<4) + throw std::runtime_error("private creature clone: truncated component word"); + return uint32_t(data[offset]) | uint32_t(data[offset+1])<<8 | + uint32_t(data[offset+2])<<16 | uint32_t(data[offset+3])<<24; + }; + auto rootLayout=[&](size_t index) { + if(index>=entries.size() || entries[index].definition!="CREATURE") + throw std::runtime_error("private creature clone: donor/original parent must be a Creature index"); + const auto& entry=entries[index];const auto decoded=defdecode::decode(entry,schema); + if(!decoded.clean() || defdecode::encode(decoded)!=entry.data || entry.data.size()<5 || + decoded.fields.empty() || decoded.fields.front().name!="Components" || + std::any_of(decoded.fields.begin(),decoded.fields.end(),[](const auto& field){return !field.tagOk;})) + throw std::runtime_error("private creature clone: unverified Creature component layout"); + const size_t count=size_t(entry.data[3]) | size_t(entry.data[4])<<8; + if(!count || decoded.prefixLen!=5+12*count) + throw std::runtime_error("private creature clone: empty or malformed Creature listing"); + return count; + }; + const auto count=rootLayout(donorIndex); + const auto& root=entries[donorIndex]; + std::array rows{}; + std::array donors{donorIndex,0,0}; + constexpr std::array kinds{"CCreatureDef","CAppearanceDef"}; + for(size_t k=0;k=entries.size() || entries[donors[k+1]].definition!=kinds[k]) + throw std::runtime_error("private creature clone: requires one correctly typed "+std::string(kinds[k])+" row"); + const auto owner=read(root.data,13+12*rows[k]); + const auto ownerCount=rootLayout(owner);size_t ownerMatches=0; + for(size_t row=0;row names{std::move(rootName),std::move(bodyName),std::move(appearanceName)}; + std::array keys{}; + for(size_t i=0;i,3> payloads; + for(size_t i=0;i>(8*byte)); + }; + for(size_t i=0;i); + file=std::move(proposed); + return {first,size_t(first)+1,size_t(first)+2}; +} + } // namespace forge::defedit diff --git a/libs/forgecore/src/defrelocate.cpp b/libs/forgecore/src/defrelocate.cpp index 9f59924f..db7fad79 100644 --- a/libs/forgecore/src/defrelocate.cpp +++ b/libs/forgecore/src/defrelocate.cpp @@ -16,7 +16,8 @@ uint32_t word(const std::vector& data, size_t at) { } } -Analysis analyze(const bin::Entry& entry, const defschema::Schema& schema, const Maps& maps) { +Analysis analyze(const bin::Entry& entry, const defschema::Schema& schema, const Maps& maps, + bool validateKnownLayouts) { std::set changed; for (const auto* domain : {&maps.gameEntries, &maps.nameOffsets}) { for (const auto& [before, after] : *domain) { @@ -29,13 +30,13 @@ Analysis analyze(const bin::Entry& entry, const defschema::Schema& schema, const } } Analysis result; - if (changed.empty()) return result; + if (changed.empty() && !validateKnownLayouts) return result; for (size_t at = 0; at + 4 <= entry.data.size(); ++at) { const auto value = word(entry.data, at); if (changed.contains(value)) result.unresolved.push_back({at, value}); } // An opaque payload with no candidate needs no schema interpretation. - if (result.unresolved.empty()) return result; + if (result.unresolved.empty() && !validateKnownLayouts) return result; const auto* type = defdecode::resolveType(schema, entry.definition, entry.data); // Native category alias verified by CControlsDef::Transfer (01939BE8) and // the literal Controls/toggle tags in CONTROL_SCHEME payloads. Keep this @@ -490,8 +491,9 @@ Analysis analyze(const bin::Entry& entry, const defschema::Schema& schema, const return result; } -std::vector relocate(const bin::Entry& entry, const defschema::Schema& schema, const Maps& maps) { - const auto analysis = analyze(entry, schema, maps); +std::vector relocate(const bin::Entry& entry, const defschema::Schema& schema, const Maps& maps, + bool validateKnownLayouts) { + const auto analysis = analyze(entry, schema, maps, validateKnownLayouts); if (!analysis.unresolved.empty()) { const auto& first = analysis.unresolved.front(); fail(entry, "unclassified possible reference " + std::to_string(first.value) + diff --git a/tests/test_creaturevisualclone.cpp b/tests/test_creaturevisualclone.cpp new file mode 100644 index 00000000..c817ebd4 --- /dev/null +++ b/tests/test_creaturevisualclone.cpp @@ -0,0 +1,140 @@ +#include "forge/defedit.hpp" +#include "forge/defdecode.hpp" +#include "forge/temporarydirectory.hpp" +#include +#include +#include +#include +#include + +using Bytes=std::vector; +static void require(bool ok,const char* why){if(!ok)throw std::runtime_error(why);} +static void word(Bytes& out,uint32_t value){for(size_t i=0;i<4;++i)out.push_back(uint8_t(value>>(8*i)));} +static void put(Bytes& out,size_t offset,uint32_t value){for(size_t i=0;i<4;++i)out.at(offset+i)=uint8_t(value>>(8*i));} +static void field(Bytes& out,const char* name,uint32_t value){word(out,forge::defdecode::fieldTag(name));word(out,value);} +static bool same(const forge::bin::Entry& a,const forge::bin::Entry& b){return a.definition==b.definition&&a.name==b.name&&a.indexInDefinition==b.indexInDefinition&&a.data==b.data;} +static void write(const std::filesystem::path& path,const Bytes& bytes){std::ofstream out(path,std::ios::binary);out.write(reinterpret_cast(bytes.data()),bytes.size());require(bool(out),"fixture write failed");} +static Bytes read(const std::filesystem::path& path){std::ifstream in(path,std::ios::binary);require(bool(in),"fixture read failed");return Bytes(std::istreambuf_iterator(in),{});} +static Bytes root(bool donor){ + Bytes out{1,0,1,3,0}; + word(out,forge::defdecode::fieldTag("CCreatureDef"));word(out,3);word(out,0); + word(out,forge::defdecode::fieldTag("CAppearanceDef"));word(out,4);word(out,donor ? 1 : 0); + word(out,forge::defdecode::fieldTag("CBrainDef"));word(out,5);word(out,donor ? 1 : 0); + field(out,"Components",0);field(out,"Self",donor ? 1 : 0);field(out,"Other",99); + return out; +} +static Bytes body(){Bytes out{1,0,1};field(out,"Link",1);field(out,"Other",99);return out;} +static Bytes appearance(){ + Bytes out{1,0,1};field(out,"Link",1); + word(out,forge::defdecode::fieldTag("Animation")); + for(const auto value:{0u,4u,1u,3u,UINT32_MAX})word(out,value); + word(out,forge::defdecode::fieldTag("Tail"));out.push_back(0);return out; +} + +static int retail(char** argv){ + namespace fs=std::filesystem; + const auto sourceRoot=fs::weakly_canonical(fs::path(argv[2])); + const auto output=fs::weakly_canonical(fs::path(argv[4])); + const auto relative=output.lexically_relative(sourceRoot); + require(!fs::exists(output)&&!relative.empty()&&*relative.begin()=="..","private retail output must be fresh and outside the native install"); + const auto folder=sourceRoot/"data"/"CompiledDefs"; + auto file=forge::bin::File::open(folder/"names.bin",folder/"game.bin"); + const auto originals=file.entries();auto expected=originals;const auto names=file.originalNames(); + const auto schema=forge::defschema::Schema::load(argv[3]);nlohmann::json rows=nlohmann::json::array(); + for(size_t i=0;i{"","1","BAD\nNAME",std::string(128,'A'),std::string("BAD\xff",4)}) + for(size_t slot=0;slot<3;++slot)refuses(source,[&](auto& candidate){ + std::vector newNames{"R","B","A"};newNames[slot]=invalid; + defedit::cloneCreatureVisuals(candidate,schema,1,newNames[0],newNames[1],newNames[2]); + }); + refuses(source,[&](auto& candidate){defedit::cloneCreatureVisuals(candidate,schema,1,"SAME","SAME","A");}); + require(bin::nameCrc("plumless")==bin::nameCrc("buckeroo"),"CRC collision fixture changed"); + refuses(source,[&](auto& candidate){defedit::cloneCreatureVisuals(candidate,schema,1,"plumless","buckeroo","A");}); + for(const auto mutation:std::vector>{ + {3,4},{5,defdecode::fieldTag("CAppearanceDef")},{9,4},{9,UINT32_MAX}, + {13,99},{13,5},{25,99},{61,1}}){ + auto bad=source;auto data=root(true); + if(mutation.first==3)data[3]=uint8_t(mutation.second);else put(data,mutation.first,mutation.second); + bad.setEntryData(1,data);refuses(bad,clone); + } + auto bad=source;auto data=root(false);put(data,9,6);bad.setEntryData(0,data);refuses(bad,clone); + bad=source;data=body();put(data,15,1);bad.setEntryData(3,data);refuses(bad,clone); + bad=source;data=appearance();put(data,15,UINT32_MAX);bad.setEntryData(4,data);refuses(bad,clone); + bad=source;data=body();data.erase(data.begin()+10);bad.setEntryData(3,data);refuses(bad,clone); + bad=source;bad.addEntry("CREATURE","NONCANONICAL",root(true));refuses(bad,clone); + + const auto original=source.entries();auto working=source;const auto result=clone(working); + require(result.rootIndex==7&&result.bodyIndex==8&&result.appearanceIndex==9,"private append slots differ"); + auto rootExpected=root(true);put(rootExpected,9,8);put(rootExpected,13,7);put(rootExpected,21,9);put(rootExpected,25,7);put(rootExpected,53,7); + auto bodyExpected=body();put(bodyExpected,7,7);auto appearanceExpected=appearance();put(appearanceExpected,7,7); + require(working.entries()[7].data==rootExpected,"private root ownership/delta differs"); + require(working.entries()[8].data==bodyExpected&&working.entries()[9].data==appearanceExpected,"private component delta differs"); + require(working.entries()[7].indexInDefinition==2&&working.entries()[8].indexInDefinition==2&&working.entries()[9].indexInDefinition==1,"private ordinals differ"); + require(std::equal(original[1].data.begin()+29,original[1].data.begin()+41,working.entries()[7].data.begin()+29),"shared gameplay triple changed"); + for(size_t i=0;i Date: Sat, 10 Oct 2026 12:35:34 -0600 Subject: [PATCH 15/75] Publish private Creature graphs transactionally and expose explicit CLI mode --- CMakeLists.txt | 2 +- docs/EDITOR.md | 4 +- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 53 +++++++++++ src/definitionclone.hpp | 26 +++++- tests/test_creaturevisualclone.cpp | 52 +++++++++++ tools/check_all.py | 2 + tools/forge-cli/main.cpp | 43 +++++++-- tools/test_creature_private_clone_cli.py | 106 ++++++++++++++++++++++ 8 files changed, 272 insertions(+), 16 deletions(-) create mode 100644 tools/test_creature_private_clone_cli.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ccf19b0..34006586 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,7 +112,7 @@ add_executable(fableforge_creatureclone_tests tests/test_creatureclone.cpp) target_link_libraries(fableforge_creatureclone_tests PRIVATE forgecore_slice) add_test(NAME fableforge_creatureclone_tests COMMAND fableforge_creatureclone_tests) add_executable(fableforge_creaturevisualclone_tests tests/test_creaturevisualclone.cpp) -target_link_libraries(fableforge_creaturevisualclone_tests PRIVATE forgecore_slice) +target_link_libraries(fableforge_creaturevisualclone_tests PRIVATE fableforge) add_test(NAME fableforge_creaturevisualclone_tests COMMAND fableforge_creaturevisualclone_tests) add_executable(fableforge_binwrite_tests tests/test_binwrite.cpp) target_link_libraries(fableforge_binwrite_tests PRIVATE forgecore_slice) diff --git a/docs/EDITOR.md b/docs/EDITOR.md index 722a0bdf..0d9e4b35 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1444,7 +1444,9 @@ Decoded int32 `GroupDef` drafts offer a searched group picker: Object, Building The distinct **Clone creature** action appears for game-bank CREATURE records. It retargets only verified self references before paired publication; components, appearance and body definitions stay shared. The draft and Create confirmation make this limitation explicit. Empty, malformed or unresolved donors refuse before a draft opens. Folder/pack destinations, stale-source refusal, Close/Install protection and paired Restore use the existing clone safeguards. **Clone selected definition** still copies the exact donor payload without retargeting. -The same guarded mode is available through `forge-tools defs clone-creature --schema --out `. Private appearance/body cloning remains unfinished. Use owned outputs first; no gameplay validation is claimed. +The same guarded shared-component mode is available through `forge-tools defs clone-creature --schema --out `. + +The CLI also offers `forge-tools defs clone-creature-private --schema --out [--names ] [--json]`. It appends a coordinated root, CCreatureDef and CAppearanceDef, with explicit unique bounded ASCII names. Selected visual rows become owned by the new root; other gameplay components retain their original data and ownership. Models, textures and animation assets remain shared. The paired publisher independently recomputes the three-record plan before writing names.bin and game.bin together, retaining recovery records. Unsupported/malformed known layouts, missing visual components, unresolved references and new name-CRC ambiguities refuse before publication. Private GUI controls are not implemented yet. Use owned outputs first; no gameplay validation is claimed. Definition `CRGBColour` CLI input uses red, green, blue, alpha (0-255; omitted alpha defaults to 255). The writer converts to the native blue, green, red, alpha byte order. Existing unexpected lengths refuse. diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 9ca5496b..62078f28 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -355,3 +355,56 @@ asz7xrrz includes them. No private publisher, CLI, GUI, archive or gameplay clai Next: a typed paired publisher contract that recomputes the whole three-record plan against the fresh source, followed by the private CLI and GUI controls. Keep prior shared/exact modes and their tested checkpoint intact. + +## Typed private paired publication + +Core checkpoint e2a3c54 was pushed; its Windows CI 38075951769 was still running +at the last observation. `definitionwrite::publishClone` now accepts an explicit +`CreatureVisualCloneContract` containing schema, donor index and all three names. +It rejects mixed shared/private contracts and requires exactly three additions +in the loaded namespace. Before staging, it recomputes the canonical private plan +from the freshly reopened source and compares every record identity, ordinal and +payload. The existing PendingBanks pair, source/destination rechecks, offline +guards, alias refusal and recovery logic are reused, not replaced. + +Extended private CTest passes fresh output, exact existing-pair replacement and +in-place independent fixtures with paired originals/creation markers. It refuses +tampering in each appended payload, a changed prior slot, wrong donor/name, mixed +contracts, incompatible already-appended output, orphan recovery records and a +hardlinked source-bank alias. Fixtures contain only small definition-bank pairs. +No live-game guard probe, injected rename-failure rollback, new native publisher +audit, CLI/private GUI wiring or release/package claim yet. Prior core native +494/23 evidence remains about core cloning, not this new publication path. + +Next: build the existing publisher callers and verify shared/exact regressions, +then expose explicit private names through CLI and the compact GUI confirmation. + +Existing GUI/CLI publisher callers rebuilt successfully (session 84749 terminal +success). Shared Creature CLI gate `creature-clone-cli-66tqgmhc` passes all thirty +cases, including paired rename-denial rollback and explicit Restore. Those checks +used the pre-private-command binary with the new publisher header. + +Private command is now wired as `defs clone-creature-private + --schema ... --out +`. It uses canonical core cloning and the typed publisher; JSON reports +all three indices/names, private visual components and shared assets. The rebuild +is still live in session 47337; do not claim this command exercised yet. +`tools/test_creature_private_clone_cli.py` is ready for that binary and audits +small independently encoded fixtures, selected owner/reference deltas, original +numeric-table/name-pool prefixes, selectors and invalid names/CRC collisions. +Next: observe the same build handle, run that gate, repeat shared CLI against the +new binary, then commit the publisher/CLI checkpoint and add private GUI mode. + +Private command rebuild session 47337 completed successfully. Final independent +gate `creature-private-cli-syb42lx7` passes 28 cases, including exact existing and +in-place pair originals/Restore, a Windows name-pool rename denial with rollback, +and destination pool mismatch. The initial fixture 42wykc1a intentionally hit the +unknown-scalar guard because opaque Components scalar zero matched moved donor +zero; changed the synthetic scalar to 99, not the production guard. Earlier +favbp55q passed 24 cases before rollback/Restore additions. Seven scoped CTests +pass again; current-binary shared Creature CLI b98vm26u passes 30 cases and generic +clone `definition-clone-ascu504p` passes 35 including owned +sleeping-helper running-owner/alias checks. No real game process was launched. +Registered private gate in check_all; no full broad-suite claim. CLI documentation +now distinguishes shared components, private visual definitions and shared assets. +Next: commit/push this checkpoint, then private GUI mode and both-size UI evidence. diff --git a/src/definitionclone.hpp b/src/definitionclone.hpp index e7b3cb41..9d6d1950 100644 --- a/src/definitionclone.hpp +++ b/src/definitionclone.hpp @@ -19,10 +19,16 @@ inline void verifyCloneRecords(const forge::bin::File& expected,const forge::bin } // Clone publication appends a name pool and bank in one rollback-capable pair. // Existing outputs must still be the exact loaded namespace and record set. +struct CreatureVisualCloneContract { + const forge::defschema::Schema& schema; + size_t donorIndex; + std::string rootName, bodyName, appearanceName; +}; inline void publishClone(const forge::bin::File& bank,const std::filesystem::path& sourceNames, const std::filesystem::path& sourceBank,const std::filesystem::path& destination, bool trackRecovery=true,const forge::defschema::Schema* creatureSchema=nullptr, - std::optional creatureDonor={}) { + std::optional creatureDonor={}, + const CreatureVisualCloneContract* visualContract=nullptr) { namespace fs=std::filesystem; const auto target=fs::absolute(destination).lexically_normal(),names=target.parent_path()/"names.bin"; const auto sameFile=[](const fs::path& a,const fs::path& b) { @@ -35,15 +41,27 @@ inline void publishClone(const forge::bin::File& bank,const std::filesystem::pat throw std::runtime_error("definition clone: output aliases a source through a hard link"); requireOffline(target);requireOffline(names); const auto source=forge::bin::File::open(sourceNames,sourceBank); - if(bank.entries().size()!=source.entries().size()+1 || bank.originalNames()!=source.originalNames()) - throw std::runtime_error("definition clone: expected one appended record in the loaded namespace"); + if(visualContract && (creatureSchema || creatureDonor)) + throw std::runtime_error("definition clone: conflicting Creature clone contracts"); + const size_t additions=visualContract?3:1; + if(bank.entries().size()!=source.entries().size()+additions || bank.originalNames()!=source.originalNames()) + throw std::runtime_error("definition clone: unexpected appended record count or loaded namespace"); verifyCloneRecords(bank,source,source.entries().size()); const auto& clone=bank.entries().back(); if(clone.name.empty() || std::any_of(source.entries().begin(),source.entries().end(),[&](const auto& e){return e.name==clone.name;})) throw std::runtime_error("definition clone: appended name is not unique"); if(bool(creatureSchema)!=creatureDonor.has_value()) throw std::runtime_error("definition clone: incomplete Creature donor contract"); - if(creatureSchema) { + if(visualContract) { + if(sourceBank.filename()!="game.bin" || target.filename()!="game.bin") + throw std::runtime_error("private creature clone: source and destination must be game.bin"); + auto verified=source; + forge::defedit::cloneCreatureVisuals(verified,visualContract->schema,visualContract->donorIndex, + visualContract->rootName,visualContract->bodyName,visualContract->appearanceName); + // Compare all three identities, dense ordinals and payloads, not merely + // the root. Caller-provided component bytes cannot bypass the core plan. + verifyCloneRecords(verified,bank,verified.entries().size()); + } else if(creatureSchema) { if(sourceBank.filename()!="game.bin" || target.filename()!="game.bin") throw std::runtime_error("creature clone: source and destination must be game.bin"); auto verified=source; diff --git a/tests/test_creaturevisualclone.cpp b/tests/test_creaturevisualclone.cpp index c817ebd4..8b0613b2 100644 --- a/tests/test_creaturevisualclone.cpp +++ b/tests/test_creaturevisualclone.cpp @@ -1,6 +1,7 @@ #include "forge/defedit.hpp" #include "forge/defdecode.hpp" #include "forge/temporarydirectory.hpp" +#include "definitionclone.hpp" #include #include #include @@ -135,6 +136,57 @@ int main(int argc,char** argv)try{ const auto poolBytes=read(outNames);require(std::equal(oldNames.begin()+20,oldNames.end(),poolBytes.begin()+20),"private output altered loaded name records"); auto again=working;const auto second=defedit::cloneCreatureVisuals(again,schema,7,"SECOND_ROOT","SECOND_PRIVATE_BODY","SECOND_PRIVATE_APPEARANCE"); require(second.rootIndex==10&&second.bodyIndex==11&&second.appearanceIndex==12,"private clone of clone failed"); + namespace fs=std::filesystem; + const albion::definitionwrite::CreatureVisualCloneContract contract{schema,1,"NEW_ROOT","NEW_BODY","NEW_APPEARANCE"}; + const auto published=temp.path()/"published"/"game.bin"; + auto publish=[&](const auto& candidate,const auto& destination,const auto* proof){ + albion::definitionwrite::publishClone(candidate,namesPath,bankPath,destination,true,nullptr,{},proof); + }; + auto deniedPublication=[&](const auto& candidate,const auto* proof){ + bool denied=false;try{publish(candidate,published,proof);}catch(const std::runtime_error&){denied=true;} + require(denied,"invalid private publisher plan accepted"); + require(!fs::exists(published.parent_path()),"private publisher refusal created output"); + require(read(namesPath)==oldNames&&read(bankPath)==oldBank,"private publisher refusal changed source"); + }; + deniedPublication(working,static_cast(nullptr)); + for(size_t slot=0;slot<3;++slot){ + auto tampered=working;auto bytes=tampered.entries()[7+slot].data;bytes.back()^=1; + tampered.setEntryData(7+slot,bytes);deniedPublication(tampered,&contract); + } + auto wrongContract=contract;wrongContract.donorIndex=0;deniedPublication(working,&wrongContract); + auto wrongNameContract=contract;wrongNameContract.bodyName="DIFFERENT_BODY";deniedPublication(working,&wrongNameContract); + bool conflictDenied=false; + try{albion::definitionwrite::publishClone(working,namesPath,bankPath,published,true,&schema,1,&contract);} + catch(const std::runtime_error&){conflictDenied=true;} + require(conflictDenied&&!fs::exists(published.parent_path()),"conflicting private/shared contracts wrote output"); + auto priorTampered=working;priorTampered.setEntryData(5,{8,9});deniedPublication(priorTampered,&contract); + publish(working,published,&contract); + const auto publishedBank=bin::File::open(published.parent_path()/"names.bin",published); + albion::definitionwrite::verifyCloneRecords(working,publishedBank,working.entries().size()); + for(const auto& target:{published,published.parent_path()/"names.bin"}) + require(fs::is_regular_file(fs::path(target.string()+albion::backups::kCreatedSuffix)),"private fresh publication lacks creation marker"); + bool repeatDenied=false;try{publish(working,published,&contract);}catch(const std::runtime_error&){repeatDenied=true;} + require(repeatDenied,"private publisher overwrote an incompatible appended namespace"); + const auto existing=temp.path()/"existing"/"game.bin";fs::create_directories(existing.parent_path()); + write(existing,oldBank);write(existing.parent_path()/"names.bin",oldNames);publish(working,existing,&contract); + require(read(fs::path(existing.string()+albion::backups::kOrigSuffix))==oldBank&& + read(fs::path((existing.parent_path()/"names.bin").string()+albion::backups::kOrigSuffix))==oldNames, + "private paired publication did not retain both original files"); + const auto failed=temp.path()/"recovery-refusal"/"game.bin";fs::create_directories(failed.parent_path()); + write(fs::path(failed.string()+albion::backups::kCreatedSuffix),{1}); + bool recoveryDenied=false;try{publish(working,failed,&contract);}catch(const std::runtime_error&){recoveryDenied=true;} + require(recoveryDenied&&!fs::exists(failed)&&!fs::exists(failed.parent_path()/"names.bin"),"orphan recovery record did not prevent paired creation"); + const auto alias=temp.path()/"hardlink-refusal"/"game.bin";fs::create_directories(alias.parent_path()); + fs::create_hard_link(bankPath,alias); + bool aliasDenied=false;try{publish(working,alias,&contract);}catch(const std::runtime_error&){aliasDenied=true;} + require(aliasDenied&&read(alias)==oldBank&&!fs::exists(alias.parent_path()/"names.bin"),"hardlinked source alias did not refuse before writes"); + // In-place publication is confined to another independent fixture pair. + const auto inPlace=temp.path()/"in-place"/"game.bin";fs::create_directories(inPlace.parent_path()); + const auto inPlaceNames=inPlace.parent_path()/"names.bin";write(inPlace,oldBank);write(inPlaceNames,oldNames); + albion::definitionwrite::publishClone(working,inPlaceNames,inPlace,inPlace,true,nullptr,{},&contract); + albion::definitionwrite::verifyCloneRecords(working,bin::File::open(inPlaceNames,inPlace),working.entries().size()); + require(read(fs::path(inPlace.string()+albion::backups::kOrigSuffix))==oldBank&& + read(fs::path(inPlaceNames.string()+albion::backups::kOrigSuffix))==oldNames,"private in-place pair lacks exact originals"); require(read(namesPath)==oldNames&&read(bankPath)==oldBank,"private test changed original source pair"); std::cout<<"Private Creature graph: selected ownership, shared gameplay, atomic refusal and paired roundtrip PASS\n";return 0; }catch(const std::exception& error){std::cerr<] --out \n" " forge defs clone --out [--names ] [--json]\n" " forge defs clone-creature --schema --out [--names ] [--json]\n" + " forge defs clone-creature-private --schema --out [--names ] [--json]\n" " forge defs set-field [--names ] \n" " forge title add --donor \n" " --gui --desc --greet --comment --self \n" @@ -7605,7 +7606,8 @@ int questCard(const std::string& gameRoot, const std::string& schemaPath, // writes the modified bin (+ names.bin) to --out or a ".set-field.bin" // sibling. Untouched entries stay byte-identical (semantic re-serialize). int defsClone(const std::string& binPath,const std::string& namesPath,const std::string& donor, - const std::string& newName,const std::string& outPath,bool asJson,const std::string& creatureSchemaPath={}) { + const std::string& newName,const std::string& outPath,bool asJson,const std::string& creatureSchemaPath={}, + const std::optional>& visualNames={}) { namespace fs=std::filesystem; const fs::path input=fs::u8path(binPath),names=namesPath.empty()?input.parent_path()/"names.bin":fs::u8path(namesPath); auto file=forge::bin::File::open(names,input);std::optional exact; @@ -7625,13 +7627,30 @@ int defsClone(const std::string& binPath,const std::string& namesPath,const std: throw std::runtime_error("defs clone-creature: source and destination must be game.bin"); creatureSchema=forge::defschema::Schema::load(fs::u8path(creatureSchemaPath)); } - const auto added=creatureSchema?forge::defedit::cloneCreature(file,*creatureSchema,index,newName):forge::defedit::cloneEntry(file,index,newName); - albion::definitionwrite::publishClone(file,names,input,output,true,creatureSchema?&*creatureSchema:nullptr, - creatureSchema?std::optional(index):std::nullopt); + size_t added; + std::optional visual; + if(visualNames) { + if(!creatureSchema) throw std::runtime_error("defs clone-creature-private: --schema is required"); + visual=forge::defedit::cloneCreatureVisuals(file,*creatureSchema,index,newName,visualNames->first,visualNames->second); + added=visual->rootIndex; + const albion::definitionwrite::CreatureVisualCloneContract contract{*creatureSchema,index,newName,visualNames->first,visualNames->second}; + albion::definitionwrite::publishClone(file,names,input,output,true,nullptr,{},&contract); + } else { + added=creatureSchema?forge::defedit::cloneCreature(file,*creatureSchema,index,newName):forge::defedit::cloneEntry(file,index,newName); + albion::definitionwrite::publishClone(file,names,input,output,true,creatureSchema?&*creatureSchema:nullptr, + creatureSchema?std::optional(index):std::nullopt); + } const auto& entry=file.entries()[added]; - if(asJson) std::puts(json{{"bin",output.string()},{"names",(output.parent_path()/"names.bin").string()}, + json report{{"bin",output.string()},{"names",(output.parent_path()/"names.bin").string()}, {"donor_index",index},{"entry_index",added},{"definition",entry.definition},{"entry_name",entry.name}, - {"index_in_definition",entry.indexInDefinition},{"payload_size",entry.data.size()}}.dump(2).c_str()); + {"index_in_definition",entry.indexInDefinition},{"payload_size",entry.data.size()}}; + if(visual) { + report["body_index"]=visual->bodyIndex;report["appearance_index"]=visual->appearanceIndex; + report["body_name"]=file.entries()[visual->bodyIndex].name; + report["appearance_name"]=file.entries()[visual->appearanceIndex].name; + report["visual_components_private"]=true;report["assets_shared"]=true; + } + if(asJson) std::puts(report.dump(2).c_str()); else std::printf("Cloned record %zu to %zu: %s\n",index,added,entry.name.c_str()); return 0; } @@ -14317,8 +14336,9 @@ int main(int argc, char** argv) { return uiSetCardModel(gameRoot, schemaPath, entryName, (uint32_t)meshId, outRoot, inPlace); } - if(args.size()>=2 && args[0]=="defs" && (args[1]=="clone" || args[1]=="clone-creature")) { - const bool creature=args[1]=="clone-creature"; + if(args.size()>=2 && args[0]=="defs" && (args[1]=="clone" || args[1]=="clone-creature" || args[1]=="clone-creature-private")) { + const bool privateVisual=args[1]=="clone-creature-private"; + const bool creature=args[1]!="clone"; std::string names,out,creatureSchema;bool asJson=false;std::vector positional; for(size_t i=2;i --out [--names ] [--json]"); + if(positional.size()!=(privateVisual?5u:3u) || out.empty()) throw std::runtime_error(privateVisual? + "defs clone-creature-private: expected --schema --out ": + "defs clone: usage: defs clone --out [--names ] [--json]"); if(creature && creatureSchema.empty()) throw std::runtime_error("defs clone-creature: --schema is required"); - return defsClone(positional[0],names,positional[1],positional[2],out,asJson,creatureSchema); + return defsClone(positional[0],names,positional[1],positional[2],out,asJson,creatureSchema, + privateVisual?std::optional>(std::make_pair(positional[3],positional[4])):std::nullopt); } // forge defs set-field [--names ] // [--schema ] [--out ] [--json] diff --git a/tools/test_creature_private_clone_cli.py b/tools/test_creature_private_clone_cli.py new file mode 100644 index 00000000..4fa6dba9 --- /dev/null +++ b/tools/test_creature_private_clone_cli.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Small independent private Creature CLI/publisher delta and refusal gate.""" +import argparse +import json +import shutil +from pathlib import Path +import struct +import subprocess +import tempfile +import zlib +from test_fmp_links import read_bank, crc0 +from test_definition_clone import deny_rename + + +def main(): + repo=Path(__file__).resolve().parents[1] + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--tool',type=Path,default=repo/'build/forge-tools.exe') + parser.add_argument('--forge',type=Path,default=repo/'build/forge.exe') + parser.add_argument('--work-root',type=Path,default=repo/'build') + args=parser.parse_args() + work=Path(tempfile.mkdtemp(prefix='creature-private-cli-',dir=args.work_root.resolve())) + print('Evidence retained at',work,flush=True) + folder=work/'source/data/CompiledDefs';folder.mkdir(parents=True) + def field(name,value):return struct.pack(' Date: Sat, 10 Oct 2026 12:42:00 -0600 Subject: [PATCH 16/75] Add explicit private Creature GUI controls with both-size preservation checks --- docs/EDITOR.md | 4 +- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 50 ++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 2 +- gui/app.hpp | 3 +- gui/definitionbrowser.cpp | 60 +++++++++--- tools/check_all.py | 2 + tools/test_creature_private_clone_ui.py | 111 ++++++++++++++++++++++ 8 files changed, 218 insertions(+), 16 deletions(-) create mode 100644 tools/test_creature_private_clone_ui.py diff --git a/docs/EDITOR.md b/docs/EDITOR.md index 0d9e4b35..d8a714cc 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1446,7 +1446,9 @@ The distinct **Clone creature** action appears for game-bank CREATURE records. I The same guarded shared-component mode is available through `forge-tools defs clone-creature --schema --out `. -The CLI also offers `forge-tools defs clone-creature-private --schema --out [--names ] [--json]`. It appends a coordinated root, CCreatureDef and CAppearanceDef, with explicit unique bounded ASCII names. Selected visual rows become owned by the new root; other gameplay components retain their original data and ownership. Models, textures and animation assets remain shared. The paired publisher independently recomputes the three-record plan before writing names.bin and game.bin together, retaining recovery records. Unsupported/malformed known layouts, missing visual components, unresolved references and new name-CRC ambiguities refuse before publication. Private GUI controls are not implemented yet. Use owned outputs first; no gameplay validation is claimed. +The separate **Clone private creature visuals** action appends a coordinated root, CCreatureDef and CAppearanceDef, with explicit unique bounded ASCII names. Selected visual rows become owned by the new root; other gameplay components retain their original data and ownership. Models, textures and animation assets remain shared. The paired publisher independently recomputes the three-record plan before writing names.bin and game.bin together, retaining recovery records. Unsupported/malformed known layouts, missing visual components, unresolved references and new name-CRC ambiguities refuse before publication. Root, body and appearance names are separate fields in the scrolling card; the Create confirmation states that assets stay shared. Cancel and close-review retain the existing draft protection. Use owned outputs first; no gameplay validation is claimed. + +The same private mode is available through `forge-tools defs clone-creature-private --schema --out [--names ] [--json]`. Definition `CRGBColour` CLI input uses red, green, blue, alpha (0-255; omitted alpha defaults to 255). The writer converts to the native blue, green, red, alpha byte order. Existing unexpected lengths refuse. diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 62078f28..aa7b88c5 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -408,3 +408,53 @@ sleeping-helper running-owner/alias checks. No real game process was launched. Registered private gate in check_all; no full broad-suite claim. CLI documentation now distinguishes shared components, private visual definitions and shared assets. Next: commit/push this checkpoint, then private GUI mode and both-size UI evidence. + +Publisher/CLI checkpoint committed and pushed as 0ec6697. Windows CI 38076414718 +and prior core CI 38075951769 were still running at the last observation. + +Private GUI implementation is now in progress: a separate Creature action keeps +the shared/exact buttons, preflights a disposable private plan with CRC-distinct +probe names, captures the same source pair and destination, and displays explicit +root/body/appearance names with shared-assets/gameplay notices. Save passes all +three names through the canonical core and typed publisher before selecting the +new root; existing stale-source/destination and draft guards remain. The live GUI +build is session 77516; do not restart just because its observation times out. +The latest root-label/success-message tweak was made while that build was live, +so a subsequent incremental build is required before claiming the final UI +compiled. No private GUI clicks/screenshots, compact-window or runtime claim yet. +Next: finish this build, incrementally rebuild the final touched GUI source, +exercise private controls at both sizes and repeat the shared GUI regression. + +Core CI 38075951769 (e2a3c54) now passed; publisher CI 38076414718 last seen running. +GUI build 77516 completed; final incremental rebuild plus private gate 6661 also +completed successfully. `creature-private-ui-ghugbpwb` passes six native-input GUI +cases: save, cancel and duplicate-body-name refusal at 1440x900/scale1 and +800x600/scale1.5. Real buttons and all three input widgets are exercised. Independent +raw parsing plus canonical relocation observations check exact selected-row +ownership, other shared rows, all three output payloads, every prior record and +numeric-table/name-pool prefixes. Fresh pair recovery markers and Restore pass; +source game/name SHA256 unchanged. Outputs are definition banks only, no full +install or texture copies. Compact confirmation screenshot inspected: body and +appearance names, shared-assets notice and Create/Cancel fit; root field remains +reachable above in the scrolling card. No claim that all fields fit simultaneously. +Private stale-source/destination, close-review, missing-component and inherited +donor UI probes remain to broaden; core/CLI evidence is not a substitute for them. + +Registered native private-GUI gate in check_all. Full existing shared Creature GUI +regression is live in session 90978, packet `creature-clone-ui-jd57wu7o`; initial +large-window save/numeric/indexed-duplicate cases passed. Observe this handle to +completion rather than restarting. No full-suite, new archive or gameplay claim. + +Shared GUI session 90978 completed successfully: all 36 cases in jd57wu7o pass, +including both-size folder/pack, Close/Install, stale source, malformed/unknown +donors and exact recovery. Private extension session 62469 completed successfully: +`creature-private-ui-7c1azaou` passes ten new both-size empty-root/body/appearance, +destination-change refusal and close-review cases. Close Cancel retains the draft; +Review returns to the explicit confirmation, then save yields the exact three-node +plan and Restore removes its created pair. Six prior native GUI checks are reused, +not rerun. The default private gate covers all eight modes at each size; --only +allows scoped continuation. Every packet verifies source hashes unchanged. + +Next: private stale-bank/name source checks on a small owned definition fixture, +missing/unresolved and inherited-donor GUI probes, then a new development package +with extracted checks. Existing shared GUI archive remains untouched. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index f3017be4..e68135e1 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Coordinated core API passes 494 native donors and 23 expected refusals with exact prior-bank preservation. Typed paired publication, CLI and GUI are next; no runtime/full-feature claim. RC2 unchanged. +> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Core API passes 494 native donors/23 expected refusals; typed publication, CLI and initial both-size GUI checks pass. Broader private donor/draft probes and a new checkpoint remain; no runtime/full-feature claim. RC2 unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 9aa9b62c..39b9f207 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Coordinated core API passes 494 native donors and 23 expected refusals with exact prior-bank preservation. Typed paired publication, CLI and GUI are next; no runtime/full-feature claim. RC2 unchanged. +> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Core API passes 494 native donors/23 expected refusals; typed publication, CLI and initial both-size GUI checks pass. Broader private donor/draft probes and a new checkpoint remain; no runtime/full-feature claim. RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there diff --git a/gui/app.hpp b/gui/app.hpp index e6c26bc5..99a58b15 100644 --- a/gui/app.hpp +++ b/gui/app.hpp @@ -589,7 +589,7 @@ class App { bool setDefinitionBank(const std::string& bank); void browseDefinition(const std::string& name); bool beginDefinitionEdit(size_t field); - bool beginDefinitionClone(bool creature=false); + bool beginDefinitionClone(bool creature=false,bool privateVisual=false); bool saveDefinitionClone(); bool setDefinitionEditValue(const std::string& value); bool saveDefinitionEdit(); @@ -631,6 +631,7 @@ class App { std::filesystem::path definitionEditNames_,definitionEditBank_,definitionEditDestination_; std::string definitionEditField_,definitionEditType_,definitionEditError_,definitionEditSuccess_; char definitionEditValue_[128]={}; + char definitionCloneBodyName_[128]={},definitionCloneAppearanceName_[128]={}; char definitionNameSearch_[128]={}; std::pair definitionPaths() const; bool selectDefinitionIndex(size_t index,bool reveal=true); diff --git a/gui/definitionbrowser.cpp b/gui/definitionbrowser.cpp index 873fcdda..2d6be5c8 100644 --- a/gui/definitionbrowser.cpp +++ b/gui/definitionbrowser.cpp @@ -157,7 +157,7 @@ bool App::beginDefinitionEdit(size_t field) { return true; } catch(const std::exception& error) {definitionEditError_=error.what();return false;} } -bool App::beginDefinitionClone(bool creature) { +bool App::beginDefinitionClone(bool creature,bool privateVisual) { definitionEditError_.clear();definitionEditSuccess_.clear(); try { if(definitionEditing_) throw std::runtime_error("Save or cancel the current definition draft first."); @@ -167,6 +167,7 @@ bool App::beginDefinitionClone(bool creature) { if(!packDest_.empty()) {if(!modpack::isPack(fs::u8path(packDest_))) throw std::runtime_error("Choose a valid FableForge pack.");(void)modpack::load(fs::u8path(packDest_));} auto current=forge::bin::File::open(definitionLoadedNames_,definitionLoadedBank_); definitionwrite::verifyEntries(definitionFile_,current); + if(privateVisual && !creature) throw std::runtime_error("Private visuals require a Creature donor."); if(creature) { if(definitionBank_!="game.bin" || current.entries()[*definitionSelected_].definition!="CREATURE") throw std::runtime_error("Choose a Creature donor in the game bank."); @@ -175,7 +176,17 @@ bool App::beginDefinitionClone(bool creature) { auto probe=current;size_t suffix=0;std::string probeName; do {probeName="FORGE_CREATURE_DRAFT_"+std::to_string(suffix++);} while(std::any_of(current.entries().begin(),current.entries().end(),[&](const auto& entry){return entry.name==probeName;})); - (void)forge::defedit::cloneCreature(probe,definitionSchema_,*definitionSelected_,probeName); + if(privateVisual) { + std::array probeNames; + std::set keys; + for(const auto& entry:current.entries()) keys.insert(forge::bin::nameCrc(entry.name)); + for(auto& name:probeNames) { + do{name="FORGE_PRIVATE_DRAFT_"+std::to_string(suffix++);} + while(keys.contains(forge::bin::nameCrc(name))); + keys.insert(forge::bin::nameCrc(name)); + } + (void)forge::defedit::cloneCreatureVisuals(probe,definitionSchema_,*definitionSelected_,probeNames[0],probeNames[1],probeNames[2]); + } else (void)forge::defedit::cloneCreature(probe,definitionSchema_,*definitionSelected_,probeName); } definitionCloneNamesBytes_=definitionwrite::cloneBytes(definitionLoadedNames_); definitionCloneBankBytes_=definitionwrite::cloneBytes(definitionLoadedBank_); @@ -185,8 +196,9 @@ bool App::beginDefinitionClone(bool creature) { definitionEditDestination_=fs::u8path(packDest_.empty()?saveRoot():packDest_)/"data"/"CompiledDefs"/definitionBank_; const auto& donor=definitionFile_.entries()[definitionEditIndex_]; definitionEditField_=donor.name.empty()?"(unnamed record)":donor.name; - definitionEditType_=creature?"clone-creature":"clone";definitionEditing_=true;definitionCloning_=true; - definitionEditConfirm_=false;definitionEditValue_[0]=0;return true; + definitionEditType_=privateVisual?"clone-creature-private":creature?"clone-creature":"clone";definitionEditing_=true;definitionCloning_=true; + definitionEditConfirm_=false;definitionEditValue_[0]=0; + definitionCloneBodyName_[0]=0;definitionCloneAppearanceName_[0]=0;return true; } catch(const std::exception& error) {definitionEditError_=error.what();return false;} } bool App::saveDefinitionClone() { @@ -204,11 +216,22 @@ bool App::saveDefinitionClone() { throw std::runtime_error("The source bank or name pool changed; start the clone again."); auto cloned=definitionCloneFile_; const bool creature=definitionEditType_=="clone-creature"; - const auto added=creature?forge::defedit::cloneCreature(cloned,definitionSchema_,definitionEditIndex_,definitionEditValue_): - forge::defedit::cloneEntry(cloned,definitionEditIndex_,definitionEditValue_); - definitionwrite::publishClone(cloned,definitionEditNames_,definitionEditBank_,destination,true, - creature?&definitionSchema_:nullptr,creature?std::optional(definitionEditIndex_):std::nullopt); - definitionEditSuccess_="Created "+std::string(definitionEditValue_)+"."; + const bool privateVisual=definitionEditType_=="clone-creature-private"; + size_t added; + if(privateVisual) { + const auto result=forge::defedit::cloneCreatureVisuals(cloned,definitionSchema_,definitionEditIndex_, + definitionEditValue_,definitionCloneBodyName_,definitionCloneAppearanceName_); + added=result.rootIndex; + const definitionwrite::CreatureVisualCloneContract contract{definitionSchema_,definitionEditIndex_, + definitionEditValue_,definitionCloneBodyName_,definitionCloneAppearanceName_}; + definitionwrite::publishClone(cloned,definitionEditNames_,definitionEditBank_,destination,true,nullptr,{},&contract); + } else { + added=creature?forge::defedit::cloneCreature(cloned,definitionSchema_,definitionEditIndex_,definitionEditValue_): + forge::defedit::cloneEntry(cloned,definitionEditIndex_,definitionEditValue_); + definitionwrite::publishClone(cloned,definitionEditNames_,definitionEditBank_,destination,true, + creature?&definitionSchema_:nullptr,creature?std::optional(definitionEditIndex_):std::nullopt); + } + definitionEditSuccess_="Created "+std::string(definitionEditValue_)+(privateVisual?" with private body and appearance definitions (assets shared).":"."); definitionEditing_=false;definitionCloning_=false;definitionEditConfirm_=false; definitionCloneFile_={};definitionCloneNamesBytes_.clear();definitionCloneBankBytes_.clear(); definitionLoaded_=false;definitionBrowseRequest_.reset();definitionBrowseIndex_=added; @@ -271,12 +294,16 @@ void App::drawDefinitionBrowser(float pad,float inner,float cardInner) { ImGui::BeginDisabled(definitionEditing_ || definitionsBusy() || modFilesBusy()); if(theme::ghostButton("Clone creature",ImVec2(cardInner,0))) beginDefinitionClone(true); auto_.registerWidget("btn_definition_clone_creature");ImGui::EndDisabled(); + ImGui::BeginDisabled(definitionsBusy() || modFilesBusy()); + if(theme::ghostButton("Clone private creature visuals",ImVec2(cardInner,0))) beginDefinitionClone(true,true); + auto_.registerWidget("btn_definition_clone_creature_private");ImGui::EndDisabled(); } } if(definitionEditing_) { if(definitionCloning_) { ImGui::TextWrapped("Clone %s",definitionEditField_.c_str()); - theme::hint(definitionEditType_=="clone-creature"? + theme::hint(definitionEditType_=="clone-creature-private"? + "New root, body and appearance definitions. Other gameplay components and all assets stay shared.":definitionEditType_=="clone-creature"? "Retargets verified self references. Components, appearance and body stay shared.": "Copies this record unchanged. References keep their existing targets."); } else ImGui::TextWrapped("Edit %s (%s)",definitionEditField_.c_str(),definitionEditType_.c_str()); @@ -394,18 +421,27 @@ void App::drawDefinitionBrowser(float pad,float inner,float cardInner) { } else { const bool text=definitionCloning_ || definitionEditType_=="CCharString" || definitionEditType_=="CWideString"; const bool wide=definitionEditType_=="CWideString"; - if(text) ImGui::TextDisabled("%s",definitionCloning_?"New name (up to 127 ASCII characters)":wide?"Unicode text (up to 127 UTF-8 bytes)":"Text (up to 127 ASCII characters)"); + if(text) ImGui::TextDisabled("%s",definitionEditType_=="clone-creature-private"?"Root name (127 ASCII characters max)":definitionCloning_?"New name (up to 127 ASCII characters)":wide?"Unicode text (up to 127 UTF-8 bytes)":"Text (up to 127 ASCII characters)"); ImGui::SetNextItemWidth(cardInner);ImGui::InputText("##definitionvalue",definitionEditValue_,sizeof definitionEditValue_, text?ImGuiInputTextFlags_CallbackCharFilter:0,text?(wide?definitionUnicodeChar:definitionASCIIChar):nullptr); } auto_.registerWidget("input_definition_edit_value"); + if(definitionCloning_ && definitionEditType_=="clone-creature-private") { + ImGui::TextDisabled("Body name (127 ASCII characters max)");ImGui::SetNextItemWidth(cardInner); + ImGui::InputText("##privatebody",definitionCloneBodyName_,sizeof definitionCloneBodyName_,ImGuiInputTextFlags_CallbackCharFilter,definitionASCIIChar); + auto_.registerWidget("input_definition_clone_body"); + ImGui::TextDisabled("Appearance name (127 ASCII characters max)");ImGui::SetNextItemWidth(cardInner); + ImGui::InputText("##privateappearance",definitionCloneAppearanceName_,sizeof definitionCloneAppearanceName_,ImGuiInputTextFlags_CallbackCharFilter,definitionASCIIChar); + auto_.registerWidget("input_definition_clone_appearance"); + } if(!definitionEditConfirm_) { if(theme::ghostButton(definitionCloning_?"Create definition":"Save field",ImVec2(cardInner,0))) definitionEditConfirm_=true; auto_.registerWidget("btn_definition_save"); if(theme::ghostButton("Cancel edit",ImVec2(cardInner,0))) {definitionEditing_=false;definitionCloning_=false;definitionEditConfirm_=false;definitionCloneFile_={};definitionCloneNamesBytes_.clear();definitionCloneBankBytes_.clear();} auto_.registerWidget("btn_definition_edit_cancel"); } else { - const char* question=definitionCloning_?(definitionEditType_=="clone-creature"? + const char* question=definitionCloning_?(definitionEditType_=="clone-creature-private"? + "Create three private definitions (assets shared)?":definitionEditType_=="clone-creature"? "Create creature with shared components?":"Create this definition in the destination bank?"): "Write this field to the destination bank?"; const int answer=confirmRow(question,definitionCloning_?"Create":"Save",cardInner,S(28),"btn_definition_save_confirm"); diff --git a/tools/check_all.py b/tools/check_all.py index 041e339b..9ab4b2b5 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -131,6 +131,8 @@ def main(): "--payload-exe", "build/fableforge_defrelocate_tests.exe"], capture_output=True) ok &= run("Private Creature visual API, localized ownership, atomic refusal and independent paired output", [sys.executable, "tools/test_creature_visual_clone_api.py", "--install", install_root], capture_output=True) + ok &= run("Private Creature GUI controls at both sizes, independent native deltas and Restore", + [sys.executable, "tools/test_creature_private_clone_ui.py", "--install", install_root], capture_output=True) ok &= run("Creature donor self-reference layout and independent relocation audit", [sys.executable, "tools/audit_creature_self_refs.py", "--install", install_root, "--clone-api"], capture_output=True) ok &= run("Retail paired Creature clone, self-only deltas and Restore", diff --git a/tools/test_creature_private_clone_ui.py b/tools/test_creature_private_clone_ui.py new file mode 100644 index 00000000..f62d0bba --- /dev/null +++ b/tools/test_creature_private_clone_ui.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Private GUI controls and independent native deltas; no game/texture copies.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import struct +import subprocess +import tempfile +from audit_creature_visual_graph import analyze, listing +from test_definition_edit_ui import click +from test_fmp_links import read_bank + +MODES=('save','cancel','duplicate-body','empty-root','empty-body','empty-appearance','destination','close-review') + +def main(): + repo=Path(__file__).resolve().parents[1] + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--install',type=Path,required=True) + parser.add_argument('--gui',type=Path,default=repo/'build/FableForge.exe') + parser.add_argument('--forge',type=Path,default=repo/'build/forge.exe') + parser.add_argument('--payload-exe',type=Path,default=repo/'build/fableforge_defrelocate_tests.exe') + parser.add_argument('--schema',type=Path,default=repo/'docs/re_reference/def_schema.json') + parser.add_argument('--work-root',type=Path,default=repo/'build') + parser.add_argument('--only',nargs='+',choices=MODES) + args=parser.parse_args();native=args.install.resolve() + assert not args.work_root.resolve().is_relative_to(native),'evidence must be outside the source install' + paths=[native/'data/CompiledDefs'/name for name in ('game.bin','names.bin')] + source_bank=paths[0].read_bytes();source_pool=paths[1].read_bytes() + hashes={str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in paths} + originals=read_bank(native);graph=analyze(originals) + result=subprocess.run([str(args.payload_exe.resolve()),'--creature-visual-audit',str(native),str(args.schema.resolve())], + capture_output=True,text=True,timeout=120) + assert result.returncode==0,result.stderr + payloads={row['root']:row['payloads'] for row in json.loads(result.stdout)['records']} + donor=next(row for row in graph['records'] if not row['issues'] and + not any(payload['unresolved'] for payload in payloads[row['index']])) + nodes=[donor['index']]+[row['component'] for row in donor['visual_rows']] + slots=list(range(len(originals),len(originals)+3));maps=dict(zip(nodes,slots)) + expected=[] + for number,(node,payload) in enumerate(zip(nodes,payloads[donor['index']])): + data=bytearray(originals[node]['data']) + for rewrite in payload['rewrites']: + assert rewrite['domain']=='game' and rewrite['before'] in maps + struct.pack_into(' Date: Sat, 10 Oct 2026 12:43:42 -0600 Subject: [PATCH 17/75] Verify private Creature GUI stale-source refusal on small owned fixtures --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 14 ++++++++ tools/test_creature_private_clone_ui.py | 40 +++++++++++++++++++---- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index aa7b88c5..38e86211 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -458,3 +458,17 @@ allows scoped continuation. Every packet verifies source hashes unchanged. Next: private stale-bank/name source checks on a small owned definition fixture, missing/unresolved and inherited-donor GUI probes, then a new development package with extracted checks. Existing shared GUI archive remains untouched. + +GUI checkpoint 96e440c committed and pushed. Private stale-source gate now covers +owned game/name pairs with a screenshot barrier before deliberate header changes. +Session 43238 completed successfully: `creature-private-ui-wuxluix8` passes four +stale-bank/stale-names cases across both window sizes. Refusal retains the draft, +creates no output or originals and leaves the deliberately changed fixture pair +exact. Native source SHA256 remains unchanged. Each pair is 907,995 + 397,006 bytes; +only a tiny synthetic texture bank and empty level marker accompany it, not any +copied game/texture archives. The default gate now covers ten modes at both sizes; +sixteen prior private checks are reused, not rerun. Publisher CI 38076414718 still +live at last observation; no repeat build/test needed for this test-only change. + +Next: missing/unresolved and inherited-donor GUI probes, then development package +validation. No gameplay or full new release claim. diff --git a/tools/test_creature_private_clone_ui.py b/tools/test_creature_private_clone_ui.py index f62d0bba..e2720e2a 100644 --- a/tools/test_creature_private_clone_ui.py +++ b/tools/test_creature_private_clone_ui.py @@ -8,11 +8,13 @@ import struct import subprocess import tempfile +import time from audit_creature_visual_graph import analyze, listing from test_definition_edit_ui import click from test_fmp_links import read_bank +from test_texture_lookup import texture_bank -MODES=('save','cancel','duplicate-body','empty-root','empty-body','empty-appearance','destination','close-review') +MODES=('save','cancel','duplicate-body','empty-root','empty-body','empty-appearance','destination','close-review','stale-bank','stale-names') def main(): repo=Path(__file__).resolve().parents[1] @@ -57,6 +59,13 @@ def main(): for size,scale in [('1440x900',1),('800x600',1.5)]: for mode in args.only or MODES: case=work/(size+'-'+mode);case.mkdir();output=case/'output' + input_root=native;owned_paths=None + if mode.startswith('stale-'): + input_root=case/'source';compiled=input_root/'data/CompiledDefs';compiled.mkdir(parents=True) + owned_paths=[compiled/'game.bin',compiled/'names.bin'] + owned_paths[0].write_bytes(source_bank);owned_paths[1].write_bytes(source_pool) + levels=input_root/'data/Levels/FinalAlbion';levels.mkdir(parents=True);(levels/'unused.lev').write_bytes(b'') + texture_bank(input_root/'data/graphics/pc/textures.big',[(1,'Existing',20)]) names=['FORGE_GUI_PRIVATE_ROOT','FORGE_GUI_PRIVATE_BODY','FORGE_GUI_PRIVATE_APPEARANCE'] if mode=='duplicate-body':names[1]=originals[donor['index']]['name'] if mode.startswith('empty-'):names[('root','body','appearance').index(mode[6:])]='' @@ -69,8 +78,9 @@ def main(): if name:lines+=click(widget)+['input_text '+name,'frames 3'] lines+=[f'screenshot {(case/"draft.png").as_posix()}'] if mode=='cancel':lines+=click('btn_definition_edit_cancel')+['assert_state definition_editing 0'] - elif mode=='duplicate-body' or mode.startswith('empty-') or mode=='destination': + elif mode=='duplicate-body' or mode.startswith(('empty-','stale-')) or mode=='destination': if mode=='destination':lines+=[f'set saveroot {(case/"other-output").as_posix()}','wait_definitions','frames 3'] + if mode.startswith('stale-'):lines+=[f'screenshot {(case/"barrier.png").as_posix()}','frames 300'] lines+=['definition_save_refused','assert_state definition_editing 1','definition_cancel'] else: if mode=='close-review': @@ -81,12 +91,28 @@ def main(): 'assert_state definition_editing 0',f'assert_state definition_selected {slots[0]}', f'screenshot {(case/"created.png").as_posix()}'] lines+=['quit'];script=case/'private.txt';script.write_text('\n'.join(lines)+'\n',encoding='utf-8') - result=subprocess.run([str(args.gui.resolve()),'--install',str(native),'--size',size,'--auto',str(script)], - cwd=repo,capture_output=True,text=True,encoding='utf-8',timeout=150, - env=dict(os.environ,FABLEFORGE_AUTOMATION_HIDDEN='1',APPDATA=str(case/'profile'),LOCALAPPDATA=str(case/'local'))) - (case/'output.log').write_text(result.stdout+result.stderr,encoding='utf-8') + changed=None + with (case/'output.log').open('w',encoding='utf-8') as log: + process=subprocess.Popen([str(args.gui.resolve()),'--install',str(input_root),'--size',size,'--auto',str(script)], + cwd=repo,stdout=log,stderr=subprocess.STDOUT, + env=dict(os.environ,FABLEFORGE_AUTOMATION_HIDDEN='1',APPDATA=str(case/'profile'),LOCALAPPDATA=str(case/'local'))) + try: + if owned_paths: + deadline=time.monotonic()+60 + while not (case/'barrier.png').exists() and process.poll() is None and time.monotonic() Date: Sat, 10 Oct 2026 12:46:47 -0600 Subject: [PATCH 18/75] Verify inherited and unsupported Creature donors through private GUI controls --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 22 ++++++++++++++++++++ tools/check_all.py | 4 ++++ tools/test_creature_private_clone_ui.py | 25 ++++++++++++++++++----- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 38e86211..70d97715 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -472,3 +472,25 @@ live at last observation; no repeat build/test needed for this test-only change. Next: missing/unresolved and inherited-donor GUI probes, then development package validation. No gameplay or full new release claim. + +Stale-source checkpoint a49ec80 push session 33618 completed successfully. An +overlapping ordinary Git status read hit shared D: loose-object permission errors; +use the established C: object directory plus D: alternate rather than deleting, +overwriting or repairing shared objects. Other safe task work continued. + +Private donor-policy gate now selects supported inherited, missing-visual and +unresolved donors from the independently decoded source graph. `lx3e4x9z` passes +two inherited-donor saves: root 264 borrowed appearance from original parent 262; +the new selected parent is localized to the new root, all unselected triples and +prior records/pool stay exact, and Restore succeeds. `o2b2ybnl` passes two missing +visual-donor preflight refusals (root 113); `k4hez073` passes two unresolved-donor +preflight refusals. Both unsupported cases leave no draft/output and native hashes +unchanged. An initial unresolved selector accessed a missing-visual root absent +from canonical payload observations; restored the explicit shape check before the +lookup, then the gate passed. Production logic unchanged. Six new both-size cases +reuse the same GUI binary; prior twenty private checks are not repeated. + +Registered all three donor-policy checks in the native optional gate. Next: +commit this test checkpoint and create a separately named private-GUI development +archive, then verify its identities, documentation and selected extracted checks. +The older shared-GUI archive and public RC2 must remain unchanged. diff --git a/tools/check_all.py b/tools/check_all.py index 9ab4b2b5..70b43d8e 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -133,6 +133,10 @@ def main(): [sys.executable, "tools/test_creature_visual_clone_api.py", "--install", install_root], capture_output=True) ok &= run("Private Creature GUI controls at both sizes, independent native deltas and Restore", [sys.executable, "tools/test_creature_private_clone_ui.py", "--install", install_root], capture_output=True) + for donor_policy in ("inherited", "missing", "unresolved"): + ok &= run("Private Creature GUI donor policy " + donor_policy, + [sys.executable, "tools/test_creature_private_clone_ui.py", "--install", install_root, + "--donor-policy", donor_policy, "--only", "save"], capture_output=True) ok &= run("Creature donor self-reference layout and independent relocation audit", [sys.executable, "tools/audit_creature_self_refs.py", "--install", install_root, "--clone-api"], capture_output=True) ok &= run("Retail paired Creature clone, self-only deltas and Restore", diff --git a/tools/test_creature_private_clone_ui.py b/tools/test_creature_private_clone_ui.py index e2720e2a..4520186a 100644 --- a/tools/test_creature_private_clone_ui.py +++ b/tools/test_creature_private_clone_ui.py @@ -26,6 +26,7 @@ def main(): parser.add_argument('--schema',type=Path,default=repo/'docs/re_reference/def_schema.json') parser.add_argument('--work-root',type=Path,default=repo/'build') parser.add_argument('--only',nargs='+',choices=MODES) + parser.add_argument('--donor-policy',choices=('first','inherited','missing','unresolved'),default='first') args=parser.parse_args();native=args.install.resolve() assert not args.work_root.resolve().is_relative_to(native),'evidence must be outside the source install' paths=[native/'data/CompiledDefs'/name for name in ('game.bin','names.bin')] @@ -36,12 +37,20 @@ def main(): capture_output=True,text=True,timeout=120) assert result.returncode==0,result.stderr payloads={row['root']:row['payloads'] for row in json.loads(result.stdout)['records']} - donor=next(row for row in graph['records'] if not row['issues'] and - not any(payload['unresolved'] for payload in payloads[row['index']])) + preflight=args.donor_policy in ('missing','unresolved') + def matches(row): + if args.donor_policy=='missing':return bool(row['issues']) + if row['issues']:return False + unresolved=any(payload['unresolved'] for payload in payloads[row['index']]) + if args.donor_policy=='unresolved':return not row['issues'] and unresolved + return not row['issues'] and not unresolved and (args.donor_policy=='first' or + any(component['owner']!=row['index'] for component in row['visual_rows'])) + donor=next(row for row in graph['records'] if matches(row)) nodes=[donor['index']]+[row['component'] for row in donor['visual_rows']] slots=list(range(len(originals),len(originals)+3));maps=dict(zip(nodes,slots)) expected=[] - for number,(node,payload) in enumerate(zip(nodes,payloads[donor['index']])): + observed_payloads=[] if preflight else payloads[donor['index']] + for number,(node,payload) in enumerate(zip(nodes,observed_payloads)): data=bytearray(originals[node]['data']) for rewrite in payload['rewrites']: assert rewrite['domain']=='game' and rewrite['before'] in maps @@ -57,7 +66,7 @@ def main(): print('Evidence retained at',work,flush=True);cases=0 try: for size,scale in [('1440x900',1),('800x600',1.5)]: - for mode in args.only or MODES: + for mode in (('save',) if preflight else args.only or MODES): case=work/(size+'-'+mode);case.mkdir();output=case/'output' input_root=native;owned_paths=None if mode.startswith('stale-'): @@ -71,6 +80,7 @@ def main(): if mode.startswith('empty-'):names[('root','body','appearance').index(mode[6:])]='' lines=['wait_maps','wait_ready',f'set uiscale {scale}',f'set saveroot {output.as_posix()}', 'assets_tab 6','wait_definitions',f'definition_select {donor["index"]}','frames 3'] + start_lines=list(lines) lines+=click('btn_definition_clone_creature_private')+['assert_state definition_editing 1'] lines+=['definition_clone_refused','definition_clone_creature_refused', 'definition_bank_refused frontend.bin','assert_state definition_editing 1'] @@ -90,6 +100,10 @@ def main(): lines+=click('btn_definition_save_confirm')+['wait_ready','wait_definitions','frames 3', 'assert_state definition_editing 0',f'assert_state definition_selected {slots[0]}', f'screenshot {(case/"created.png").as_posix()}'] + if preflight: + # A refused donor must never open the three-name draft. + lines=start_lines+click('btn_definition_clone_creature_private')+[ + 'assert_state definition_editing 0',f'screenshot {(case/"refused.png").as_posix()}'] lines+=['quit'];script=case/'private.txt';script.write_text('\n'.join(lines)+'\n',encoding='utf-8') changed=None with (case/'output.log').open('w',encoding='utf-8') as log: @@ -113,7 +127,7 @@ def main(): assert owned_paths[0].read_bytes()==(changed if mode=='stale-bank' else source_bank) assert owned_paths[1].read_bytes()==(changed if mode=='stale-names' else source_pool) assert not list(input_root.rglob('*.forge-orig')),'stale refusal created originals' - if mode in ('save','close-review'): + if mode in ('save','close-review') and not preflight: created=read_bank(output);assert len(created)==len(originals)+3 and created[:len(originals)]==originals for node,slot,name,data in zip(nodes,slots,names,expected): assert created[slot]==dict(originals[node],name=name,data=data) @@ -131,6 +145,7 @@ def main(): finally: assert {str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in paths}==hashes,'native inputs changed' (work/'result.json').write_text(json.dumps({'passed':True,'cases':cases,'donor':donor['index'], + 'donor_policy':args.donor_policy,'visual_original_parents':[row['owner'] for row in donor['visual_rows']], 'native_hashes':hashes,'full_install_copies':False,'gameplay_tested':False},indent=2)+'\n',encoding='utf-8') From 390b5f1d714db02c09751ec10ef7973cff2171ea Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:51:05 -0600 Subject: [PATCH 19/75] Validate private Creature workflows from extracted development packages --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 30 +++++++++++++++++++++++ tools/test_creature_private_clone_cli.py | 5 ++-- tools/test_creature_private_clone_ui.py | 5 ++-- tools/test_package.py | 7 ++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 70d97715..97ced849 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -494,3 +494,33 @@ Registered all three donor-policy checks in the native optional gate. Next: commit this test checkpoint and create a separately named private-GUI development archive, then verify its identities, documentation and selected extracted checks. The older shared-GUI archive and public RC2 must remain unchanged. + +Donor-policy checkpoint 07a84cb committed/pushed. All three shipping targets are +current; forge.exe required relinking while GUI/tools were already up to date. +Added unrelated-cwd routing to the private CLI/GUI scripts and extracted-package +workflow registrations. Generated CLI.md unchanged: it documents forge.exe, +whereas private forge-tools syntax is in shipped EDITOR.md and tool help. + +Fresh local package (no public publication): +`dist/development/20261010-creature-private-gui/FableForge-0.18.1-dev-win64.zip` +SHA256 6422b166165a6aac34e8d51dbacb72b0f40c0ca71fd896ac82574619be4746c9, +37.2 MB. Existing shared-GUI archive hash remains +51d135c1e95f4fb51cfe4a501f164a7064aa3c1577d4467c137c789a54ffca4e. +Package manifest accurately records checkout 07a84cb with tracked_changes=true +for three test-harness changes; it is not a clean-checkout/executable-provenance +claim. Archive will not be overwritten just to refresh that metadata. + +Extracted gate is live in session 28722, packet `package-smoke-oey2hj9e`: +all 117 identities and 141 links pass, defs/shared CLI/private CLI pass so far. +Nine workflows were requested, including all private donor policies and shared +GUI/selector regressions. Observe this handle through completion, recording +omissions/skips accurately. No broad-suite/live-game/other-machine claim. + +Extracted progress: private first/inherited/missing/unresolved GUI workflows now +all pass using packaged GUI/Restore/schema from an unrelated working directory. +That includes twenty first-donor controls/draft/stale-source checks plus six donor +policy cases. Together with defs and both CLI checks, seven requested workflows +have passed. Session 28722 remains live in shared GUI regression, with selector +still to follow; retain the same archive and handle. Do not report all nine or +final omitted/skipped counts before result.json exists. Test-harness changes are +ready to commit independently of the still-running final regression. diff --git a/tools/test_creature_private_clone_cli.py b/tools/test_creature_private_clone_cli.py index 4fa6dba9..0557894d 100644 --- a/tools/test_creature_private_clone_cli.py +++ b/tools/test_creature_private_clone_cli.py @@ -18,6 +18,7 @@ def main(): parser.add_argument('--tool',type=Path,default=repo/'build/forge-tools.exe') parser.add_argument('--forge',type=Path,default=repo/'build/forge.exe') parser.add_argument('--work-root',type=Path,default=repo/'build') + parser.add_argument('--cwd',type=Path,default=repo) args=parser.parse_args() work=Path(tempfile.mkdtemp(prefix='creature-private-cli-',dir=args.work_root.resolve())) print('Evidence retained at',work,flush=True) @@ -53,7 +54,7 @@ def run(root_name='NEW_ROOT',body_name='NEW_BODY',appearance_name='NEW_APPEARANC output=output_root or work/f'case-{cases}';target=output/'data/CompiledDefs/game.bin' result=subprocess.run([str(args.tool.resolve()),'defs','clone-creature-private',str(input_bank or bank),donor, root_name,body_name,appearance_name,'--schema',str(schema),'--out',str(target),'--json'], - cwd=repo,capture_output=True,text=True,encoding='utf-8',timeout=60) + cwd=args.cwd.resolve(),capture_output=True,text=True,encoding='utf-8',timeout=60) (work/f'case-{cases}.log').write_text(result.stdout+result.stderr,encoding='utf-8');cases+=1 assert (result.returncode==0)==accepted,result.stdout+result.stderr assert bank.read_bytes()==original and names.read_bytes()==old_pool @@ -96,7 +97,7 @@ def run(root_name='NEW_ROOT',body_name='NEW_BODY',appearance_name='NEW_APPEARANC assert Path(str(target)+'.forge-orig').read_bytes()==original assert Path(str(target_names)+'.forge-orig').read_bytes()==old_pool restored=subprocess.run([str(args.forge.resolve()),'restore','--install',str(output)], - cwd=repo,capture_output=True,text=True,timeout=60) + cwd=args.cwd.resolve(),capture_output=True,text=True,timeout=60) assert restored.returncode==0,restored.stdout+restored.stderr assert target.read_bytes()==prior_bank and target_names.read_bytes()==prior_names (work/'result.json').write_text(json.dumps({'passed':True,'cases':cases})+'\n',encoding='utf-8') diff --git a/tools/test_creature_private_clone_ui.py b/tools/test_creature_private_clone_ui.py index 4520186a..641c8d78 100644 --- a/tools/test_creature_private_clone_ui.py +++ b/tools/test_creature_private_clone_ui.py @@ -25,6 +25,7 @@ def main(): parser.add_argument('--payload-exe',type=Path,default=repo/'build/fableforge_defrelocate_tests.exe') parser.add_argument('--schema',type=Path,default=repo/'docs/re_reference/def_schema.json') parser.add_argument('--work-root',type=Path,default=repo/'build') + parser.add_argument('--cwd',type=Path,default=repo) parser.add_argument('--only',nargs='+',choices=MODES) parser.add_argument('--donor-policy',choices=('first','inherited','missing','unresolved'),default='first') args=parser.parse_args();native=args.install.resolve() @@ -108,7 +109,7 @@ def matches(row): changed=None with (case/'output.log').open('w',encoding='utf-8') as log: process=subprocess.Popen([str(args.gui.resolve()),'--install',str(input_root),'--size',size,'--auto',str(script)], - cwd=repo,stdout=log,stderr=subprocess.STDOUT, + cwd=args.cwd.resolve(),stdout=log,stderr=subprocess.STDOUT, env=dict(os.environ,FABLEFORGE_AUTOMATION_HIDDEN='1',APPDATA=str(case/'profile'),LOCALAPPDATA=str(case/'local'))) try: if owned_paths: @@ -136,7 +137,7 @@ def matches(row): assert pool.read_bytes()[20:len(source_pool)]==source_pool[20:] for path in (bank,pool):assert Path(str(path)+'.forge-created').is_file() restore=subprocess.run([str(args.forge.resolve()),'restore','--install',str(output)], - cwd=repo,capture_output=True,text=True,timeout=60) + cwd=args.cwd.resolve(),capture_output=True,text=True,timeout=60) assert restore.returncode==0,restore.stdout+restore.stderr assert not bank.exists() and not pool.exists() else: diff --git a/tools/test_package.py b/tools/test_package.py index f9cbeba6..7c7fc3e9 100644 --- a/tools/test_package.py +++ b/tools/test_package.py @@ -237,6 +237,13 @@ def captured(value): '--tool', package / 'forge-tools.exe', '--forge', package / 'forge.exe', '--work-root', work]) run('creature_clone_cli', [sys.executable, repo / 'tools/test_creature_clone_cli.py', '--tool', package / 'forge-tools.exe', '--forge', package / 'forge.exe', '--cwd', cwd, '--work-root', work]) + run('creature_private_clone_cli', [sys.executable, repo / 'tools/test_creature_private_clone_cli.py', + '--tool', package / 'forge-tools.exe', '--forge', package / 'forge.exe', '--cwd', cwd, '--work-root', work]) + for donor_policy in ('first', 'inherited', 'missing', 'unresolved'): + run('creature_private_clone_ui_' + donor_policy, [sys.executable, repo / 'tools/test_creature_private_clone_ui.py', + '--install', args.root.resolve(), '--gui', package / 'FableForge.exe', '--forge', package / 'forge.exe', + '--schema', package / 'docs/re_reference/def_schema.json', '--donor-policy', donor_policy, + '--cwd', cwd, '--work-root', work, *([] if donor_policy=='first' else ['--only','save'])]) run('creature_clone_retail', [sys.executable, repo / 'tools/test_creature_clone_retail.py', '--install', args.root.resolve(), '--tool', package / 'forge-tools.exe', '--forge', package / 'forge.exe', '--schema', package / 'docs/re_reference/def_schema.json', '--cwd', cwd, '--work-root', work]) From 92f52d289ffffffefca23ad6e97479dcd86fa475 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:52:40 -0600 Subject: [PATCH 20/75] Record accepted extracted private Creature checkpoint and remaining custom-body scope --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 17 +++++++++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 97ced849..11f0fded 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -524,3 +524,20 @@ have passed. Session 28722 remains live in shared GUI regression, with selector still to follow; retain the same archive and handle. Do not report all nine or final omitted/skipped counts before result.json exists. Test-harness changes are ready to commit independently of the still-running final regression. + +Package-gate harness checkpoint 390b5f1 committed/pushed. Session 28722 is now +terminal success: final oey2hj9e result.json proves nine selected workflows passed, +130 not run and two unrelated missing-corpus skips (controller/Aeon and F2 FMP). +All 117 shipped identities and 141 documentation links passed. Private package +checks exercise 28 CLI cases and 26 both-size GUI cases, including stale source, +inherited ownership and missing/unresolved preflight. Shared CLI/36 GUI cases and +the selector also passed using extracted executables. The archive hash stays +6422b166165a6aac34e8d51dbacb72b0f40c0ca71fd896ac82574619be4746c9; old shared +archive and RC2 are unchanged. No full-suite, clean-machine or gameplay claim. + +Next roadmap step: make the root's verified body/appearance definitions directly +reachable in the browser so privately cloned components can be inspected/edited. +Reuse counted CDefs and explicit index navigation, reject ambiguous/wrong-class +targets and disable navigation during drafts. This is read-only navigation, not +private asset copying or completion of custom skinned mesh import. Graphic/body +asset binding and optional custom-body import remain real outstanding scope. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index e68135e1..f1e01601 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Core API passes 494 native donors/23 expected refusals; typed publication, CLI and initial both-size GUI checks pass. Broader private donor/draft probes and a new checkpoint remain; no runtime/full-feature claim. RC2 unchanged. +> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Core, paired publication, CLI and 26 both-size private GUI checks pass. Extracted private checkpoint passes nine workflows; 130 not run/two unrelated skips. Body/appearance navigation and custom body asset work are next; no runtime/full-feature claim. RC2 unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 39b9f207..c688e839 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Core API passes 494 native donors/23 expected refusals; typed publication, CLI and initial both-size GUI checks pass. Broader private donor/draft probes and a new checkpoint remain; no runtime/full-feature claim. RC2 unchanged. +> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Core, paired publication, CLI and 26 both-size private GUI checks pass. Extracted private checkpoint passes nine workflows; 130 not run/two unrelated skips. Body/appearance navigation and custom body asset work are next; no runtime/full-feature claim. RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there From 21315c4005d6c6ceca0dee2227f32637aaba3703 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:01:38 -0600 Subject: [PATCH 21/75] Add verified Creature visual definition navigation and UI gates --- docs/AUTOMATION.md | 4 +- docs/EDITOR.md | 2 + docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 38 ++++++++ gui/automation.cpp | 8 +- gui/definitionbrowser.cpp | 13 +++ src/definitionread.hpp | 25 ++++++ tests/test_definitionfield.cpp | 27 ++++++ tools/check_all.py | 4 + ...t_creature_visual_navigation_fixture_ui.py | 88 +++++++++++++++++++ tools/test_creature_visual_navigation_ui.py | 74 ++++++++++++++++ 10 files changed, 280 insertions(+), 3 deletions(-) create mode 100644 tools/test_creature_visual_navigation_fixture_ui.py create mode 100644 tools/test_creature_visual_navigation_ui.py diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md index e144adc1..a9d4a9ea 100644 --- a/docs/AUTOMATION.md +++ b/docs/AUTOMATION.md @@ -581,7 +581,9 @@ Settings persistence is disabled under `--auto` so runs are deterministic. | `assert_file_contains ` | the file exists and contains the text (the rest of the line, e.g. `ScriptName UiNamedBarrel;`) | | `assert_state ` | see `dump_state` for keys | | `assert_state_min ` / `assert_state_max ` | a numeric state key is at least / at most n (e.g. `texture_thumbs_made`, the Textures list thumbnails decoded so far) | -| `assert_widget ` | the widget was drawn this frame | +| `assert_widget ` | the widget has been registered (registrations persist across frames) | +| `clear_widget_registry` | clear retained widget rectangles; wait for redraw before checking current presence | +| `assert_widget_absent ` | the widget is not registered; clear the registry and wait for redraw first to test current absence | | `assert_log ` | some app log line contains the text (background job notes, e.g. `1 stitched`) | | `world_tab 0\|1`, `world_select `, `world_move `, `world_move_refused ...`, `world_owner `, `world_sees <0\|1>`, `world_stitch <0\|1> [feather]`, `world_revert`, `world_undo`, `world_redo`, `world_apply`, `wait_world` | the World tab: queue moves / region edits, stitch seams after the apply (feather -1 = auto), write them; state keys `world_*` | | `dump_state` | write every state key to the log | diff --git a/docs/EDITOR.md b/docs/EDITOR.md index d8a714cc..bc400831 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1450,6 +1450,8 @@ The separate **Clone private creature visuals** action appends a coordinated roo The same private mode is available through `forge-tools defs clone-creature-private --schema --out [--names ] [--json]`. +Game-bank CREATURE details offer **Open body definition** and **Open appearance definition** when the counted component listing has one valid target of the matching class. These links use record indices, including for inherited components, and are disabled while a definition draft is open. Editing a component affects every creature sharing it; navigation alone does not make components or their model/texture/animation assets private. Missing, ambiguous, wrong-class or out-of-range links are not offered. + Definition `CRGBColour` CLI input uses red, green, blue, alpha (0-255; omitted alpha defaults to 255). The writer converts to the native blue, green, red, alpha byte order. Existing unexpected lengths refuse. Inline CCharString drafts accept up to 127 printable ASCII characters, including empty text. Non-ASCII narrow input is filtered; legacy byte strings remain read-only and exact. CWideString drafts accept valid Unicode up to 127 UTF-8 bytes, including supplementary characters and empty text. Existing Unicode values within that limit can be edited or saved unchanged. C0/C1 controls, malformed encoding, overlong originals, missing terminators and opaque trailers remain unsupported. The shared writer requires one complete NUL-terminated original string before replacement. diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 11f0fded..2dd80bb1 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -541,3 +541,41 @@ Reuse counted CDefs and explicit index navigation, reject ambiguous/wrong-class targets and disable navigation during drafts. This is read-only navigation, not private asset copying or completion of custom skinned mesh import. Graphic/body asset binding and optional custom-body import remain real outstanding scope. + +Read-only body/appearance navigation implementation is in progress. New +definitionread helper requires a clean matching CREATURE decode, exact counted +CDefs prefix and one correctly typed/in-range component row. Only CCreatureDef +and CAppearanceDef are supported; names never become indices. GUI queues explicit +record navigation for the next frame and disables both links during a draft. +Links live inside the scrolling detail panel with a shared-component edit warning. +No bank writes or private ownership/asset inference are introduced. + +Focused definitionfield CTest passes valid numeric-name links and duplicate, +missing, wrong-class, out-of-range and malformed listing refusal. Initial malformed +fixture decode itself threw before link lookup; the test now mirrors the browser's +no-decoded-selection refusal path. GUI build is live in session 35899; no actual +link clicks or both-size navigation acceptance claim yet. The already validated +private archive remains unchanged, not rebuilt with this new UI work. +Next: observe that build, add independent native/synthetic navigation controls +checks, then proceed toward Graphic/body asset binding. + +Navigation GUI build completed successfully. New native read-only gate +`tools/test_creature_visual_navigation_ui.py` passes at 1440x900/scale 1 and +800x600/scale 1.5, with eight independently parsed component-target jumps and +eight disabled-link draft guards across own/inherited roots. All compiled-bank +hashes remain unchanged. Evidence: external 20261010 packet +`creature-visual-navigation-ui-0459c39j`; compact body screenshot inspected. +Focused definitionfield CTest passes again. Registered the native gate in +check_all; broad suite not rerun. Numeric-name and malformed-reference coverage +is currently helper-unit coverage, not synthetic GUI acceptance. Source work +only: existing private/shared archives and public RC2 remain unchanged. + +Synthetic navigation acceptance now passes both sizes in packet +`creature-visual-navigation-fixture-ui-qv727sfw`: deliberately crossed numeric +component names resolve by index, wrong-class/out-of-range/duplicate/empty/bad-count +roots offer no links, and a body-only root retains its valid body link. All tiny +fixture files remain exact. First packet g4vd_ybt failed because the automation +widget registry retains prior frames, not because invalid links were rendered. +Added explicit clear_widget_registry and assert_widget_absent commands and +corrected AUTOMATION.md's prior inaccurate this-frame assertion description. +Registered the synthetic gate; no broad-suite or packaged-navigation claim. diff --git a/gui/automation.cpp b/gui/automation.cpp index 1e074a6b..24a9b0ba 100644 --- a/gui/automation.cpp +++ b/gui/automation.cpp @@ -1687,8 +1687,12 @@ bool Automation::tick(App& app) { if (!app.logContains(rest)) fail("assert_log: no log line contains '" + rest + "'"); else note("ok " + line); ++pc_; } - else if (cmd == "assert_widget") { - if (!widgets_.count(rest)) fail("widget not on screen: " + rest); else note("ok " + line); + else if (cmd == "clear_widget_registry") { + widgets_.clear();widgetClips_.clear();note("ok " + line);++pc_; + } + else if (cmd == "assert_widget" || cmd == "assert_widget_absent") { + const bool present=widgets_.count(rest)!=0; + if (present!=(cmd=="assert_widget")) fail("unexpected widget presence: " + rest); else note("ok " + line); ++pc_; } else if (cmd == "dump_widget") { diff --git a/gui/definitionbrowser.cpp b/gui/definitionbrowser.cpp index 2d6be5c8..90d77bd7 100644 --- a/gui/definitionbrowser.cpp +++ b/gui/definitionbrowser.cpp @@ -582,6 +582,19 @@ void App::drawDefinitionViewport(const ImVec2&,const ImVec2&) { } if(!definitionDecodeError_.empty()) {ImGui::TextWrapped("%s",definitionDecodeError_.c_str());auto_.registerWidget("definition_decode_error");} ImGui::BeginChild("##definitiondetail",ImGui::GetContentRegionAvail()); + if(definitionBank_=="game.bin" && definitionDecoded_ && entry.definition=="CREATURE") { + for(const auto& [kind,label,widget]:std::array,2>{ + std::tuple{"CCreatureDef","Open body definition","btn_definition_body"}, + std::tuple{"CAppearanceDef","Open appearance definition","btn_definition_appearance"}}) { + if(const auto target=definitionread::creatureVisualTarget(definitionFile_,entry,*definitionDecoded_,kind)) { + const auto text=std::string(label)+" (entry "+std::to_string(*target)+")"; + ImGui::BeginDisabled(definitionEditing_); + if(ImGui::SmallButton(text.c_str())) definitionBrowseIndex_=*target; + auto_.registerWidget(widget);ImGui::EndDisabled(); + } + } + theme::hint("Component edits affect every creature that shares that definition."); + } if(*definitionSelected_ +#include #include "forge/bin.hpp" #include "forge/defdecode.hpp" namespace albion::definitionread { @@ -14,6 +15,30 @@ inline bool groupChoice(const forge::bin::File& file,const forge::bin::Entry& ow const auto family=groupFamily(owner); return family && index creatureVisualTarget(const forge::bin::File& file, + const forge::bin::Entry& entry,const forge::defdecode::Decoded& decoded,std::string_view kind) { + if(entry.definition!="CREATURE" || (kind!="CCreatureDef" && kind!="CAppearanceDef") || + !decoded.clean() || entry.data.size()<5 || decoded.fields.empty() || + decoded.fields.front().name!="Components" || + std::any_of(decoded.fields.begin(),decoded.fields.end(),[](const auto& field){return !field.tagOk;})) return std::nullopt; + const size_t count=size_t(entry.data[3]) | size_t(entry.data[4])<<8; + const size_t end=5+12*count; + if(!count || end>entry.data.size() || decoded.prefixLen!=end || + forge::defdecode::encode(decoded)!=entry.data) return std::nullopt; + const auto word=[&](size_t at) { + return uint32_t(entry.data[at]) | uint32_t(entry.data[at+1])<<8 | + uint32_t(entry.data[at+2])<<16 | uint32_t(entry.data[at+3])<<24; + }; + size_t matches=0;uint32_t index=0; + for(size_t row=0;row=file.entries().size() || file.entries()[index].definition!=kind) return std::nullopt; + return size_t(index); +} // Game-bank GroupDef stores a runtime entry index. The existing placement palette // resolves it only when the target definition is a *_GROUP type. inline std::optional groupTarget(const forge::bin::File& file,const forge::defdecode::Decoded& decoded) { diff --git a/tests/test_definitionfield.cpp b/tests/test_definitionfield.cpp index 800d7ab6..fcccc43c 100644 --- a/tests/test_definitionfield.cpp +++ b/tests/test_definitionfield.cpp @@ -43,6 +43,33 @@ int main() try { require(!albion::definitionread::groupChoice(groups,owner,0) && albion::definitionread::groupChoice(groups,owner,1),"Theme group family differs"); require(!albion::definitionread::groupChoice(groups,owner,size_t(-1)),"Out-of-range group accepted"); owner.definition="UNKNOWN";require(!albion::definitionread::groupFamily(owner),"Unknown category inferred"); + const auto schema=forge::defschema::Schema::loadText(R"({"CREATURE":{"fields":[{"name":"Components","type":"uint32"}]}})"); + auto append=[](auto& bytes,uint32_t value){for(size_t i=0;i<4;++i)bytes.push_back(uint8_t(value>>(8*i)));}; + std::vector data{1,0,1,2,0}; + for(const auto* kind:{"CCreatureDef","CAppearanceDef"}) { + append(data,forge::defdecode::fieldTag(kind));append(data,kind==std::string_view("CCreatureDef")?1:2);append(data,0); + } + append(data,forge::defdecode::fieldTag("Components"));append(data,99); + forge::bin::File visuals;visuals.addEntry("CREATURE","ROOT",data); + visuals.addEntry("CCreatureDef","2",{});visuals.addEntry("CAppearanceDef","1",{}); + auto target=[&](forge::bin::Entry entry,std::string_view kind)->std::optional{ + try{return albion::definitionread::creatureVisualTarget(visuals,entry,forge::defdecode::decode(entry,schema),kind);} + catch(const std::runtime_error&){return std::nullopt;} // Browser has no decoded selection after this refusal. + }; + require(target(visuals.entries()[0],"CCreatureDef")==1&&target(visuals.entries()[0],"CAppearanceDef")==2,"Visual link used numeric name rather than index"); + require(!target(visuals.entries()[0],"CBrainDef"),"Unknown component kind inferred"); + auto invalidRoot=visuals.entries()[0];invalidRoot.data[3]=1; + require(!target(invalidRoot,"CCreatureDef"),"Mismatched counted prefix accepted"); + invalidRoot=visuals.entries()[0];invalidRoot.data[9]=2; + require(!target(invalidRoot,"CCreatureDef"),"Wrong-class visual target accepted"); + invalidRoot=visuals.entries()[0];invalidRoot.data[9]=99; + require(!target(invalidRoot,"CCreatureDef"),"Out-of-range visual target accepted"); + invalidRoot=visuals.entries()[0]; + for(size_t i=0;i<4;++i)invalidRoot.data[17+i]=invalidRoot.data[5+i]; + require(!target(invalidRoot,"CCreatureDef"),"Duplicate component class accepted"); + require(!target(invalidRoot,"CAppearanceDef"),"Missing component class accepted"); + invalidRoot=visuals.entries()[0];invalidRoot.data.resize(4); + require(!target(invalidRoot,"CCreatureDef"),"Truncated visual listing accepted"); std::cout<<"Definition draft Unicode, exact originals, malformed data, byte limits and group families: PASS\n"; return 0; } catch(const std::exception& error) {std::cerr< Date: Sat, 10 Oct 2026 13:04:07 -0600 Subject: [PATCH 22/75] Expose typed Creature body model slots from validated prefix layout --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 22 ++++++++++++++++++++++ libs/forgecore/include/forge/fmplinks.hpp | 4 ++++ libs/forgecore/src/fmplinks.cpp | 5 ++++- tests/test_fmplinks.cpp | 12 ++++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 2dd80bb1..1a242a98 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -579,3 +579,25 @@ widget registry retains prior frames, not because invalid links were rendered. Added explicit clear_widget_registry and assert_widget_absent commands and corrected AUTOMATION.md's prior inaccurate this-frame assertion description. Registered the synthetic gate; no broad-suite or packaged-navigation claim. + +## Body mesh binding preparation + +Starting from pushed navigation checkpoint 21315c4. The active schema exposes +CCreatureDef fields only after its composite prefix: there is no ordinary +Graphic field to patch. Existing counted RandomAppearanceMorph parsing in +forgecore/fmplinks already validates body-part/alternative model words, texture +pairs and skeletal name offsets; expression meshes share the MeshEntry domain +but are not body meshes. Prior original nonempty-array evidence is recorded in +HANDOFF_NAV's 2026-10-05 recovery (9,935 audited words across 186 records), not +new gameplay or native-disassembly proof in this turn. + +CreaturePrefixLayout now exposes bodyModels {bodyPart, alternative, offset} +alongside its unchanged reference enumeration. It reuses the complete bounded +prefix parser, preserves native indices including empty body parts, and excludes +facial-expression meshes. No writer or asset mutation added. Targeted fmplinks +and defrelocate CTests pass; the mixed-domain fixture tests body-only membership +and an empty leading body part without renumbering. No broad-suite claim. +Next: independently audit these typed slots on retail components, then expose +body model selection/binding with exact four-byte deltas, mesh identity/type +validation, stale-source/destination protection and paired recovery. Skinned +custom import and runtime skeleton compatibility remain outstanding scope. diff --git a/libs/forgecore/include/forge/fmplinks.hpp b/libs/forgecore/include/forge/fmplinks.hpp index 01cb313e..9308cf4a 100644 --- a/libs/forgecore/include/forge/fmplinks.hpp +++ b/libs/forgecore/include/forge/fmplinks.hpp @@ -28,6 +28,10 @@ enum class Destination { GameEntry, NameOffset, NameCrc, MeshEntry, TextureEntry OptionalObjectEntry }; struct Location { size_t offset, part; Destination destination; }; struct CreaturePrefixLayout { + struct BodyModel { size_t bodyPart, alternative, offset; }; + // RandomAppearanceMorph model words only, not facial-expression meshes. + // Indices preserve empty body parts and the native alternative order. + std::vector bodyModels; std::vector references; std::vector structuralWords; // validated four-byte tags/counts }; diff --git a/libs/forgecore/src/fmplinks.cpp b/libs/forgecore/src/fmplinks.cpp index 4e317724..0fbcc298 100644 --- a/libs/forgecore/src/fmplinks.cpp +++ b/libs/forgecore/src/fmplinks.cpp @@ -295,7 +295,10 @@ CreaturePrefixLayout creaturePrefixLayout(std::span bytes) { for (size_t row = 0; row < prefix.wounds; ++row) for (size_t slot = 0; slot < 6; ++slot) result.references.push_back({prefix.woundRows + row * 44 + 8 + slot * 4, 0, Destination::TextureEntry}); - for (const auto& body : prefix.bodies) for (const auto& model : body) { + for (size_t bodyPart=0;bodyPart> (8*n)); auto relocatedExpected = relocatable.data; From 736054643d070d4c624ffeee24ec07c9fd2be9e8 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:06:15 -0600 Subject: [PATCH 23/75] Audit retail Creature body model slots independently without asset copies --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 13 ++++ tests/test_defrelocate.cpp | 28 ++++++++ tools/audit_creature_body_models.py | 84 +++++++++++++++++++++++ tools/check_all.py | 2 + 4 files changed, 127 insertions(+) create mode 100644 tools/audit_creature_body_models.py diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 1a242a98..fd94d739 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -601,3 +601,16 @@ Next: independently audit these typed slots on retail components, then expose body model selection/binding with exact four-byte deltas, mesh identity/type validation, stale-source/destination protection and paired recovery. Skinned custom import and runtime skeleton compatibility remain outstanding scope. + +Independent retail body-slot audit passes in external 20261010 packet +`creature-body-model-audit-c7_5_3vw`: all 330 CCreatureDef records accounted for, +329 parsed, one unsupported prefix refused, 1,745 exact model words across 188 +populated components. The independent parser matches body-part/alternative order, +offset and ID against canonical output, walks wound/texture/skeletal/expression +boundaries and later markers, and checks InitialActions at the decoded boundary. +All compiled-bank hashes unchanged. Stats itself remains opaque: the first audit +incorrectly assumed a four-byte Stats value; retail has a larger block, so the +diagnostic now explicitly stops structural interpretation at its marker. +No Stats grammar, graphics-bank identity/type, skeleton or runtime compatibility +claim. Targeted linker/relocation tests pass; registered the retail audit in +check_all without repeating the broad suite. Test executable mode is read-only. diff --git a/tests/test_defrelocate.cpp b/tests/test_defrelocate.cpp index ab5a3267..09678149 100644 --- a/tests/test_defrelocate.cpp +++ b/tests/test_defrelocate.cpp @@ -1,5 +1,6 @@ #include "forge/defrelocate.hpp" #include "forge/defdecode.hpp" +#include "forge/fmplinks.hpp" #include #include #include @@ -97,6 +98,33 @@ int audit(char** argv) { } int main(int argc, char** argv) { + if(argc==4 && std::string_view(argv[1])=="--creature-body-audit") { + try { + const auto folder=std::filesystem::path(argv[2])/"data"/"CompiledDefs"; + const auto schema=forge::defschema::Schema::load(argv[3]); + const auto source=forge::bin::File::open(folder/"names.bin",folder/"game.bin"); + auto rows=nlohmann::json::array(); + for(size_t i=0;ilen(data)-cursor:raise ValueError('truncated prefix') + start=cursor;cursor+=size;return start + def word():return struct.unpack_from('4096 or value>(len(data)-cursor)//minimum:raise ValueError('invalid count') + return value + marker(crc0('WoundMorphs'));take(count(44)*44) + marker(0x01ab6cee);marker(crc0('RandomAppearanceMorph')) + slots=[];total_models=total_textures=0 + for body in range(count(4)): + models=count(17);total_models+=models + if total_models>4096:raise ValueError('model total') + for alternative in range(models): + offset=cursor;model=word();word() + textures=count(12);total_textures+=textures + if total_textures>4096:raise ValueError('texture total') + word();take(textures*12+1) + slots.append(dict(body_part=body,alternative=alternative,offset=offset,id=model)) + take(27);take(count(13)*13);take(13) + marker(crc0('Expressions'));take(count(12)*12) + marker(0x01ab6d32);marker(crc0('Stats')) + # Stats remains opaque. The body contract ends at its validated marker; + # don't guess this unrelated block's length or interpret its scalar values. + return cursor,slots + + +def main(): + repo=Path(__file__).resolve().parents[1] + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--install',type=Path,required=True) + parser.add_argument('--payload-exe',type=Path,default=repo/'build/fableforge_defrelocate_tests.exe') + parser.add_argument('--schema',type=Path,default=repo/'docs/re_reference/def_schema.json') + parser.add_argument('--work-root',type=Path,default=repo/'build') + args=parser.parse_args();native=args.install.resolve() + assert not args.work_root.resolve().is_relative_to(native) + inputs=sorted((native/'data/CompiledDefs').glob('*.bin')) + def hashes():return {str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in inputs} + before=hashes();records=read_bank(native) + result=subprocess.run([str(args.payload_exe.resolve()),'--creature-body-audit',str(native),str(args.schema.resolve())], + cwd=repo,capture_output=True,text=True,timeout=120) + assert result.returncode==0,result.stderr + report=json.loads(result.stdout);rows=report['records'] + assert {r['index'] for r in rows}=={i for i,r in enumerate(records) if r['definition']=='CCreatureDef'} + accepted=refused=slots=nonempty=0 + for row in rows: + try:prefix,expected=body_slots(records[row['index']]['data']) + except ValueError: + assert 'refused' in row,row + refused+=1;continue + assert 'refused' not in row,row + data=records[row['index']]['data'];boundary=row['prefix_length'] + assert prefix<=boundary and data[boundary:boundary+4]==struct.pack(' Date: Sat, 10 Oct 2026 13:07:32 -0600 Subject: [PATCH 24/75] Audit Creature body mesh identities and stored types without copying assets --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 14 ++++++ tools/audit_creature_body_models.py | 52 +++++++++++++++++++++++ tools/check_all.py | 2 +- 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index fd94d739..59728692 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -614,3 +614,17 @@ diagnostic now explicitly stops structural interpretation at its marker. No Stats grammar, graphics-bank identity/type, skeleton or runtime compatibility claim. Targeted linker/relocation tests pass; registered the retail audit in check_all without repeating the broad suite. Test executable mode is read-only. + +Graphics identity follow-up passes in packet `creature-body-model-audit-faubfraq`. +The independent mmap directory walk preserves duplicate IDs instead of silently +collapsing them and reads no mesh payloads. All 1,745 body slots resolve uniquely +in MBANK_ALLMESHES, all with stored type 5. Graphics SHA256 checked before/after +by streaming; all compiled-bank hashes also unchanged. No large copies created. +Audit observations now include per-slot target name/type and missing/ambiguous +counts; check_all requests --graphics. These identities/types do not prove +skeleton compatibility or successful rendering. Existing selectModelById refuses +missing/ambiguous IDs before clearing preview, but a unique undecodable mesh can +still clear preview and fail decode; do not overstate that preservation contract. +Next: body-slot preview through this guarded selector, then typed binding with +independent exact four-byte delta and recovery checks. Custom skinned import is +still outstanding; no writer or package changed by this metadata audit. diff --git a/tools/audit_creature_body_models.py b/tools/audit_creature_body_models.py index a87ed483..26501519 100644 --- a/tools/audit_creature_body_models.py +++ b/tools/audit_creature_body_models.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 """Independent read-only body-part slot audit; not skeleton/runtime compatibility.""" import argparse +from collections import Counter, defaultdict import hashlib import json +import mmap from pathlib import Path import struct import subprocess @@ -10,6 +12,39 @@ from test_fmp_links import read_bank, crc0 +def mesh_directory(path): + """Read only BIG directory metadata; preserve duplicate IDs for diagnosis.""" + with path.open('rb') as stream, mmap.mmap(stream.fileno(),0,access=mmap.ACCESS_READ) as data: + def word(at):return struct.unpack_from('1 else 'unique' + counts[status]+=1 + if status=='unique':types[targets[0]['type']]+=1 + identities.append(dict(component=row['index'],**slot,status=status,targets=targets)) + assert file_hash(graphics)==graphics_hash,'graphics bank changed' + report['graphics']=dict(path=str(graphics),sha256=graphics_hash,counts=dict(counts), + stored_types=dict(types),slots=identities) + print('Graphics identity observations:',dict(counts),'stored types:',dict(types),flush=True) assert hashes()==before,'source banks changed' work=Path(tempfile.mkdtemp(prefix='creature-body-model-audit-',dir=args.work_root.resolve())) report.update(dict(passed=True,accepted=accepted,refused=refused,body_slots=slots,nonempty_components=nonempty, diff --git a/tools/check_all.py b/tools/check_all.py index 80798a34..665d5826 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -127,7 +127,7 @@ def main(): [sys.executable, "tools/audit_creature_visual_graph.py", "--install", install_root, "--payload-exe", "build/fableforge_defrelocate_tests.exe"], capture_output=True) ok &= run("Independent retail Creature body-part model slot audit (not runtime compatibility)", - [sys.executable, "tools/audit_creature_body_models.py", "--install", install_root], capture_output=True) + [sys.executable, "tools/audit_creature_body_models.py", "--install", install_root, "--graphics"], capture_output=True) ok &= run("Read-only Creature animation boundaries and numeric candidate observations", [sys.executable, "tools/audit_creature_animation.py", "--install", install_root, "--payload-exe", "build/fableforge_defrelocate_tests.exe"], capture_output=True) From 028817a207cfa49ca5d72ee5c84657f83bf7c0b5 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:09:14 -0600 Subject: [PATCH 25/75] Preview verified Creature body-part model alternatives through guarded selector --- docs/EDITOR.md | 2 + docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 14 +++++ gui/definitionbrowser.cpp | 22 ++++++++ tools/check_all.py | 2 + tools/test_creature_body_preview_ui.py | 67 +++++++++++++++++++++++ 5 files changed, 107 insertions(+) create mode 100644 tools/test_creature_body_preview_ui.py diff --git a/docs/EDITOR.md b/docs/EDITOR.md index bc400831..f908e6bf 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1454,6 +1454,8 @@ Game-bank CREATURE details offer **Open body definition** and **Open appearance Definition `CRGBColour` CLI input uses red, green, blue, alpha (0-255; omitted alpha defaults to 255). The writer converts to the native blue, green, red, alpha byte order. Existing unexpected lengths refuse. +Clean game-bank CCreatureDef details list verified body-part model alternatives as **Open body / alternative model** links. They open the individual mesh in Models by graphics-bank ID and remain disabled during definition drafts or asset file operations. Unsupported counted prefixes offer no guessed links. This is an individual mesh preview, not creature assembly, animation, rebinding or skeleton-compatibility validation. + Inline CCharString drafts accept up to 127 printable ASCII characters, including empty text. Non-ASCII narrow input is filtered; legacy byte strings remain read-only and exact. CWideString drafts accept valid Unicode up to 127 UTF-8 bytes, including supplementary characters and empty text. Existing Unicode values within that limit can be edited or saved unchanged. C0/C1 controls, malformed encoding, overlong originals, missing terminators and opaque trailers remain unsupported. The shared writer requires one complete NUL-terminated original string before replacement. CWideString uses UTF-16LE with a final zero code unit, without a count prefix. CLI and GUI edits use strict UTF-8 conversion. Odd lengths, malformed surrogates, missing or embedded terminators and opaque trailers refuse before writes. Unchanged supported drafts retain their exact original bytes. diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 59728692..0677ef40 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -628,3 +628,17 @@ still clear preview and fail decode; do not overstate that preservation contract Next: body-slot preview through this guarded selector, then typed binding with independent exact four-byte delta and recovery checks. Custom skinned import is still outstanding; no writer or package changed by this metadata audit. + +Body-slot preview controls now reuse validated bodyModels and selectModelById. +They require clean exact decode/encode, game-bank CCreatureDef and a supported +counted prefix; malformed prefixes have no guessed links. Labels/widgets preserve +body-part and alternative indices. Drafts and asset writes disable links. Successful +selection opens Models; missing/ambiguous IDs use the existing refusal path. +No model writer, body rebinding or private asset copying introduced. +GUI build and three focused linker/relocation/definitionfield CTests pass. +Both-size native UI acceptance passes in `creature-body-preview-ui-28eh7100`: +CCreatureDef 9116, body 0 alternatives 0/1, IDs 4534/4537 select named meshes +and render; four draft guards retain state. Compact render visually inspected. +All compiled-bank and graphics hashes unchanged. This is individual body-part +preview, not assembled/animated NPC or skeleton compatibility. Registered the +gate; broad suite not rerun and existing archives/public RC2 unchanged. diff --git a/gui/definitionbrowser.cpp b/gui/definitionbrowser.cpp index 90d77bd7..f63b0f6f 100644 --- a/gui/definitionbrowser.cpp +++ b/gui/definitionbrowser.cpp @@ -4,6 +4,7 @@ #include "definitionfield.hpp" #include "definitionclone.hpp" #include "definitionread.hpp" +#include "forge/fmplinks.hpp" #include "texturebrowse.hpp" #include "modpack.hpp" #include "../vendor/embedded_schema.hpp" @@ -582,6 +583,27 @@ void App::drawDefinitionViewport(const ImVec2&,const ImVec2&) { } if(!definitionDecodeError_.empty()) {ImGui::TextWrapped("%s",definitionDecodeError_.c_str());auto_.registerWidget("definition_decode_error");} ImGui::BeginChild("##definitiondetail",ImGui::GetContentRegionAvail()); + if(definitionBank_=="game.bin" && definitionDecoded_ && entry.definition=="CCreatureDef" && + definitionDecoded_->clean() && forge::defdecode::encode(*definitionDecoded_)==entry.data) { + try { + const auto layout=forge::fmplinks::creaturePrefixLayout(definitionDecoded_->prefix); + if(!layout.bodyModels.empty()) { + theme::hint("Body-part model alternatives (preview only; assets remain shared)."); + for(const auto& slot:layout.bodyModels) { + uint32_t id=0; + for(size_t n=0;n<4;++n)id|=uint32_t(entry.data.at(slot.offset+n))<<(8*n); + const auto key=std::to_string(slot.bodyPart)+"_"+std::to_string(slot.alternative); + const auto label="Open body "+std::to_string(slot.bodyPart)+" / alternative "+ + std::to_string(slot.alternative)+" model "+std::to_string(id); + ImGui::BeginDisabled(definitionEditing_ || modFilesBusy() || meshReplaceFuture_.valid()); + if(ImGui::SmallButton(label.c_str()) && selectModelById(id)) assetsTab_=1; + auto_.registerWidget(("definition_body_model_"+key).c_str());ImGui::EndDisabled(); + } + } + } catch(const std::exception&) { + // Unsupported counted prefixes stay inspectable, with no guessed links. + } + } if(definitionBank_=="game.bin" && definitionDecoded_ && entry.definition=="CREATURE") { for(const auto& [kind,label,widget]:std::array,2>{ std::tuple{"CCreatureDef","Open body definition","btn_definition_body"}, diff --git a/tools/check_all.py b/tools/check_all.py index 665d5826..01f355b5 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -128,6 +128,8 @@ def main(): "--payload-exe", "build/fableforge_defrelocate_tests.exe"], capture_output=True) ok &= run("Independent retail Creature body-part model slot audit (not runtime compatibility)", [sys.executable, "tools/audit_creature_body_models.py", "--install", install_root, "--graphics"], capture_output=True) + ok &= run("Read-only Creature body-slot mesh preview and draft guards at both sizes", + [sys.executable, "tools/test_creature_body_preview_ui.py", "--install", install_root], capture_output=True) ok &= run("Read-only Creature animation boundaries and numeric candidate observations", [sys.executable, "tools/audit_creature_animation.py", "--install", install_root, "--payload-exe", "build/fableforge_defrelocate_tests.exe"], capture_output=True) diff --git a/tools/test_creature_body_preview_ui.py b/tools/test_creature_body_preview_ui.py new file mode 100644 index 00000000..fd600789 --- /dev/null +++ b/tools/test_creature_body_preview_ui.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Read-only body model preview links and draft protection at both window sizes.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import tempfile +from audit_creature_body_models import body_slots, mesh_directory, file_hash +from test_definition_edit_ui import click +from test_fmp_links import read_bank + + +def main(): + repo=Path(__file__).resolve().parents[1] + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--install',type=Path,required=True) + parser.add_argument('--gui',type=Path,default=repo/'build/FableForge.exe') + parser.add_argument('--cwd',type=Path,default=repo) + parser.add_argument('--work-root',type=Path,default=repo/'build') + args=parser.parse_args();native=args.install.resolve() + assert not args.work_root.resolve().is_relative_to(native) + graphics=native/'data/graphics/graphics.big' + inputs=sorted((native/'data/CompiledDefs').glob('*.bin'))+[graphics] + def hashes():return {str(p):file_hash(p) for p in inputs} + before=hashes();records=read_bank(native);directory=mesh_directory(graphics) + candidates=[] + for index,record in enumerate(records): + if record['definition']!='CCreatureDef':continue + try:_,slots=body_slots(record['data']) + except ValueError:continue + for slot in slots: + targets=[entry for entry in directory if entry['id']==slot['id']] + if len(targets)==1 and targets[0]['type']==5: + candidates.append(dict(index=index,**slot,name=targets[0]['name'])) + chosen=[candidates[0],next(row for row in candidates if row['body_part']>0 or row['alternative']>0)] + # Component selectors prefer names over numeric strings. Guard this fixture + # choice so native record indices cannot be accidentally redirected. + for row in chosen:assert not any(r['name']==str(row['index']) for r in records) + work=Path(tempfile.mkdtemp(prefix='creature-body-preview-ui-',dir=args.work_root.resolve())) + print('Evidence retained at',work,flush=True) + for size,scale in [('1440x900',1),('800x600',1.5)]: + case=work/size;case.mkdir() + lines=['wait_maps','wait_ready',f'set uiscale {scale}','assets_tab 6','wait_definitions'] + for row in chosen: + widget=f'definition_body_model_{row["body_part"]}_{row["alternative"]}' + lines+=['assets_tab 6',f'definition_select {row["index"]}','frames 3','definition_clone', + 'assert_state definition_editing 1'] + lines+=click(widget)+['assert_state assets_tab 6',f'assert_state definition_selected {row["index"]}', + 'assert_state definition_editing 1','definition_cancel'] + lines+=click(widget)+['assert_state assets_tab 1','assert_state model_ready 1', + f'assert_state model_selected {row["name"]}'] + lines+=[f'screenshot {(case/"preview.png").as_posix()}','quit'] + script=case/'preview.txt';script.write_text('\n'.join(lines)+'\n',encoding='utf-8') + result=subprocess.run([str(args.gui.resolve()),'--install',str(native),'--size',size,'--auto',str(script)], + cwd=args.cwd.resolve(),env=dict(os.environ,FABLEFORGE_AUTOMATION_HIDDEN='1', + APPDATA=str(case/'profile'),LOCALAPPDATA=str(case/'local')), + capture_output=True,text=True,encoding='utf-8',errors='replace',timeout=120) + (case/'output.log').write_text(result.stdout+result.stderr,encoding='utf-8') + trace=Path(str(script)+'.log').read_text(encoding='utf-8') + assert result.returncode==0 and 'RESULT PASS' in trace and 'FAIL ' not in trace,trace[-6500:] + assert hashes()==before,'source bank changed' + print(size,'body-slot preview, explicit mesh identity and draft guards: PASS',flush=True) + (work/'result.json').write_text(json.dumps(dict(passed=True,chosen=chosen,source_hashes=before),indent=2)+'\n',encoding='utf-8') + + +if __name__=='__main__':main() From a8d8bae0765dc82fcbd43eab7b2ca3163a3502d2 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:10:58 -0600 Subject: [PATCH 26/75] Add exact in-memory Creature body model binding with typed mesh guards --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 21 +++++++++++ libs/forgecore/include/forge/defedit.hpp | 15 ++++++++ libs/forgecore/src/defedit.cpp | 37 +++++++++++++++++++ tests/test_fmplinks.cpp | 43 +++++++++++++++++++++++ 4 files changed, 116 insertions(+) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 0677ef40..f6f9b8b6 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -642,3 +642,24 @@ and render; four draft guards retain state. Compact render visually inspected. All compiled-bank and graphics hashes unchanged. This is individual body-part preview, not assembled/animated NPC or skeleton compatibility. Registered the gate; broad suite not rerun and existing archives/public RC2 unchanged. + +## In-memory body model rebinding + +defedit::setCreatureBodyModel now targets an explicit CCreatureDef entry index, +body-part index and alternative index using the validated canonical prefix. +The entire decode/encode must be clean/exact, and the complete prefix must parse +before changing anything. Both current and replacement IDs must resolve uniquely +to type-5 entries in exactly one MBANK_ALLMESHES bank. All checks and payload-copy +allocation precede mutation; only the selected four-byte word changes. Same ID +is a validated byte-exact no-op. Names, identities, other records and all mesh/ +texture/animation bytes remain unchanged. This is memory only, not publication. + +Targeted fmplinks and defrelocate CTests pass. Added exact intended-delta/no-op +checks plus twelve refusal cases for entry/class/body/alternative, missing ID, +wrong current/replacement types, duplicate ID/bank, absent bank, later malformed +marker and truncated component; every refusal verifies all records unchanged. +No runtime/skeleton compatibility claim follows from type checks. Publisher/CLI/ +GUI promotion and independent retail output/recovery acceptance remain required. +Reuse existing-entry guarded publication, but add a typed recomputation contract +and stale graphics identity checks rather than treating arbitrary changed payloads +as a verified body binding. Existing packages and RC2 remain unchanged. diff --git a/libs/forgecore/include/forge/defedit.hpp b/libs/forgecore/include/forge/defedit.hpp index d4486c64..973d83ee 100644 --- a/libs/forgecore/include/forge/defedit.hpp +++ b/libs/forgecore/include/forge/defedit.hpp @@ -32,6 +32,7 @@ #include #include "forge/bin.hpp" +#include "forge/big.hpp" #include "forge/defschema.hpp" namespace forge::defedit { @@ -48,6 +49,20 @@ struct SetFieldResult { size_t newPayloadSize = 0; // payload byte length after the edit }; +struct CreatureBodyModelResult { + size_t entryIndex, bodyPart, alternative, offset; + uint32_t oldModelId, newModelId; +}; +// Change exactly one RandomAppearanceMorph body-model word in memory. Requires +// a clean exact CCreatureDef decode, complete supported prefix and unique type-5 +// current/replacement IDs in one MBANK_ALLMESHES bank. No mesh payload, texture, +// names or other definition bytes change. Invalid requests refuse before mutation. +// Identity/type checks do NOT establish skeleton or runtime compatibility. +// Callers must publish through the guarded existing-entry transaction. +CreatureBodyModelResult setCreatureBodyModel(bin::File& file, + const defschema::Schema& schema,size_t entryIndex,size_t bodyPart,size_t alternative, + uint32_t modelId,const big::File& graphics); + // Edit one field of the entry named `entryName` (empty name never matches; // use the index overload for unnamed sub-defs). Locates the field by CRC tag, // parses `valueString` per the field's declared type, rewrites just that value, diff --git a/libs/forgecore/src/defedit.cpp b/libs/forgecore/src/defedit.cpp index 0e9e38bc..7d36edef 100644 --- a/libs/forgecore/src/defedit.cpp +++ b/libs/forgecore/src/defedit.cpp @@ -1,5 +1,6 @@ #include "forge/defedit.hpp" #include "forge/defrelocate.hpp" +#include "forge/fmplinks.hpp" #include "forge/unicode.hpp" #include @@ -19,6 +20,42 @@ #include "forge/defdecode.hpp" namespace forge::defedit { +CreatureBodyModelResult setCreatureBodyModel(bin::File& file,const defschema::Schema& schema, + size_t entryIndex,size_t bodyPart,size_t alternative,uint32_t modelId,const big::File& graphics) { + const auto refuse=[](const char* text){throw std::runtime_error(std::string("defedit body model: ")+text);}; + if(entryIndex>=file.entries().size())refuse("entry index out of range"); + const auto& entry=file.entries()[entryIndex]; + if(entry.definition!="CCreatureDef")refuse("requires CCreatureDef"); + const auto decoded=defdecode::decode(entry,schema); + if(!decoded.clean() || defdecode::encode(decoded)!=entry.data)refuse("requires clean exact decode"); + const auto layout=fmplinks::creaturePrefixLayout(decoded.prefix); + const auto slot=std::find_if(layout.bodyModels.begin(),layout.bodyModels.end(),[&](const auto& value){ + return value.bodyPart==bodyPart && value.alternative==alternative; + }); + if(slot==layout.bodyModels.end())refuse("body part or alternative is absent"); + uint32_t old=0; + for(size_t n=0;n<4;++n)old|=uint32_t(entry.data.at(slot->offset+n))<<(8*n); + const big::Bank* meshes=nullptr; + for(const auto& bank:graphics.banks())if(bank.name=="MBANK_ALLMESHES") { + if(meshes)refuse("ambiguous mesh bank");meshes=&bank; + } + if(!meshes)refuse("missing mesh bank"); + for(const auto id:{old,modelId}) { + const big::Entry* target=nullptr; + for(const auto& mesh:meshes->entries)if(mesh.id==id) { + if(target)refuse("ambiguous mesh ID");target=&mesh; + } + if(!target)refuse("missing mesh ID"); + if(target->type!=5)refuse("requires skinned type-5 mesh"); + } + const CreatureBodyModelResult result{entryIndex,bodyPart,alternative,slot->offset,old,modelId}; + if(old!=modelId) { + auto data=entry.data; + for(size_t n=0;n<4;++n)data[slot->offset+n]=uint8_t(modelId>>(8*n)); + file.setEntryData(entryIndex,std::move(data)); + } + return result; +} namespace { // Split "a,b,c" into pieces (no trimming beyond surrounding spaces per piece). diff --git a/tests/test_fmplinks.cpp b/tests/test_fmplinks.cpp index 80b48f12..2ca385da 100644 --- a/tests/test_fmplinks.cpp +++ b/tests/test_fmplinks.cpp @@ -1,6 +1,7 @@ #include "forge/fmplinks.hpp" #include "forge/defrelocate.hpp" #include "forge/defdecode.hpp" +#include "forge/defedit.hpp" #include #include #include @@ -253,6 +254,48 @@ int main() { check(shiftedLayout.bodyModels.size()==1 && shiftedLayout.bodyModels[0].bodyPart==1 && shiftedLayout.bodyModels[0].offset==meshAt+4, "empty leading body part is not renumbered in body model slots"); + { + forge::bin::File bank; + bank.addEntry("CCreatureDef","BODY",prefix); + bank.addEntry("OTHER","UNCHANGED",{9,0,0,0}); + forge::big::File graphics; + auto& meshes=graphics.addBank("MBANK_ALLMESHES",0); + for(const auto id:{9u,99u}) { + forge::big::Entry mesh;mesh.id=id;mesh.type=5;mesh.name=std::to_string(id); + meshes.entries.push_back(std::move(mesh)); + } + auto changed=bank; + const auto result=forge::defedit::setCreatureBodyModel(changed,prefixSchema,0,0,0,99,graphics); + auto expectedBody=prefix; + for(size_t n=0;n<4;++n)expectedBody[meshAt+n]=uint8_t(99u>>(8*n)); + check(result.offset==meshAt && result.oldModelId==9 && result.newModelId==99 && + changed.entries()[0].data==expectedBody && changed.entries()[1].data==bank.entries()[1].data && + changed.originalNames()==bank.originalNames(),"body binding changes only one verified model word"); + forge::defedit::setCreatureBodyModel(changed,prefixSchema,0,0,0,99,graphics); + check(changed.entries()[0].data==expectedBody,"same body model is an exact no-op"); + const auto refused=[&](forge::bin::File input,size_t entry,size_t body,size_t alternative, + uint32_t id,const forge::big::File& assets) { + const auto original=input.entries(); + rejects([&]{forge::defedit::setCreatureBodyModel(input,prefixSchema,entry,body,alternative,id,assets);}, + "invalid body binding refuses"); + bool exact=input.entries().size()==original.size(); + for(size_t n=0;n> (8*n)); auto relocatedExpected = relocatable.data; From 2b96ea3699323772893cf28321cf4cb135dbed1a Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:14:35 -0600 Subject: [PATCH 27/75] Publish typed Creature body bindings with late source and graphics checks --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 20 +++++++ src/definitionbody.hpp | 70 +++++++++++++++++++++++ src/definitionwrite.hpp | 5 +- tests/test_creaturevisualclone.cpp | 63 +++++++++++++++++++- 4 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 src/definitionbody.hpp diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index f6f9b8b6..7531ff75 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -663,3 +663,23 @@ GUI promotion and independent retail output/recovery acceptance remain required. Reuse existing-entry guarded publication, but add a typed recomputation contract and stale graphics identity checks rather than treating arbitrary changed payloads as a verified body binding. Existing packages and RC2 remain unchanged. + +Typed body publication is implemented in src/definitionbody.hpp. Its contract +captures schema, graphics directory snapshot, exact loaded source bank/name bytes +and requested entry/body/alternative/model ID. It reopens source and recomputes +the canonical edit, compares every proposed record/name, and requires existing +outputs to match the loaded pair. Raw bank/name bytes and graphics directory +metadata are rechecked immediately before PendingBanks installation through a +new optional callback in the existing-entry publisher. Hardlink/cross-bank/asset +aliases, offline guards and creation-record protections remain enforced. +An existing destination graphics directory must also match; absent graphics uses +the loaded source fallback. This is directory identity validation, not a full +asset-payload hash or runtime/skeleton compatibility guarantee. + +Synthetic publisher checks pass fresh and in-place output, exact name-pool +preservation, tampered-plan/source-byte/graphics-type refusals, late callback +refusal before installation, fresh-file Restore removal and exact in-place +original Restore. Three focused Creature/linker/relocation CTests pass. No native +input writes, CLI/GUI promotion, independent retail published delta, complete +race/rollback matrix or broad-suite acceptance yet. Next: broaden publisher +negatives and independently verify retail publication/Restore, then CLI/UI. diff --git a/src/definitionbody.hpp b/src/definitionbody.hpp new file mode 100644 index 00000000..2498683d --- /dev/null +++ b/src/definitionbody.hpp @@ -0,0 +1,70 @@ +#pragma once +#include "definitionclone.hpp" +namespace albion::definitionwrite { +struct CreatureBodyContract { + const forge::defschema::Schema& schema; + const forge::big::File& graphics; + const std::vector& sourceBankBytes; + const std::vector& sourceNamesBytes; + size_t entryIndex,bodyPart,alternative; + uint32_t modelId; +}; +// Binding uses graphics directory identities, not mesh payload equivalence. +inline bool sameBodyGraphics(const forge::big::File& a,const forge::big::File& b) { + if(a.banks().size()!=b.banks().size())return false; + for(size_t i=0;i #include #include +#include #include #include #include "forge/bin.hpp" @@ -32,7 +33,8 @@ inline void verifyEntries(const forge::bin::File& expected,const forge::bin::Fil // Existing-entry edits add no names. Retain the original pool bytes and publish // only a missing destination pool; an existing pool must decode every identity. inline void publish(const forge::bin::File& bank,const std::filesystem::path& sourceNames, - const std::filesystem::path& destination,bool trackRecovery=false) { + const std::filesystem::path& destination,bool trackRecovery=false, + const std::function& validateBeforeInstall={}) { namespace fs=std::filesystem; const auto target=fs::absolute(destination).lexically_normal(); const auto names=target.parent_path()/"names.bin"; @@ -71,6 +73,7 @@ inline void publish(const forge::bin::File& bank,const std::filesystem::path& so if(newBank && fs::exists(target)) throw std::runtime_error("definition write: bank destination appeared during preparation"); if(newNames && fs::exists(names)) throw std::runtime_error("definition write: name pool destination appeared during preparation"); if(!newNames) verifyEntries(bank,forge::bin::File::open(names,preparedBank)); + if(validateBeforeInstall)validateBeforeInstall(); std::string error;if(!pending.install(trackRecovery,error)) throw std::runtime_error("definition write: "+error); } } // namespace albion::definitionwrite diff --git a/tests/test_creaturevisualclone.cpp b/tests/test_creaturevisualclone.cpp index 8b0613b2..07efbb2c 100644 --- a/tests/test_creaturevisualclone.cpp +++ b/tests/test_creaturevisualclone.cpp @@ -2,6 +2,7 @@ #include "forge/defdecode.hpp" #include "forge/temporarydirectory.hpp" #include "definitionclone.hpp" +#include "definitionbody.hpp" #include #include #include @@ -188,5 +189,65 @@ int main(int argc,char** argv)try{ require(read(fs::path(inPlace.string()+albion::backups::kOrigSuffix))==oldBank&& read(fs::path(inPlaceNames.string()+albion::backups::kOrigSuffix))==oldNames,"private in-place pair lacks exact originals"); require(read(namesPath)==oldNames&&read(bankPath)==oldBank,"private test changed original source pair"); - std::cout<<"Private Creature graph: selected ownership, shared gameplay, atomic refusal and paired roundtrip PASS\n";return 0; + { + const auto bindingSchema=defschema::Schema::loadText(R"({"CCreatureDef":{"fields":[{"name":"InitialActions","type":"uint32"}]}})"); + Bytes payload{1,0,1};field(payload,"WoundMorphs",0);word(payload,0x01ab6cee); + field(payload,"RandomAppearanceMorph",1);word(payload,1); + const size_t modelAt=payload.size();word(payload,10);word(payload,0);word(payload,0);word(payload,0);payload.push_back(0); + payload.insert(payload.end(),27,0);word(payload,0);payload.insert(payload.end(),13,0); + field(payload,"Expressions",0);word(payload,0x01ab6d32);field(payload,"Stats",0);field(payload,"InitialActions",0); + auto baseline=source;baseline.setEntryData(3,payload); + const auto folder=temp.path()/"body-source";fs::create_directories(folder); + const auto bodyNames=folder/"names.bin",bodyBank=folder/"game.bin",graphicsPath=folder/"graphics.big"; + baseline.save(bodyNames,bodyBank); + big::File assets;auto& meshBank=assets.addBank("MBANK_ALLMESHES",0); + for(const auto id:{10u,20u}) {big::Entry mesh;mesh.id=id;mesh.type=5;mesh.name=std::to_string(id);mesh.data={1};meshBank.entries.push_back(mesh);} + write(graphicsPath,assets.serialize());const auto graphics=big::File::open(graphicsPath); + const auto bankBytes=read(bodyBank),nameBytes=read(bodyNames); + baseline=bin::File::open(bodyNames,bodyBank);auto plan=baseline; + defedit::setCreatureBodyModel(plan,bindingSchema,3,0,0,20,graphics); + const albion::definitionwrite::CreatureBodyContract proof{bindingSchema,graphics,bankBytes,nameBytes,3,0,0,20}; + const auto fresh=temp.path()/"body-fresh"/"game.bin"; + auto bad=plan;auto badPayload=bad.entries()[3].data;badPayload.back()^=1;bad.setEntryData(3,badPayload); + bool denied=false;try{albion::definitionwrite::publishCreatureBody(bad,bodyNames,bodyBank,graphicsPath,fresh,proof);} + catch(const std::runtime_error&){denied=true;} + require(denied&&!fs::exists(fresh.parent_path()),"tampered body plan created output"); + albion::definitionwrite::publishCreatureBody(plan,bodyNames,bodyBank,graphicsPath,fresh,proof); + albion::definitionwrite::verifyEntries(plan,bin::File::open(fresh.parent_path()/"names.bin",fresh)); + auto expected=payload;put(expected,modelAt,20);require(plan.entries()[3].data==expected,"body publication delta differs"); + require(read(fresh.parent_path()/"names.bin")==nameBytes,"body publication normalized original names"); + require(fs::exists(fs::path(fresh.string()+albion::backups::kCreatedSuffix)),"body fresh publication lacks recovery marker"); + for(const auto& path:{fresh,fresh.parent_path()/"names.bin"}) { + albion::backups::Entry recovery;recovery.file=path; + recovery.backup=fs::path(path.string()+albion::backups::kCreatedSuffix); + recovery.kind=albion::backups::Kind::Created;recovery.created=true; + std::string error;require(albion::backups::restore(recovery,false,error),"body created restore failed"); + require(!fs::exists(path)&&!fs::exists(recovery.backup),"body created restore left file or marker"); + } + const auto late=temp.path()/"late-body"/"game.bin";bool callbackCalled=false; + denied=false;try { + albion::definitionwrite::publish(plan,bodyNames,late,true,[&] { + callbackCalled=true;require(!fs::exists(late),"callback ran after installation"); + throw std::runtime_error("late fixture refusal"); + }); + }catch(const std::runtime_error&){denied=true;} + require(denied&&callbackCalled&&!fs::exists(late)&&!fs::exists(late.parent_path()/"names.bin"), + "late publication refusal installed outputs"); + auto stale=bankBytes;stale[0]^=1;write(bodyBank,stale); + denied=false;try{albion::definitionwrite::publishCreatureBody(plan,bodyNames,bodyBank,graphicsPath,temp.path()/"stale-body"/"game.bin",proof);} + catch(const std::runtime_error&){denied=true;} + require(denied,"stale body bank accepted");write(bodyBank,bankBytes); + auto staleGraphics=assets;staleGraphics.banks()[0].entries[1].type=1;write(graphicsPath,staleGraphics.serialize()); + denied=false;try{albion::definitionwrite::publishCreatureBody(plan,bodyNames,bodyBank,graphicsPath,temp.path()/"stale-graphics"/"game.bin",proof);} + catch(const std::runtime_error&){denied=true;} + require(denied,"stale graphics directory accepted");write(graphicsPath,assets.serialize()); + albion::definitionwrite::publishCreatureBody(plan,bodyNames,bodyBank,graphicsPath,bodyBank,proof); + require(read(fs::path(bodyBank.string()+albion::backups::kOrigSuffix))==bankBytes,"body in-place original differs"); + require(read(bodyNames)==nameBytes,"body in-place namespace changed"); + albion::backups::Entry original;original.file=bodyBank; + original.backup=fs::path(bodyBank.string()+albion::backups::kOrigSuffix); + std::string restoreError;require(albion::backups::restore(original,false,restoreError),"body original restore failed"); + require(read(bodyBank)==bankBytes&&read(bodyNames)==nameBytes,"body restore did not recover exact source pair"); + } + std::cout<<"Private Creature graph and typed body publication PASS\n";return 0; }catch(const std::exception& error){std::cerr< Date: Sat, 10 Oct 2026 13:17:05 -0600 Subject: [PATCH 28/75] Verify retail body publication exact deltas and Restore with refusal gates --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 18 ++++++ tests/test_creaturevisualclone.cpp | 47 +++++++++++++++ tools/check_all.py | 2 + tools/test_creature_body_publication.py | 72 +++++++++++++++++++++++ 4 files changed, 139 insertions(+) create mode 100644 tools/test_creature_body_publication.py diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 7531ff75..ce86b23d 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -683,3 +683,21 @@ original Restore. Three focused Creature/linker/relocation CTests pass. No nativ input writes, CLI/GUI promotion, independent retail published delta, complete race/rollback matrix or broad-suite acceptance yet. Next: broaden publisher negatives and independently verify retail publication/Restore, then CLI/UI. + +Retail publisher test mode --body-publish writes only a fresh output outside the +source install and invokes the typed contract, not raw serialization. Independent +Python gate passes in `creature-body-publication-3s59jcbe`: native components +9116/9118 select another existing alternative in their same body part. Both outputs +match all 14,764 records except the intended four-byte model word, preserve raw +header/identity tables and names.bin exactly, and contain both creation markers. +Explicit forge restore removes both created banks and markers. Compiled and +graphics hashes remain unchanged; no asset banks copied. This is offline binding +and recovery, not runtime or skeleton compatibility. Gate registered in check_all. + +Six additional synthetic publisher negatives pass: wrong model contract, absent +alternative, unauthorized extra record, stale name-pool bytes, hardlinked source +output and incompatible destination graphics directory. Refusals retain output +existence and prior bytes. Initial compile caught a test-only assignment to a +reference-bearing contract; use a separately constructed proof copy instead. +Three affected CTests pass again. Complete race/locked-file rollback matrix and +CLI/UI promotion still remain; no broad suite or new package claim. diff --git a/tests/test_creaturevisualclone.cpp b/tests/test_creaturevisualclone.cpp index 07efbb2c..9f294c01 100644 --- a/tests/test_creaturevisualclone.cpp +++ b/tests/test_creaturevisualclone.cpp @@ -1,6 +1,7 @@ #include "forge/defedit.hpp" #include "forge/defdecode.hpp" #include "forge/temporarydirectory.hpp" +#include "forge/fmplinks.hpp" #include "definitionclone.hpp" #include "definitionbody.hpp" #include @@ -69,6 +70,26 @@ static int retail(char** argv){ } int main(int argc,char** argv)try{ using namespace forge; + if(argc==9 && std::string_view(argv[1])=="--body-publish") { + namespace fs=std::filesystem; + const auto native=fs::weakly_canonical(fs::path(argv[2])); + const auto output=fs::weakly_canonical(fs::path(argv[4])); + const auto relative=output.lexically_relative(native); + require(!fs::exists(output)&&!relative.empty()&&*relative.begin()=="..","body output must be fresh and outside source install"); + const auto folder=native/"data"/"CompiledDefs"; + const auto names=folder/"names.bin",bank=folder/"game.bin",graphicsPath=native/"data"/"graphics"/"graphics.big"; + const auto schema=defschema::Schema::load(argv[3]); + const auto graphics=big::File::open(graphicsPath);auto plan=bin::File::open(names,bank); + const auto bankBytes=read(bank),nameBytes=read(names); + const size_t entry=std::stoull(argv[5]),bodyPart=std::stoull(argv[6]),alternative=std::stoull(argv[7]); + const auto model=std::stoull(argv[8]);require(model<=UINT32_MAX,"model ID overflow"); + const auto result=defedit::setCreatureBodyModel(plan,schema,entry,bodyPart,alternative,uint32_t(model),graphics); + const albion::definitionwrite::CreatureBodyContract proof{schema,graphics,bankBytes,nameBytes,entry,bodyPart,alternative,uint32_t(model)}; + albion::definitionwrite::publishCreatureBody(plan,names,bank,graphicsPath,output/"data"/"CompiledDefs"/"game.bin",proof); + require(read(bank)==bankBytes&&read(names)==nameBytes,"body test changed native source banks"); + std::cout< Date: Sat, 10 Oct 2026 13:19:16 -0600 Subject: [PATCH 29/75] Exercise Windows body publication rollback after partial installation --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 12 +++++++++ tests/test_creaturevisualclone.cpp | 33 +++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index ce86b23d..3848d545 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -701,3 +701,15 @@ existence and prior bytes. Initial compile caught a test-only assignment to a reference-bearing contract; use a separately constructed proof copy instead. Three affected CTests pass again. Complete race/locked-file rollback matrix and CLI/UI promotion still remain; no broad suite or new package claim. + +Windows partial-install rollback checks now use tiny owned prepared files held +without FILE_SHARE_DELETE: names.bin forces refusal after bank installation; +game.bin.forge-created forces refusal after both banks install. Both routes +verify the exception identifies the intended rename, no bank/name/creation marker +remains installed, and source bank/name bytes remain exact. This tests the shared +existing-entry transaction used by body publication, not a forced race hook in +the typed wrapper. Conditional Windows-only coverage exercised locally; other +platforms do not run these lock cases. No all-races/full-matrix claim. +The locking handle is released after observing refusal. Temporary fixture cleanup +remains confined to its exclusively created test directory. Next: explicit-index +body-binding CLI with required schema/graphics/output paths, then GUI drafts. diff --git a/tests/test_creaturevisualclone.cpp b/tests/test_creaturevisualclone.cpp index 9f294c01..492661cf 100644 --- a/tests/test_creaturevisualclone.cpp +++ b/tests/test_creaturevisualclone.cpp @@ -9,6 +9,12 @@ #include #include #include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif using Bytes=std::vector; static void require(bool ok,const char* why){if(!ok)throw std::runtime_error(why);} @@ -280,6 +286,33 @@ int main(int argc,char** argv)try{ }catch(const std::runtime_error&){denied=true;} require(denied&&callbackCalled&&!fs::exists(late)&&!fs::exists(late.parent_path()/"names.bin"), "late publication refusal installed outputs"); +#ifdef _WIN32 + // Hold a prepared file without FILE_SHARE_DELETE. Installation has + // already renamed earlier banks when this Windows rename must fail. + for(const auto& lockedName:{std::string("names.bin"),std::string("game.bin.forge-created")}) { + const auto rollback=temp.path()/("body-rollback-"+lockedName)/"game.bin"; + HANDLE held=INVALID_HANDLE_VALUE;bool lockReached=false;denied=false;std::string failure; + try { + albion::definitionwrite::publish(plan,bodyNames,rollback,true,[&] { + for(const auto& child:fs::directory_iterator(rollback.parent_path())) { + if(!child.is_directory() || !child.path().filename().string().starts_with("forge-def-prepare-"))continue; + const auto prepared=child.path()/"new"/lockedName; + held=CreateFileW(prepared.c_str(),GENERIC_READ,FILE_SHARE_READ,nullptr,OPEN_EXISTING,FILE_ATTRIBUTE_NORMAL,nullptr); + require(held!=INVALID_HANDLE_VALUE,"could not lock prepared rollback fixture"); + lockReached=true;break; + } + require(lockReached,"prepared rollback fixture missing"); + }); + }catch(const std::runtime_error& error){denied=true;failure=error.what();} + if(held!=INVALID_HANDLE_VALUE)CloseHandle(held); + require(denied&&lockReached,"locked later publication file did not refuse"); + require(failure.find("rename")!=std::string::npos && failure.find(lockedName)!=std::string::npos, + "rollback fixture did not fail at the intended later rename"); + for(const auto& name:{"game.bin","names.bin","game.bin.forge-created","names.bin.forge-created"}) + require(!fs::exists(rollback.parent_path()/name),"partial body publication rollback left installed file"); + require(read(bodyBank)==bankBytes&&read(bodyNames)==nameBytes,"body rollback altered source banks"); + } +#endif auto stale=bankBytes;stale[0]^=1;write(bodyBank,stale); denied=false;try{albion::definitionwrite::publishCreatureBody(plan,bodyNames,bodyBank,graphicsPath,temp.path()/"stale-body"/"game.bin",proof);} catch(const std::runtime_error&){denied=true;} From 22b621479c8a3bf0866456797becc2f564579890 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:22:15 -0600 Subject: [PATCH 30/75] Expose guarded explicit-index Creature body binding CLI and retail gates --- docs/EDITOR.md | 2 + docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 18 ++++++++ tools/check_all.py | 3 ++ tools/forge-cli/main.cpp | 50 +++++++++++++++++++++++ tools/test_creature_body_publication.py | 34 +++++++++++++-- 5 files changed, 103 insertions(+), 4 deletions(-) diff --git a/docs/EDITOR.md b/docs/EDITOR.md index f908e6bf..24b2cd77 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1456,6 +1456,8 @@ Definition `CRGBColour` CLI input uses red, green, blue, alpha (0-255; omitted a Clean game-bank CCreatureDef details list verified body-part model alternatives as **Open body / alternative model** links. They open the individual mesh in Models by graphics-bank ID and remain disabled during definition drafts or asset file operations. Unsupported counted prefixes offer no guessed links. This is an individual mesh preview, not creature assembly, animation, rebinding or skeleton-compatibility validation. +`forge-tools defs bind-body-model --schema --graphics --out [--names ] [--json]` changes one verified body-model ID in a CCreatureDef. Indices are explicit unsigned decimal values, not name selectors. Both current and replacement models must be unique type-5 graphics entries. The publisher recomputes the edit, refuses stale source banks or graphics-directory metadata, preserves names.bin bytes, and retains recovery records. Component users and assets stay shared; use private cloned components and owned outputs when appropriate. This does not validate skeleton/runtime compatibility or import custom meshes. + Inline CCharString drafts accept up to 127 printable ASCII characters, including empty text. Non-ASCII narrow input is filtered; legacy byte strings remain read-only and exact. CWideString drafts accept valid Unicode up to 127 UTF-8 bytes, including supplementary characters and empty text. Existing Unicode values within that limit can be edited or saved unchanged. C0/C1 controls, malformed encoding, overlong originals, missing terminators and opaque trailers remain unsupported. The shared writer requires one complete NUL-terminated original string before replacement. CWideString uses UTF-16LE with a final zero code unit, without a count prefix. CLI and GUI edits use strict UTF-8 conversion. Odd lengths, malformed surrogates, missing or embedded terminators and opaque trailers refuse before writes. Unchanged supported drafts retain their exact original bytes. diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 3848d545..099b9958 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -713,3 +713,21 @@ platforms do not run these lock cases. No all-races/full-matrix claim. The locking handle is released after observing refusal. Temporary fixture cleanup remains confined to its exclusively created test directory. Next: explicit-index body-binding CLI with required schema/graphics/output paths, then GUI drafts. + +Body-binding CLI implemented: forge-tools defs bind-body-model + --schema +--graphics --out [--names ] [--json]. +Selectors are explicit complete unsigned decimal indices, not name lookups; +overflow, duplicate/unknown options and missing required paths refuse. The CLI +captures raw sources, uses the canonical setter and typed publisher, and reports +old/new IDs plus runtime_compatibility_verified=false. Assets remain shared. +GUI draft integration is still pending. Existing packages/RC2 unchanged. + +New executable builds. CLI retail gate passes in packet +`creature-body-publication-749nycre`: two native four-byte publication/Restore +cases and fifteen selector/option/range/class refusals. Native compiled/graphics +hashes unchanged; no asset copies. Registered source gate and documented syntax +in EDITOR.md/tool help (generated CLI.md describes forge.exe, not forge-tools). +Neighboring private-clone CLI regression passes all 28 cases in packet +`creature-private-cli-56lfcfgv` using the same rebuilt executable. No broad-suite +claim; next is GUI body-binding drafts and save/cancel/stale-source coverage. diff --git a/tools/check_all.py b/tools/check_all.py index 6ddf5230..42dd2050 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -132,6 +132,9 @@ def main(): [sys.executable, "tools/test_creature_body_preview_ui.py", "--install", install_root], capture_output=True) ok &= run("Independent retail body-binding publication, exact deltas and Restore", [sys.executable, "tools/test_creature_body_publication.py", "--install", install_root], capture_output=True) + ok &= run("Body-binding CLI retail publication, Restore and argument refusals", + [sys.executable, "tools/test_creature_body_publication.py", "--install", install_root, + "--tools", "build/forge-tools.exe"], capture_output=True) ok &= run("Read-only Creature animation boundaries and numeric candidate observations", [sys.executable, "tools/audit_creature_animation.py", "--install", install_root, "--payload-exe", "build/fableforge_defrelocate_tests.exe"], capture_output=True) diff --git a/tools/forge-cli/main.cpp b/tools/forge-cli/main.cpp index e30a5bbd..65209a79 100644 --- a/tools/forge-cli/main.cpp +++ b/tools/forge-cli/main.cpp @@ -37,6 +37,7 @@ #include "textwrite.hpp" #include "definitionwrite.hpp" #include "definitionclone.hpp" +#include "definitionbody.hpp" #include "textdisplay.hpp" #include "temporarydirectory.hpp" #include "forge/bin.hpp" @@ -261,6 +262,7 @@ int usage() { " forge defs clone --out [--names ] [--json]\n" " forge defs clone-creature --schema --out [--names ] [--json]\n" " forge defs clone-creature-private --schema --out [--names ] [--json]\n" + " forge defs bind-body-model --schema --graphics --out [--names ] [--json]\n" " forge defs set-field [--names ] \n" " forge title add --donor \n" " --gui --desc --greet --comment --self \n" @@ -7605,6 +7607,37 @@ int questCard(const std::string& gameRoot, const std::string& schemaPath, // parses into the field's typed bytes, rewrites only that value, and // writes the modified bin (+ names.bin) to --out or a ".set-field.bin" // sibling. Untouched entries stay byte-identical (semantic re-serialize). +int defsBindBody(const std::vector& positional,const std::string& namesPath, + const std::string& schemaPath,const std::string& graphicsPath, + const std::string& outputPath,bool asJson) { + namespace fs=std::filesystem; + const auto number=[](const std::string& text) { + uint64_t value=0;const auto parsed=std::from_chars(text.data(),text.data()+text.size(),value); + if(parsed.ec!=std::errc{} || parsed.ptr!=text.data()+text.size()) + throw std::runtime_error("body binding: indices and model ID must be complete unsigned decimal values"); + return value; + }; + const auto entry=number(positional[1]),body=number(positional[2]),alternative=number(positional[3]),model=number(positional[4]); + if(entry>SIZE_MAX || body>SIZE_MAX || alternative>SIZE_MAX || model>UINT32_MAX) + throw std::runtime_error("body binding: index or model ID overflow"); + const auto input=fs::u8path(positional[0]),names=namesPath.empty()?input.parent_path()/"names.bin":fs::u8path(namesPath); + const auto output=fs::u8path(outputPath),graphicsFile=fs::u8path(graphicsPath); + const auto bankBytes=albion::definitionwrite::cloneBytes(input),nameBytes=albion::definitionwrite::cloneBytes(names); + const auto schema=forge::defschema::Schema::load(fs::u8path(schemaPath)); + const auto graphics=forge::big::File::open(graphicsFile);auto plan=forge::bin::File::open(names,input); + const auto result=forge::defedit::setCreatureBodyModel(plan,schema,size_t(entry),size_t(body),size_t(alternative),uint32_t(model),graphics); + const albion::definitionwrite::CreatureBodyContract contract{schema,graphics,bankBytes,nameBytes, + size_t(entry),size_t(body),size_t(alternative),uint32_t(model)}; + albion::definitionwrite::publishCreatureBody(plan,names,input,graphicsFile,output,contract); + const json report{{"entry_index",entry},{"body_part",body},{"alternative",alternative}, + {"offset",result.offset},{"old_model_id",result.oldModelId},{"new_model_id",result.newModelId}, + {"bin",output.string()},{"assets_shared",true},{"runtime_compatibility_verified",false}}; + if(asJson)std::puts(report.dump(2).c_str()); + else std::printf("Bound body part %zu alternative %zu in entry %zu to model %u (assets shared; runtime compatibility unverified)\n", + size_t(body),size_t(alternative),size_t(entry),uint32_t(model)); + return 0; +} + int defsClone(const std::string& binPath,const std::string& namesPath,const std::string& donor, const std::string& newName,const std::string& outPath,bool asJson,const std::string& creatureSchemaPath={}, const std::optional>& visualNames={}) { @@ -14336,6 +14369,23 @@ int main(int argc, char** argv) { return uiSetCardModel(gameRoot, schemaPath, entryName, (uint32_t)meshId, outRoot, inPlace); } + if(args.size()>=2 && args[0]=="defs" && args[1]=="bind-body-model") { + std::string names,schema,graphics,out;bool asJson=false;std::vector positional; + for(size_t i=2;i=args.size() || args[i+1].starts_with("--"))throw std::runtime_error("body binding: option requires a path"); + auto& value=arg=="--names"?names:arg=="--schema"?schema:arg=="--graphics"?graphics:out; + if(!value.empty())throw std::runtime_error("body binding: duplicate option "+arg); + value=args[++i]; + }else if(arg=="--json")asJson=true; + else if(arg.starts_with("--"))throw std::runtime_error("body binding: unknown option "+arg); + else positional.push_back(arg); + } + if(positional.size()!=5 || schema.empty() || graphics.empty() || out.empty()) + throw std::runtime_error("body binding: expected --schema --graphics --out "); + return defsBindBody(positional,names,schema,graphics,out,asJson); + } if(args.size()>=2 && args[0]=="defs" && (args[1]=="clone" || args[1]=="clone-creature" || args[1]=="clone-creature-private")) { const bool privateVisual=args[1]=="clone-creature-private"; const bool creature=args[1]!="clone"; diff --git a/tools/test_creature_body_publication.py b/tools/test_creature_body_publication.py index 1bcf56ce..0c8b34ae 100644 --- a/tools/test_creature_body_publication.py +++ b/tools/test_creature_body_publication.py @@ -15,6 +15,7 @@ def main(): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--install',type=Path,required=True) parser.add_argument('--publisher',type=Path,default=repo/'build/fableforge_creaturevisualclone_tests.exe') + parser.add_argument('--tools',type=Path,help='Exercise the public forge-tools CLI instead of the test publisher') parser.add_argument('--forge',type=Path,default=repo/'build/forge.exe') parser.add_argument('--schema',type=Path,default=repo/'docs/re_reference/def_schema.json') parser.add_argument('--work-root',type=Path,default=repo/'build') @@ -43,12 +44,17 @@ def hashes():return {str(p):file_hash(p) for p in inputs} source_names=(native/'data/CompiledDefs/names.bin').read_bytes() for number,choice in enumerate(choices): output=work/f'output-{number}' - result=subprocess.run([str(args.publisher.resolve()),'--body-publish',str(native),str(args.schema.resolve()),str(output), - str(choice['entry']),str(choice['body_part']),str(choice['alternative']),str(choice['replacement'])], + command=([str(args.tools.resolve()),'defs','bind-body-model',str(native/'data/CompiledDefs/game.bin'), + str(choice['entry']),str(choice['body_part']),str(choice['alternative']),str(choice['replacement']), + '--schema',str(args.schema.resolve()),'--graphics',str(inputs[-1]),'--out',str(output/'data/CompiledDefs/game.bin'),'--json'] + if args.tools else + [str(args.publisher.resolve()),'--body-publish',str(native),str(args.schema.resolve()),str(output), + str(choice['entry']),str(choice['body_part']),str(choice['alternative']),str(choice['replacement'])]) + result=subprocess.run(command, cwd=repo,capture_output=True,text=True,timeout=120) (work/f'publisher-{number}.log').write_text(result.stdout+result.stderr,encoding='utf-8') assert result.returncode==0,result.stdout+result.stderr - proof=json.loads(result.stdout);assert proof['offset']==choice['offset'] and proof['old_id']==choice['id'] + proof=json.loads(result.stdout);assert proof['offset']==choice['offset'] and proof['old_model_id' if args.tools else 'old_id']==choice['id'] expected=[dict(record) for record in records] data=bytearray(expected[choice['entry']]['data']);struct.pack_into(' Date: Sat, 10 Oct 2026 13:27:17 -0600 Subject: [PATCH 31/75] Add guarded Creature body binding GUI drafts with both-size preservation tests --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 22 ++++++ gui/app.hpp | 4 ++ gui/definitionbrowser.cpp | 59 +++++++++++++++- tools/check_all.py | 2 + tools/test_creature_body_binding_ui.py | 84 +++++++++++++++++++++++ 5 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 tools/test_creature_body_binding_ui.py diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 099b9958..89ad37a3 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -731,3 +731,25 @@ in EDITOR.md/tool help (generated CLI.md describes forge.exe, not forge-tools). Neighboring private-clone CLI regression passes all 28 cases in packet `creature-private-cli-56lfcfgv` using the same rebuilt executable. No broad-suite claim; next is GUI body-binding drafts and save/cancel/stale-source coverage. + +Body-binding GUI draft integration implemented. Verified CCreatureDef body slots +offer Bind body/alternative actions; a draft captures source bytes, definition +bank, graphics directory/path, slot and destination. Existing save/cancel/close +protection applies. Save requires a complete uint32 model ID, unchanged sources/ +graphics, canonical type-5 edit and typed publication. Valid no-op saves write +nothing. Confirmation warns about shared users and unverified compatibility. +The initial UI uses an explicit model-ID field; a named model chooser remains +useful follow-up, alongside the larger custom skinned-import scope. + +GUI build and twelve both-size cases pass in `creature-body-binding-ui-jizkwumk`: +save with independent exact intended record delta and Restore, cancel, invalid ID, +no-op, destination change and close review. All native compiled/graphics hashes +remain unchanged; no asset copies. First packet 8oxqjg_n failed because harness +input appended to the initial ID; screenshot confirmed the combined number was +refused. Corrected Ctrl+A typing, without weakening production validation. +Compact confirmation inspected: shared-user/compatibility warning and Save/Cancel +fit inside the scrolling card. Preview regressions pass in snyq9g6h and three +focused CTests pass. Registered source gate; no broad-suite/package claim. +Next: isolated stale bank/name/graphics UI checks, pack destination coverage and +named mesh choice before the next extracted package. Existing archives/RC2 stay +unchanged. No assembled-creature/animation/runtime compatibility claim. diff --git a/gui/app.hpp b/gui/app.hpp index 99a58b15..47144d91 100644 --- a/gui/app.hpp +++ b/gui/app.hpp @@ -590,6 +590,7 @@ class App { void browseDefinition(const std::string& name); bool beginDefinitionEdit(size_t field); bool beginDefinitionClone(bool creature=false,bool privateVisual=false); + bool beginDefinitionBodyModel(size_t bodyPart,size_t alternative); bool saveDefinitionClone(); bool setDefinitionEditValue(const std::string& value); bool saveDefinitionEdit(); @@ -623,6 +624,9 @@ class App { bool definitionEditing_=false,definitionEditConfirm_=false; bool definitionCloning_=false; forge::bin::File definitionCloneFile_; + forge::big::File definitionBodyGraphics_; + std::filesystem::path definitionBodyGraphicsPath_; + size_t definitionBodyPart_=0,definitionBodyAlternative_=0; std::vector definitionCloneNamesBytes_,definitionCloneBankBytes_; size_t definitionEditIndex_=0; forge::bin::Entry definitionEditExpected_; diff --git a/gui/definitionbrowser.cpp b/gui/definitionbrowser.cpp index f63b0f6f..1d7a3553 100644 --- a/gui/definitionbrowser.cpp +++ b/gui/definitionbrowser.cpp @@ -3,6 +3,7 @@ #include "windowsprocess.hpp" #include "definitionfield.hpp" #include "definitionclone.hpp" +#include "definitionbody.hpp" #include "definitionread.hpp" #include "forge/fmplinks.hpp" #include "texturebrowse.hpp" @@ -158,6 +159,34 @@ bool App::beginDefinitionEdit(size_t field) { return true; } catch(const std::exception& error) {definitionEditError_=error.what();return false;} } +bool App::beginDefinitionBodyModel(size_t bodyPart,size_t alternative) { + definitionEditError_.clear();definitionEditSuccess_.clear(); + try { + if(definitionEditing_)throw std::runtime_error("Save or cancel the current definition draft first."); + if(definitionsBusy() || modFilesBusy() || !definitionSelected_ || definitionBank_!="game.bin" || + definitionPaths()!=std::pair{definitionLoadedNames_,definitionLoadedBank_}) + throw std::runtime_error("Choose a body component in the current game bank."); + if(!packDest_.empty()) {if(!modpack::isPack(fs::u8path(packDest_)))throw std::runtime_error("Choose a valid FableForge pack.");(void)modpack::load(fs::u8path(packDest_));} + auto current=forge::bin::File::open(definitionLoadedNames_,definitionLoadedBank_); + definitionwrite::verifyEntries(definitionFile_,current); + const auto decoded=forge::defdecode::decode(current.entries()[*definitionSelected_],definitionSchema_); + const auto layout=forge::fmplinks::creaturePrefixLayout(decoded.prefix); + const auto slot=std::find_if(layout.bodyModels.begin(),layout.bodyModels.end(),[&](const auto& value){return value.bodyPart==bodyPart && value.alternative==alternative;}); + if(slot==layout.bodyModels.end())throw std::runtime_error("This body model alternative is absent."); + uint32_t id=0;for(size_t n=0;n<4;++n)id|=uint32_t(current.entries()[*definitionSelected_].data.at(slot->offset+n))<<(8*n); + const auto graphicsPath=graphicsBigPath();auto graphics=forge::big::File::open(graphicsPath); + (void)forge::defedit::setCreatureBodyModel(current,definitionSchema_,*definitionSelected_,bodyPart,alternative,id,graphics); + definitionCloneNamesBytes_=definitionwrite::cloneBytes(definitionLoadedNames_); + definitionCloneBankBytes_=definitionwrite::cloneBytes(definitionLoadedBank_); + definitionCloneFile_=std::move(current);definitionBodyGraphics_=std::move(graphics);definitionBodyGraphicsPath_=graphicsPath; + definitionBodyPart_=bodyPart;definitionBodyAlternative_=alternative;definitionEditIndex_=*definitionSelected_; + definitionEditNames_=definitionLoadedNames_;definitionEditBank_=definitionLoadedBank_; + definitionEditDestination_=fs::u8path(packDest_.empty()?saveRoot():packDest_)/"data"/"CompiledDefs"/definitionBank_; + definitionEditField_="Body "+std::to_string(bodyPart)+" / alternative "+std::to_string(alternative); + definitionEditType_="body-model";definitionEditing_=true;definitionCloning_=false;definitionEditConfirm_=false; + std::snprintf(definitionEditValue_,sizeof definitionEditValue_,"%u",id);return true; + }catch(const std::exception& error){definitionEditError_=error.what();return false;} +} bool App::beginDefinitionClone(bool creature,bool privateVisual) { definitionEditError_.clear();definitionEditSuccess_.clear(); try { @@ -258,7 +287,26 @@ bool App::saveDefinitionEdit() { throw std::runtime_error("Definition source or destination changed; start the edit again."); if(definitionEditBank_!=destination && fs::exists(destination)) throw std::runtime_error("The destination bank appeared; refresh before editing."); if(!packDest_.empty()) {if(!modpack::isPack(fs::u8path(packDest_))) throw std::runtime_error("Choose a valid FableForge pack.");(void)modpack::load(fs::u8path(packDest_));} - const bool changed=definitionwrite::replaceField(definitionEditNames_,definitionEditBank_,destination,definitionEditIndex_, + bool changed; + if(definitionEditType_=="body-model") { + if(graphicsBigPath()!=definitionBodyGraphicsPath_ || + !definitionwrite::sameBodyGraphics(definitionBodyGraphics_,forge::big::File::open(definitionBodyGraphicsPath_))) + throw std::runtime_error("The graphics source changed; start the body edit again."); + if(definitionwrite::cloneBytes(definitionEditBank_)!=definitionCloneBankBytes_ || + definitionwrite::cloneBytes(definitionEditNames_)!=definitionCloneNamesBytes_) + throw std::runtime_error("The source bank or name pool changed; start the body edit again."); + uint32_t id=0;const std::string_view value=definitionEditValue_; + const auto parsed=std::from_chars(value.data(),value.data()+value.size(),id); + if(parsed.ec!=std::errc{} || parsed.ptr!=value.data()+value.size())throw std::runtime_error("Choose a complete unsigned decimal model ID."); + auto plan=definitionCloneFile_; + const auto result=forge::defedit::setCreatureBodyModel(plan,definitionSchema_,definitionEditIndex_,definitionBodyPart_,definitionBodyAlternative_,id,definitionBodyGraphics_); + changed=result.oldModelId!=result.newModelId; + if(changed) { + const definitionwrite::CreatureBodyContract contract{definitionSchema_,definitionBodyGraphics_,definitionCloneBankBytes_,definitionCloneNamesBytes_, + definitionEditIndex_,definitionBodyPart_,definitionBodyAlternative_,id}; + definitionwrite::publishCreatureBody(plan,definitionEditNames_,definitionEditBank_,definitionBodyGraphicsPath_,destination,contract); + } + }else changed=definitionwrite::replaceField(definitionEditNames_,definitionEditBank_,destination,definitionEditIndex_, definitionEditExpected_,definitionEditExpectedNames_,definitionSchema_,definitionEditField_,definitionEditValue_,&definitionEditExpectedGroups_); definitionEditSuccess_=changed?"Saved "+definitionEditField_+".":"The field is unchanged."; definitionEditing_=false;definitionEditConfirm_=false;definitionLoaded_=false;definitionBrowseRequest_.reset();definitionBrowseIndex_=definitionEditIndex_; @@ -308,6 +356,7 @@ void App::drawDefinitionBrowser(float pad,float inner,float cardInner) { "Retargets verified self references. Components, appearance and body stay shared.": "Copies this record unchanged. References keep their existing targets."); } else ImGui::TextWrapped("Edit %s (%s)",definitionEditField_.c_str(),definitionEditType_.c_str()); + if(definitionEditType_=="body-model")theme::hint("Model ID: unique skinned type-5 mesh. All users of this component change; assets stay shared. Skeleton/runtime compatibility is not verified."); ImGui::TextWrapped("Destination: %s",definitionEditDestination_.string().c_str()); if(definitionEditType_=="CDefString") { std::string preview=definitionEditValue_; @@ -444,7 +493,7 @@ void App::drawDefinitionBrowser(float pad,float inner,float cardInner) { const char* question=definitionCloning_?(definitionEditType_=="clone-creature-private"? "Create three private definitions (assets shared)?":definitionEditType_=="clone-creature"? "Create creature with shared components?":"Create this definition in the destination bank?"): - "Write this field to the destination bank?"; + definitionEditType_=="body-model"?"Rebind this body model (shared users; compatibility unverified)?":"Write this field to the destination bank?"; const int answer=confirmRow(question,definitionCloning_?"Create":"Save",cardInner,S(28),"btn_definition_save_confirm"); if(answer>0) saveDefinitionEdit();if(answer) definitionEditConfirm_=false; } @@ -588,7 +637,7 @@ void App::drawDefinitionViewport(const ImVec2&,const ImVec2&) { try { const auto layout=forge::fmplinks::creaturePrefixLayout(definitionDecoded_->prefix); if(!layout.bodyModels.empty()) { - theme::hint("Body-part model alternatives (preview only; assets remain shared)."); + theme::hint("Body-part model alternatives. Binding changes shared component users; assets remain shared."); for(const auto& slot:layout.bodyModels) { uint32_t id=0; for(size_t n=0;n<4;++n)id|=uint32_t(entry.data.at(slot.offset+n))<<(8*n); @@ -598,6 +647,10 @@ void App::drawDefinitionViewport(const ImVec2&,const ImVec2&) { ImGui::BeginDisabled(definitionEditing_ || modFilesBusy() || meshReplaceFuture_.valid()); if(ImGui::SmallButton(label.c_str()) && selectModelById(id)) assetsTab_=1; auto_.registerWidget(("definition_body_model_"+key).c_str());ImGui::EndDisabled(); + ImGui::BeginDisabled(definitionEditing_ || definitionsBusy() || modFilesBusy()); + if(ImGui::SmallButton(("Bind body "+std::to_string(slot.bodyPart)+" / alternative "+std::to_string(slot.alternative)).c_str())) + beginDefinitionBodyModel(slot.bodyPart,slot.alternative); + auto_.registerWidget(("definition_bind_body_"+key).c_str());ImGui::EndDisabled(); } } } catch(const std::exception&) { diff --git a/tools/check_all.py b/tools/check_all.py index 42dd2050..3a4ce3f2 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -135,6 +135,8 @@ def main(): ok &= run("Body-binding CLI retail publication, Restore and argument refusals", [sys.executable, "tools/test_creature_body_publication.py", "--install", install_root, "--tools", "build/forge-tools.exe"], capture_output=True) + ok &= run("Body-binding GUI drafts, exact native deltas, guards and Restore at both sizes", + [sys.executable, "tools/test_creature_body_binding_ui.py", "--install", install_root], capture_output=True) ok &= run("Read-only Creature animation boundaries and numeric candidate observations", [sys.executable, "tools/audit_creature_animation.py", "--install", install_root, "--payload-exe", "build/fableforge_defrelocate_tests.exe"], capture_output=True) diff --git a/tools/test_creature_body_binding_ui.py b/tools/test_creature_body_binding_ui.py new file mode 100644 index 00000000..bd80fcf2 --- /dev/null +++ b/tools/test_creature_body_binding_ui.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Body binding controls, independent native delta and Restore without asset copies.""" +import argparse +import json +import os +from pathlib import Path +import struct +import subprocess +import tempfile +from audit_creature_body_models import body_slots, file_hash +from test_definition_edit_ui import click +from test_fmp_links import read_bank + + +def main(): + repo=Path(__file__).resolve().parents[1] + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--install',type=Path,required=True) + parser.add_argument('--gui',type=Path,default=repo/'build/FableForge.exe') + parser.add_argument('--forge',type=Path,default=repo/'build/forge.exe') + parser.add_argument('--work-root',type=Path,default=repo/'build') + args=parser.parse_args();native=args.install.resolve() + assert not args.work_root.resolve().is_relative_to(native) + inputs=sorted((native/'data/CompiledDefs').glob('*.bin'))+[native/'data/graphics/graphics.big'] + def hashes():return {str(p):file_hash(p) for p in inputs} + before=hashes();records=read_bank(native);choice=None + for index,record in enumerate(records): + if record['definition']!='CCreatureDef':continue + try:_,slots=body_slots(record['data']) + except ValueError:continue + for slot in slots: + other=next((s for s in slots if s['body_part']==slot['body_part'] and s['id']!=slot['id']),None) + if other:choice=dict(entry=index,**slot,replacement=other['id']);break + if choice:break + assert choice + assert not any(r['name']==str(choice['entry']) for r in records) + work=Path(tempfile.mkdtemp(prefix='creature-body-binding-ui-',dir=args.work_root.resolve())) + print('Evidence retained at',work,flush=True);cases=0 + for size,scale in [('1440x900',1),('800x600',1.5)]: + for mode in ('save','cancel','invalid','noop','destination','close-review'): + case=work/(size+'-'+mode);case.mkdir();output=case/'output' + widget=f'definition_bind_body_{choice["body_part"]}_{choice["alternative"]}' + lines=['wait_maps','wait_ready',f'set uiscale {scale}',f'set saveroot {output.as_posix()}', + 'assets_tab 6','wait_definitions',f'definition_select {choice["entry"]}','frames 3'] + lines+=click(widget)+['assert_state definition_editing 1','definition_clone_refused', + 'definition_bank_refused frontend.bin'] + value=choice['id'] if mode=='noop' else 999999 if mode=='invalid' else choice['replacement'] + lines+=click('input_definition_edit_value')+['key_down Ctrl','key_down A','key_up A','key_up Ctrl','input_text '+str(value),'frames 3', + f'screenshot {(case/"draft.png").as_posix()}'] + if mode=='cancel':lines+=click('btn_definition_edit_cancel')+['assert_state definition_editing 0'] + elif mode=='close-review': + lines+=['close','frames 3','assert_state close_prompt 1','click btn_unsaved_cancel','frames 3', + 'assert_state definition_editing 1']+click('btn_definition_edit_cancel') + elif mode in ('invalid','destination'): + if mode=='destination':lines+=[f'set saveroot {(case/"other").as_posix()}','frames 3'] + lines+=['definition_save_refused','assert_state definition_editing 1','definition_cancel'] + else: + lines+=click('btn_definition_save')+[f'screenshot {(case/"confirmation.png").as_posix()}'] + lines+=click('btn_definition_save_confirm')+['wait_ready','wait_definitions','frames 3','assert_state definition_editing 0'] + lines+=['quit'];script=case/'binding.txt';script.write_text('\n'.join(lines)+'\n',encoding='utf-8') + result=subprocess.run([str(args.gui.resolve()),'--install',str(native),'--size',size,'--auto',str(script)], + cwd=repo,env=dict(os.environ,FABLEFORGE_AUTOMATION_HIDDEN='1', + APPDATA=str(case/'profile'),LOCALAPPDATA=str(case/'local')), + capture_output=True,text=True,encoding='utf-8',errors='replace',timeout=120) + (case/'output.log').write_text(result.stdout+result.stderr,encoding='utf-8') + trace=Path(str(script)+'.log').read_text(encoding='utf-8') + assert result.returncode==0 and 'RESULT PASS' in trace and 'FAIL ' not in trace,trace[-6500:] + if mode=='save': + expected=[dict(r) for r in records];data=bytearray(expected[choice['entry']]['data']) + struct.pack_into(' Date: Sat, 10 Oct 2026 13:30:07 -0600 Subject: [PATCH 32/75] Verify body-binding GUI stale bank name and graphics guards on tiny fixtures --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 14 +++ tools/check_all.py | 2 + tools/test_creature_body_stale_ui.py | 104 ++++++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 tools/test_creature_body_stale_ui.py diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 89ad37a3..038b1744 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -753,3 +753,17 @@ focused CTests pass. Registered source gate; no broad-suite/package claim. Next: isolated stale bank/name/graphics UI checks, pack destination coverage and named mesh choice before the next extracted package. Existing archives/RC2 stay unchanged. No assembled-creature/animation/runtime compatibility claim. + +Tiny independent stale-source GUI gate now passes eight both-size cases in +`creature-body-stale-ui-li7pdpxp`: save/Restore controls plus live draft bank, +names and graphics-directory changes. Synthetic source has one CCreatureDef, +two tiny type-5 graphics directory entries and one tiny texture slot; no retail +bank copied. A screenshot barrier after opening the draft precedes mutation. +Each refusal retains editing state, logs the specific expected stale bank/name +or graphics reason, creates no output/recovery files, and leaves every fixture +byte exact except the deliberate mutation. Both positive controls independently +check the one-word delta and Restore. Initial 549_zyuf packet passed state checks; +the final gate adds specific error-log assertions to exclude unrelated refusals. +Production code unchanged. Registered unconditional source gate; no rebuild or +broad-suite repetition needed. Pack destinations and named model chooser remain +next before new extracted-package acceptance. Existing packages/RC2 unchanged. diff --git a/tools/check_all.py b/tools/check_all.py index 3a4ce3f2..ebd32279 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -375,6 +375,8 @@ def main(): [sys.executable, "tools/audit_creature_visual_graph.py"], capture_output=True) ok &= run("Creature visual navigation numeric names and malformed-reference UI refusal", [sys.executable, "tools/test_creature_visual_navigation_fixture_ui.py"], capture_output=True) + ok &= run("Creature body GUI stale bank/name/graphics refusal on tiny independent fixtures", + [sys.executable, "tools/test_creature_body_stale_ui.py"], capture_output=True) ok &= run("Creature animation synthetic boundaries and malformed component refusal", [sys.executable, "tools/audit_creature_animation.py"], capture_output=True) ok &= run("Definition clone controls, exact donors, stale drafts and paired Restore", diff --git a/tools/test_creature_body_stale_ui.py b/tools/test_creature_body_stale_ui.py new file mode 100644 index 00000000..e821c2de --- /dev/null +++ b/tools/test_creature_body_stale_ui.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Tiny independent body fixtures: GUI save control and stale bank/name/graphics refusal.""" +import argparse +import json +import os +from pathlib import Path +import struct +import subprocess +import tempfile +import time +import zlib +from test_fmp_links import crc0, read_bank +from test_definition_edit_ui import click +from test_mod_asset_choice_contributors import bank +from test_texture_lookup import texture_bank + + +def graphics_fixture(path, replacement_type=5): + bank(path,'MBANK_ALLMESHES',{ + 'MODEL_CURRENT':dict(id=10,type=5,payload=b'x',header=b''), + 'MODEL_REPLACEMENT':dict(id=20,type=replacement_type,payload=b'y',header=b'')}) + + +def fixture(root, repo): + def word(value):return struct.pack(' Date: Sat, 10 Oct 2026 13:34:55 -0600 Subject: [PATCH 33/75] test(gui): cover creature body binding pack destinations --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 19 +++++ tools/check_all.py | 2 + tools/test_creature_body_pack_ui.py | 95 +++++++++++++++++++++++ tools/test_creature_body_stale_ui.py | 4 +- 4 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 tools/test_creature_body_pack_ui.py diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 038b1744..6e8305cf 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -767,3 +767,22 @@ the final gate adds specific error-log assertions to exclude unrelated refusals. Production code unchanged. Registered unconditional source gate; no rebuild or broad-suite repetition needed. Pack destinations and named model chooser remain next before new extracted-package acceptance. Existing packages/RC2 unchanged. + +Body-binding pack GUI gate passes eight both-size cases in +`creature-body-pack-ui-j0m04p_v`: fresh publication with creation tracking and +Restore, existing-pack source preference and in-place Restore, destination switch +refusal, and a manifest removed after the draft barrier. Existing pack starts +with model 20 while base starts with 10; rebinding the pack to 10 must perform +a real edit, proving the pack donor is used rather than the base no-op donor. +Independent record checks require only the four-byte model word to change; +raw names, base files, manifest and unrelated pack files remain exact. Existing +bank backup remains exact after Restore, matching default keep-backup behavior; +fresh files and creation markers are removed. Refusals retain drafts, log the +specific reason and leave destinations unchanged except intentional manifest +removal. Initial harness packets lzoqsg4z/7xgrx3lb exposed incorrect expectations +for an unchanged names backup and default retained bank backup; corrected to the +actual preservation contract, with full restored inventory checks. Tiny fixture +helper accepts an initial model ID; default stale-source fixtures unchanged. +Registered unconditional gate. No production changes, rebuild, broad-suite or +package claim. Next: named model chooser and new extracted-package acceptance; +custom skinned import and skeleton/runtime compatibility remain outstanding. diff --git a/tools/check_all.py b/tools/check_all.py index ebd32279..36196ee4 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -377,6 +377,8 @@ def main(): [sys.executable, "tools/test_creature_visual_navigation_fixture_ui.py"], capture_output=True) ok &= run("Creature body GUI stale bank/name/graphics refusal on tiny independent fixtures", [sys.executable, "tools/test_creature_body_stale_ui.py"], capture_output=True) + ok &= run("Creature body GUI pack source preference, destination guards and Restore", + [sys.executable, "tools/test_creature_body_pack_ui.py"], capture_output=True) ok &= run("Creature animation synthetic boundaries and malformed component refusal", [sys.executable, "tools/audit_creature_animation.py"], capture_output=True) ok &= run("Definition clone controls, exact donors, stale drafts and paired Restore", diff --git a/tools/test_creature_body_pack_ui.py b/tools/test_creature_body_pack_ui.py new file mode 100644 index 00000000..8fe5eaf0 --- /dev/null +++ b/tools/test_creature_body_pack_ui.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Tiny body-binding pack routing, source preference, draft guards and Restore.""" +import argparse +import json +import os +from pathlib import Path +import struct +import subprocess +import tempfile +import time +from test_creature_body_stale_ui import fixture +from test_definition_edit_ui import click +from test_fmp_links import read_bank + + +def main(): + repo=Path(__file__).resolve().parents[1] + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--gui',type=Path,default=repo/'build/FableForge.exe') + parser.add_argument('--forge',type=Path,default=repo/'build/forge.exe') + parser.add_argument('--work-root',type=Path,default=repo/'build') + args=parser.parse_args() + work=Path(tempfile.mkdtemp(prefix='creature-body-pack-ui-',dir=args.work_root.resolve())) + print('Evidence retained at',work,flush=True);cases=0 + manifest=b'{"version":1,"name":"Body binding test"}\n' + for size,scale in [('1440x900',1),('800x600',1.5)]: + for mode in ('fresh','existing','destination','manifest'): + case=work/(size+'-'+mode);case.mkdir();root=case/'install';fixture(root,repo) + pack=case/'pack';pack.mkdir() + if mode=='existing':fixture(pack,repo,20) + (pack/'forge_pack.json').write_bytes(manifest) + source=pack if mode=='existing' else root + originals={p:p.read_bytes() for p in root.rglob('*') if p.is_file()} + pack_originals={p:p.read_bytes() for p in pack.rglob('*') if p.is_file()} + records=read_bank(source);replacement=10 if mode=='existing' else 20 + lines=['wait_maps','wait_ready',f'set uiscale {scale}',f'pack_dest {pack.as_posix()}', + 'assets_tab 6','wait_definitions','definition_select BODY','frames 3'] + lines+=click('definition_bind_body_0_0')+['assert_state definition_editing 1',f'definition_value {replacement}'] + if mode in ('fresh','existing'): + lines+=click('btn_definition_save')+click('btn_definition_save_confirm')+[ + 'wait_ready','wait_definitions','frames 3','assert_state definition_editing 0'] + else: + if mode=='destination': + other=case/'other';other.mkdir();(other/'forge_pack.json').write_bytes(manifest) + lines+=[f'pack_dest {other.as_posix()}','frames 3'] + reason='Definition source or destination changed' + else: + lines+=[f'screenshot {(case/"barrier.png").as_posix()}','frames 300'] + reason='Choose a valid FableForge pack' + lines+=['definition_save_refused','assert_state definition_editing 1', + 'assert_state definition_edit_failed 1','assert_log save definition: '+reason,'definition_cancel'] + lines+=['quit'];script=case/'pack.txt';script.write_text('\n'.join(lines)+'\n',encoding='utf-8') + with (case/'output.log').open('w',encoding='utf-8') as log: + process=subprocess.Popen([str(args.gui.resolve()),'--install',str(root),'--size',size,'--auto',str(script)], + cwd=repo,stdout=log,stderr=subprocess.STDOUT, + env=dict(os.environ,FABLEFORGE_AUTOMATION_HIDDEN='1',APPDATA=str(case/'profile'),LOCALAPPDATA=str(case/'local'))) + try: + if mode=='manifest': + deadline=time.monotonic()+45 + while not (case/'barrier.png').exists() and process.poll() is None and time.monotonic() Date: Sat, 10 Oct 2026 13:38:02 -0600 Subject: [PATCH 34/75] feat(gui): add named creature body model chooser --- docs/EDITOR.md | 2 + docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 23 +++++++ gui/definitionbrowser.cpp | 41 ++++++++++++ tools/check_all.py | 2 + tools/test_creature_body_model_picker_ui.py | 73 +++++++++++++++++++++ 5 files changed, 141 insertions(+) create mode 100644 tools/test_creature_body_model_picker_ui.py diff --git a/docs/EDITOR.md b/docs/EDITOR.md index 24b2cd77..10003539 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1458,6 +1458,8 @@ Clean game-bank CCreatureDef details list verified body-part model alternatives `forge-tools defs bind-body-model --schema --graphics --out [--names ] [--json]` changes one verified body-model ID in a CCreatureDef. Indices are explicit unsigned decimal values, not name selectors. Both current and replacement models must be unique type-5 graphics entries. The publisher recomputes the edit, refuses stale source banks or graphics-directory metadata, preserves names.bin bytes, and retains recovery records. Component users and assets stay shared; use private cloned components and owned outputs when appropriate. This does not validate skeleton/runtime compatibility or import custom meshes. +In Definitions, a verified CCreatureDef body alternative offers **Bind body / alternative**. The draft provides the model-ID field plus a named model chooser searchable by name or decimal ID. Each label includes its ID, so duplicate names stay distinct; unnamed entries remain selectable. Only unique type-5 IDs from the captured mesh bank appear. Save rechecks the source banks, graphics directory and destination, changes just the selected model-ID word, and preserves the name pool. Cancel, close protection, folder/pack destinations and Restore follow the other definition drafts. This changes all users of the component; it does not prove skeleton or runtime compatibility. + Inline CCharString drafts accept up to 127 printable ASCII characters, including empty text. Non-ASCII narrow input is filtered; legacy byte strings remain read-only and exact. CWideString drafts accept valid Unicode up to 127 UTF-8 bytes, including supplementary characters and empty text. Existing Unicode values within that limit can be edited or saved unchanged. C0/C1 controls, malformed encoding, overlong originals, missing terminators and opaque trailers remain unsupported. The shared writer requires one complete NUL-terminated original string before replacement. CWideString uses UTF-16LE with a final zero code unit, without a count prefix. CLI and GUI edits use strict UTF-8 conversion. Odd lengths, malformed surrogates, missing or embedded terminators and opaque trailers refuse before writes. Unchanged supported drafts retain their exact original bytes. diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 6e8305cf..7fd98a0e 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -786,3 +786,26 @@ helper accepts an initial model ID; default stale-source fixtures unchanged. Registered unconditional gate. No production changes, rebuild, broad-suite or package claim. Next: named model chooser and new extracted-package acceptance; custom skinned import and skeleton/runtime compatibility remain outstanding. + +Named body-model chooser implemented alongside the explicit numeric ID field. +It uses only the draft's captured MBANK_ALLMESHES directory, counts all entries +by ID (including other types), and lists unique type-5 IDs. Labels show ID plus +name or an unnamed placeholder; duplicate names remain distinct. Case-insensitive +name/decimal-ID search and a clipped list follow existing definition choosers. +Search resets on beginning a body draft. Selection sets only the existing draft +value; no new write path or relaxed publication/stale-source guard. + +GUI builds; twelve both-size cases pass in +`creature-body-model-picker-ui-txdybjn1`: named selection, name search, ID search, +unnamed entry, duplicate names with different IDs, and exclusion of wrong types +plus an ID duplicated across type 5/type 1. Tests click actual popup controls, +clear widget registrations for filtered-absence assertions, independently check +the sole model-word delta and raw name preservation, then Restore. Compact +duplicate-name screenshot inspected: ID-labelled alternatives and search fit. +Three focused CTests pass. Rebuilt-GUI pack regressions pass eight cases in +`creature-body-pack-ui-0fh4qcac`; stale bank/names/graphics regressions pass eight +in `creature-body-stale-ui-0a_hgqog`. All fixtures tiny and source bytes exact; +no retail asset copies. Registered chooser gate and documented GUI semantics. +Existing immutable packages/RC2 do not include these changes. Next is a fresh +development package with extracted acceptance, then the outstanding custom +skinned-import/compatibility work. No broad-suite or runtime claim. diff --git a/gui/definitionbrowser.cpp b/gui/definitionbrowser.cpp index 1d7a3553..bd9cf9c0 100644 --- a/gui/definitionbrowser.cpp +++ b/gui/definitionbrowser.cpp @@ -17,6 +17,7 @@ #include #include #include +#include namespace albion::gui { namespace fs=std::filesystem; namespace { @@ -184,6 +185,7 @@ bool App::beginDefinitionBodyModel(size_t bodyPart,size_t alternative) { definitionEditDestination_=fs::u8path(packDest_.empty()?saveRoot():packDest_)/"data"/"CompiledDefs"/definitionBank_; definitionEditField_="Body "+std::to_string(bodyPart)+" / alternative "+std::to_string(alternative); definitionEditType_="body-model";definitionEditing_=true;definitionCloning_=false;definitionEditConfirm_=false; + definitionNameSearch_[0]=0; std::snprintf(definitionEditValue_,sizeof definitionEditValue_,"%u",id);return true; }catch(const std::exception& error){definitionEditError_=error.what();return false;} } @@ -476,6 +478,45 @@ void App::drawDefinitionBrowser(float pad,float inner,float cardInner) { text?ImGuiInputTextFlags_CallbackCharFilter:0,text?(wide?definitionUnicodeChar:definitionASCIIChar):nullptr); } auto_.registerWidget("input_definition_edit_value"); + if(definitionEditType_=="body-model") { + // Use the captured directory, never a freshly loaded bank while a + // draft is open. Save still revalidates the directory and sources. + const auto* meshes=definitionBodyGraphics_.findBank("MBANK_ALLMESHES"); + if(meshes) { + std::unordered_map counts; + for(const auto& mesh:meshes->entries) ++counts[mesh.id]; + const std::string_view value=definitionEditValue_;uint32_t selected=0; + const auto parsed=std::from_chars(value.data(),value.data()+value.size(),selected); + const bool valid=parsed.ec==std::errc{} && parsed.ptr==value.data()+value.size(); + std::string preview="Choose a named model"; + for(const auto& mesh:meshes->entries) if(valid && mesh.id==selected && mesh.type==5 && counts[mesh.id]==1) { + preview=std::to_string(mesh.id)+": "+(mesh.name.empty()?"(unnamed model)":mesh.name);break; + } + ImGui::SetNextItemWidth(cardInner); + const bool open=ImGui::BeginCombo("##definitionbodymodel",preview.c_str()); + auto_.registerWidget("combo_definition_body_model"); + if(open) { + ImGui::InputText("Search models",definitionNameSearch_,sizeof definitionNameSearch_); + auto_.registerWidget("input_definition_body_model_search"); + const auto search=definitionLower(definitionNameSearch_); + std::vector options; + for(size_t i=0;ientries.size();++i) { + const auto& mesh=meshes->entries[i]; + if(mesh.type==5 && counts[mesh.id]==1 && (search.empty() || + definitionLower(mesh.name).find(search)!=std::string::npos || + std::to_string(mesh.id).find(search)!=std::string::npos)) options.push_back(i); + } + ImGuiListClipper clipper;clipper.Begin(int(options.size())); + while(clipper.Step()) for(int i=clipper.DisplayStart;ientries[options[size_t(i)]]; + const auto label=std::to_string(mesh.id)+": "+(mesh.name.empty()?"(unnamed model)":mesh.name); + if(ImGui::Selectable(label.c_str(),valid && selected==mesh.id)) setDefinitionEditValue(std::to_string(mesh.id)); + auto_.registerWidget(("definition_body_model_option_"+std::to_string(mesh.id)).c_str()); + } + ImGui::EndCombo(); + } + } + } if(definitionCloning_ && definitionEditType_=="clone-creature-private") { ImGui::TextDisabled("Body name (127 ASCII characters max)");ImGui::SetNextItemWidth(cardInner); ImGui::InputText("##privatebody",definitionCloneBodyName_,sizeof definitionCloneBodyName_,ImGuiInputTextFlags_CallbackCharFilter,definitionASCIIChar); diff --git a/tools/check_all.py b/tools/check_all.py index 36196ee4..fb46ffb0 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -379,6 +379,8 @@ def main(): [sys.executable, "tools/test_creature_body_stale_ui.py"], capture_output=True) ok &= run("Creature body GUI pack source preference, destination guards and Restore", [sys.executable, "tools/test_creature_body_pack_ui.py"], capture_output=True) + ok &= run("Creature body named model chooser, ID ambiguity exclusion and Restore", + [sys.executable, "tools/test_creature_body_model_picker_ui.py"], capture_output=True) ok &= run("Creature animation synthetic boundaries and malformed component refusal", [sys.executable, "tools/audit_creature_animation.py"], capture_output=True) ok &= run("Definition clone controls, exact donors, stale drafts and paired Restore", diff --git a/tools/test_creature_body_model_picker_ui.py b/tools/test_creature_body_model_picker_ui.py new file mode 100644 index 00000000..0670951a --- /dev/null +++ b/tools/test_creature_body_model_picker_ui.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Choose body mesh names/IDs, disambiguate labels and exclude invalid IDs.""" +import argparse +import json +import os +from pathlib import Path +import struct +import subprocess +import tempfile +from test_creature_body_stale_ui import fixture +from test_definition_edit_ui import click +from test_fmp_links import read_bank +from test_mod_asset_choice_contributors import bank + + +def main(): + repo=Path(__file__).resolve().parents[1] + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--gui',type=Path,default=repo/'build/FableForge.exe') + parser.add_argument('--forge',type=Path,default=repo/'build/forge.exe') + parser.add_argument('--work-root',type=Path,default=repo/'build') + args=parser.parse_args() + work=Path(tempfile.mkdtemp(prefix='creature-body-model-picker-ui-',dir=args.work_root.resolve())) + print('Evidence retained at',work,flush=True);cases=0 + for size,scale in [('1440x900',1),('800x600',1.5)]: + for mode in ('named','name-search','id-search','unnamed','duplicate-name','exclusions'): + case=work/(size+'-'+mode);case.mkdir();root=case/'install';fixture(root,repo);output=case/'output' + graphics=root/'data/graphics/graphics.big' + records={'CURRENT':dict(id=10,type=5,payload=b'x',header=b''), + 'REPLACEMENT':dict(id=20,type=5,payload=b'y',header=b'')} + if mode=='unnamed':records['']=records.pop('REPLACEMENT') + if mode=='duplicate-name': + records['DUP_A']=records.pop('REPLACEMENT') + records['DUP_B']=dict(id=30,type=5,payload=b'z',header=b'') + if mode=='exclusions': + records.update(WRONG_TYPE=dict(id=30,type=1,payload=b'z',header=b''), + AMBIGUOUS_MESH=dict(id=40,type=5,payload=b'a',header=b''), + AMBIGUOUS_OTHER=dict(id=40,type=1,payload=b'b',header=b'')) + bank(graphics,'MBANK_ALLMESHES',records) + if mode=='duplicate-name':graphics.write_bytes(graphics.read_bytes().replace(b'DUP_B',b'DUP_A')) + originals={p:p.read_bytes() for p in root.rglob('*') if p.is_file()} + expected=read_bank(root);choice=30 if mode=='duplicate-name' else 20 + payload=bytearray(expected[0]['data']);struct.pack_into(' Date: Sat, 10 Oct 2026 13:42:08 -0600 Subject: [PATCH 35/75] test(package): validate extracted creature body workflows --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 27 +++++++++++++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 2 +- tools/test_creature_body_binding_ui.py | 5 ++-- tools/test_creature_body_model_picker_ui.py | 5 ++-- tools/test_creature_body_pack_ui.py | 5 ++-- tools/test_creature_body_publication.py | 7 +++--- tools/test_creature_body_stale_ui.py | 5 ++-- tools/test_package.py | 15 ++++++++++++ 9 files changed, 60 insertions(+), 13 deletions(-) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 7fd98a0e..b310dfc2 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -809,3 +809,30 @@ no retail asset copies. Registered chooser gate and documented GUI semantics. Existing immutable packages/RC2 do not include these changes. Next is a fresh development package with extracted acceptance, then the outstanding custom skinned-import/compatibility work. No broad-suite or runtime claim. + +Fresh immutable development checkpoint: +`dist/development/20261010-creature-body-gui/FableForge-0.18.1-dev-win64.zip`, +SHA256 `eccdbb4d78a876b60f7987dd9d57425f4d7651999f5a08947ff0b4ec2558cebf`. +Application 37.2 MB, separate guide 0.83 MB. Shipping targets built in this owned +cache before packaging. Manifest records packaging checkout e8ea956 with tracked +test-harness edits, not a clean-checkout/executable-provenance guarantee. Earlier +archives, release tags and RC2 untouched. Package includes navigation, body preview, +body binding and named choice; it does not include custom skinned import. + +Extracted acceptance terminal-success packet `package-smoke-zcdunryg` verifies +117 file identities and 141 documentation links, and passes nine workflows: +private clone CLI (28 cases), native visual navigation, synthetic navigation, +native body preview, native body-binding CLI publication/Restore plus fifteen +refusals, native both-size body draft controls (12 cases), stale-source fixtures +(8), pack routing/Restore (8), named model chooser (12). Executables run from the +extracted path containing literal percent/ampersand characters, with unrelated +working directory and private profiles. Five body harnesses now accept explicit +--cwd; package runner registers all eight new visual/body workflows. Native +compiled/graphics hashes unchanged; only independent compiled-definition outputs +and tiny synthetic assets used, no full asset-bank copies. Report: 138 workflows +not run; two unrelated missing-corpus skips (named FMP links, F2 refusal UI). +This is focused extracted acceptance, not the full offline suite, clean-machine +trial or in-game compatibility. Source CTests reused from the accepted chooser +checkpoint; no core writer change. Navigation/roadmap routing now points to this +checkpoint. Next: custom skinned body import against donor skeleton evidence and +the remaining compatibility/placement/runtime validation, preserving full scope. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index f1e01601..0bdac8f4 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Core, paired publication, CLI and 26 both-size private GUI checks pass. Extracted private checkpoint passes nine workflows; 130 not run/two unrelated skips. Body/appearance navigation and custom body asset work are next; no runtime/full-feature claim. RC2 unchanged. +> Active continuation: [Creature visual workflows](HANDOFF_CREATURE_CLONE_2026-10-10.md). Private cloning, body/appearance navigation, body-model binding and named model choice are implemented. Body checkpoint extracted acceptance passes nine selected workflows; 138 not run/two unrelated skips. Custom skinned import and skeleton/runtime compatibility remain next; no full-feature claim. RC2 unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index c688e839..79e4a99a 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [private Creature visual cloning](HANDOFF_CREATURE_CLONE_2026-10-10.md). Core, paired publication, CLI and 26 both-size private GUI checks pass. Extracted private checkpoint passes nine workflows; 130 not run/two unrelated skips. Body/appearance navigation and custom body asset work are next; no runtime/full-feature claim. RC2 unchanged. +> Active continuation: [Creature visual workflows](HANDOFF_CREATURE_CLONE_2026-10-10.md). Private cloning, body/appearance navigation, body-model binding and named model choice are implemented. Body checkpoint extracted acceptance passes nine selected workflows; 138 not run/two unrelated skips. Custom skinned import and skeleton/runtime compatibility remain next; no full-feature claim. RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there diff --git a/tools/test_creature_body_binding_ui.py b/tools/test_creature_body_binding_ui.py index bd80fcf2..0de84700 100644 --- a/tools/test_creature_body_binding_ui.py +++ b/tools/test_creature_body_binding_ui.py @@ -19,6 +19,7 @@ def main(): parser.add_argument('--gui',type=Path,default=repo/'build/FableForge.exe') parser.add_argument('--forge',type=Path,default=repo/'build/forge.exe') parser.add_argument('--work-root',type=Path,default=repo/'build') + parser.add_argument('--cwd',type=Path,default=repo) args=parser.parse_args();native=args.install.resolve() assert not args.work_root.resolve().is_relative_to(native) inputs=sorted((native/'data/CompiledDefs').glob('*.bin'))+[native/'data/graphics/graphics.big'] @@ -59,7 +60,7 @@ def hashes():return {str(p):file_hash(p) for p in inputs} lines+=click('btn_definition_save_confirm')+['wait_ready','wait_definitions','frames 3','assert_state definition_editing 0'] lines+=['quit'];script=case/'binding.txt';script.write_text('\n'.join(lines)+'\n',encoding='utf-8') result=subprocess.run([str(args.gui.resolve()),'--install',str(native),'--size',size,'--auto',str(script)], - cwd=repo,env=dict(os.environ,FABLEFORGE_AUTOMATION_HIDDEN='1', + cwd=args.cwd.resolve(),env=dict(os.environ,FABLEFORGE_AUTOMATION_HIDDEN='1', APPDATA=str(case/'profile'),LOCALAPPDATA=str(case/'local')), capture_output=True,text=True,encoding='utf-8',errors='replace',timeout=120) (case/'output.log').write_text(result.stdout+result.stderr,encoding='utf-8') @@ -71,7 +72,7 @@ def hashes():return {str(p):file_hash(p) for p in inputs} assert read_bank(output)==expected,'GUI body delta differs' bank=output/'data/CompiledDefs/game.bin';names=bank.parent/'names.bin' assert names.read_bytes()==(native/'data/CompiledDefs/names.bin').read_bytes() - restore=subprocess.run([str(args.forge.resolve()),'restore','--install',str(output)],cwd=repo, + restore=subprocess.run([str(args.forge.resolve()),'restore','--install',str(output)],cwd=args.cwd.resolve(), capture_output=True,text=True,timeout=60) assert restore.returncode==0,restore.stdout+restore.stderr assert not bank.exists() and not names.exists() diff --git a/tools/test_creature_body_model_picker_ui.py b/tools/test_creature_body_model_picker_ui.py index 0670951a..b8175cb6 100644 --- a/tools/test_creature_body_model_picker_ui.py +++ b/tools/test_creature_body_model_picker_ui.py @@ -19,6 +19,7 @@ def main(): parser.add_argument('--gui',type=Path,default=repo/'build/FableForge.exe') parser.add_argument('--forge',type=Path,default=repo/'build/forge.exe') parser.add_argument('--work-root',type=Path,default=repo/'build') + parser.add_argument('--cwd',type=Path,default=repo) args=parser.parse_args() work=Path(tempfile.mkdtemp(prefix='creature-body-model-picker-ui-',dir=args.work_root.resolve())) print('Evidence retained at',work,flush=True);cases=0 @@ -56,13 +57,13 @@ def main(): script=case/'picker.txt';script.write_text('\n'.join(lines)+'\n',encoding='utf-8') with (case/'output.log').open('w',encoding='utf-8') as log: result=subprocess.run([str(args.gui.resolve()),'--install',str(root),'--size',size,'--auto',str(script)], - cwd=repo,stdout=log,stderr=subprocess.STDOUT,timeout=90, + cwd=args.cwd.resolve(),stdout=log,stderr=subprocess.STDOUT,timeout=90, env=dict(os.environ,FABLEFORGE_AUTOMATION_HIDDEN='1',APPDATA=str(case/'profile'),LOCALAPPDATA=str(case/'local'))) trace=Path(str(script)+'.log').read_text(encoding='utf-8') assert result.returncode==0 and 'RESULT PASS' in trace and 'FAIL ' not in trace,trace[-6000:] assert read_bank(output)==expected,'named model choice changed more than one word or chose wrong ID' assert (output/'data/CompiledDefs/names.bin').read_bytes()==originals[root/'data/CompiledDefs/names.bin'] - result=subprocess.run([str(args.forge.resolve()),'restore','--install',str(output)],cwd=repo,capture_output=True,text=True,timeout=60) + result=subprocess.run([str(args.forge.resolve()),'restore','--install',str(output)],cwd=args.cwd.resolve(),capture_output=True,text=True,timeout=60) assert result.returncode==0,result.stdout+result.stderr assert not list(output.rglob('*.bin')) and not list(output.rglob('*.forge-created')) assert {p:p.read_bytes() for p in root.rglob('*') if p.is_file()}==originals,'source fixture changed' diff --git a/tools/test_creature_body_pack_ui.py b/tools/test_creature_body_pack_ui.py index 8fe5eaf0..78d8315a 100644 --- a/tools/test_creature_body_pack_ui.py +++ b/tools/test_creature_body_pack_ui.py @@ -19,6 +19,7 @@ def main(): parser.add_argument('--gui',type=Path,default=repo/'build/FableForge.exe') parser.add_argument('--forge',type=Path,default=repo/'build/forge.exe') parser.add_argument('--work-root',type=Path,default=repo/'build') + parser.add_argument('--cwd',type=Path,default=repo) args=parser.parse_args() work=Path(tempfile.mkdtemp(prefix='creature-body-pack-ui-',dir=args.work_root.resolve())) print('Evidence retained at',work,flush=True);cases=0 @@ -52,7 +53,7 @@ def main(): lines+=['quit'];script=case/'pack.txt';script.write_text('\n'.join(lines)+'\n',encoding='utf-8') with (case/'output.log').open('w',encoding='utf-8') as log: process=subprocess.Popen([str(args.gui.resolve()),'--install',str(root),'--size',size,'--auto',str(script)], - cwd=repo,stdout=log,stderr=subprocess.STDOUT, + cwd=args.cwd.resolve(),stdout=log,stderr=subprocess.STDOUT, env=dict(os.environ,FABLEFORGE_AUTOMATION_HIDDEN='1',APPDATA=str(case/'profile'),LOCALAPPDATA=str(case/'local'))) try: if mode=='manifest': @@ -79,7 +80,7 @@ def main(): if name=='game.bin':assert Path(str(path)+'.forge-orig').read_bytes()==pack_originals[path] else:assert not Path(str(path)+'.forge-orig').exists(),'unchanged names need no backup' else:assert Path(str(path)+'.forge-created').is_file() - result=subprocess.run([str(args.forge.resolve()),'restore','--install',str(pack)],cwd=repo,capture_output=True,text=True,timeout=60) + result=subprocess.run([str(args.forge.resolve()),'restore','--install',str(pack)],cwd=args.cwd.resolve(),capture_output=True,text=True,timeout=60) assert result.returncode==0,result.stdout+result.stderr expected_pack=dict(pack_originals) if mode=='existing': diff --git a/tools/test_creature_body_publication.py b/tools/test_creature_body_publication.py index 0c8b34ae..cd63ca62 100644 --- a/tools/test_creature_body_publication.py +++ b/tools/test_creature_body_publication.py @@ -19,6 +19,7 @@ def main(): parser.add_argument('--forge',type=Path,default=repo/'build/forge.exe') parser.add_argument('--schema',type=Path,default=repo/'docs/re_reference/def_schema.json') parser.add_argument('--work-root',type=Path,default=repo/'build') + parser.add_argument('--cwd',type=Path,default=repo) args=parser.parse_args();native=args.install.resolve() assert not args.work_root.resolve().is_relative_to(native) inputs=sorted((native/'data/CompiledDefs').glob('*.bin'))+[native/'data/graphics/graphics.big'] @@ -51,7 +52,7 @@ def hashes():return {str(p):file_hash(p) for p in inputs} [str(args.publisher.resolve()),'--body-publish',str(native),str(args.schema.resolve()),str(output), str(choice['entry']),str(choice['body_part']),str(choice['alternative']),str(choice['replacement'])]) result=subprocess.run(command, - cwd=repo,capture_output=True,text=True,timeout=120) + cwd=args.cwd.resolve(),capture_output=True,text=True,timeout=120) (work/f'publisher-{number}.log').write_text(result.stdout+result.stderr,encoding='utf-8') assert result.returncode==0,result.stdout+result.stderr proof=json.loads(result.stdout);assert proof['offset']==choice['offset'] and proof['old_model_id' if args.tools else 'old_id']==choice['id'] @@ -63,7 +64,7 @@ def hashes():return {str(p):file_hash(p) for p in inputs} assert bank.read_bytes()[:13+12*len(records)]==source_bank[:13+12*len(records)],'identity/header table changed' assert names.read_bytes()==source_names,'name pool changed' for path in (bank,names):assert Path(str(path)+'.forge-created').is_file() - restored=subprocess.run([str(args.forge.resolve()),'restore','--install',str(output)],cwd=repo, + restored=subprocess.run([str(args.forge.resolve()),'restore','--install',str(output)],cwd=args.cwd.resolve(), capture_output=True,text=True,timeout=60) (work/f'restore-{number}.log').write_text(restored.stdout+restored.stderr,encoding='utf-8') assert restored.returncode==0,restored.stdout+restored.stderr @@ -84,7 +85,7 @@ def hashes():return {str(p):file_hash(p) for p in inputs} variants+=[command+['--unknown'],command+['--out',str(refusal)],command[:8]+command[10:], command[:10]+command[12:],command[:-3]+['--out']] for index,variant in enumerate(variants): - result=subprocess.run(variant,cwd=repo,capture_output=True,text=True,timeout=60) + result=subprocess.run(variant,cwd=args.cwd.resolve(),capture_output=True,text=True,timeout=60) (work/f'refusal-{index}.log').write_text(result.stdout+result.stderr,encoding='utf-8') assert result.returncode!=0,'invalid binding command accepted' assert not refusal.parent.exists(),'refused command created output' diff --git a/tools/test_creature_body_stale_ui.py b/tools/test_creature_body_stale_ui.py index f0a26835..5b3873e1 100644 --- a/tools/test_creature_body_stale_ui.py +++ b/tools/test_creature_body_stale_ui.py @@ -48,6 +48,7 @@ def main(): parser.add_argument('--gui',type=Path,default=repo/'build/FableForge.exe') parser.add_argument('--forge',type=Path,default=repo/'build/forge.exe') parser.add_argument('--work-root',type=Path,default=repo/'build') + parser.add_argument('--cwd',type=Path,default=repo) args=parser.parse_args() work=Path(tempfile.mkdtemp(prefix='creature-body-stale-ui-',dir=args.work_root.resolve())) print('Evidence retained at',work,flush=True);cases=0 @@ -70,7 +71,7 @@ def main(): changed=None with (case/'output.log').open('w',encoding='utf-8') as log: process=subprocess.Popen([str(args.gui.resolve()),'--install',str(root),'--size',size,'--auto',str(script)], - cwd=repo,stdout=log,stderr=subprocess.STDOUT, + cwd=args.cwd.resolve(),stdout=log,stderr=subprocess.STDOUT, env=dict(os.environ,FABLEFORGE_AUTOMATION_HIDDEN='1',APPDATA=str(case/'profile'),LOCALAPPDATA=str(case/'local'))) try: if mode!='save': @@ -93,7 +94,7 @@ def main(): if mode=='save': expected=read_bank(root);payload=bytearray(expected[0]['data']);struct.pack_into(' Date: Sat, 10 Oct 2026 13:44:52 -0600 Subject: [PATCH 36/75] feat(mesh): retain exact donor skeleton blocks for composition --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 30 ++++++++++++++++++++ libs/forgecore/include/forge/meshpreview.hpp | 11 +++++++ libs/forgecore/src/meshpreview.cpp | 4 ++- tests/test_meshbounds.cpp | 1 + tests/test_meshskeleton.cpp | 26 +++++++++++++---- 5 files changed, 65 insertions(+), 7 deletions(-) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index b310dfc2..0e8886f9 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -836,3 +836,33 @@ trial or in-game compatibility. Source CTests reused from the accepted chooser checkpoint; no core writer change. Navigation/roadmap routing now points to this checkpoint. Next: custom skinned body import against donor skeleton evidence and the remaining compatibility/placement/runtime validation, preserving full scope. + +Custom skinned-import foundation: canonical meshpreview::Geometry now retains an +optional SkeletonBlocks containing the decoded donor global-ID, name, C3DBone, +bind-pose TRS and full inverse-bind matrix byte spans. Existing Bone is a preview +representation that normalizes IBM padding and previously discarded TRS, so it +cannot alone reconstruct the donor. Zero-bone meshes have no donor blocks. +Existing decoder validation still precedes successful return; no writer or install +mutation added. This supplies exact raw records to the upcoming skinned composer, +not a substitute for arbitrary geometry, weights, palette partition or import UI. + +Independent read-only Python mesh_rw.clone_skeleton oracle on installed graphics +produced concatenated five-span CRC32 values: 3985=caf34886, 4521=23ca606a, +5110=8bc2ee81, 5121=f62e93d9, 5150=9d6a7aaa. Direct rebuilt +fableforge_meshskeleton_tests native gate passes all five (17-28 bones each), +raw lengths/checksums plus the existing normalized-bone and skin CRC assertions. +Graphics SHA256 observed: 8b909c4036a89840413f66828d855ee3ccf29796feaeb1f05f36727a5086f617. +Focused rebuilt meshbounds/headpose CTests both pass; static controls explicitly +require absence of donor skeleton spans. No broad-suite/GUI/package claim. Shipping +GUI/tools require rebuild after the public Geometry layout change; immutable body +checkpoint stays as validated, without this foundation. + +Evidence inspected: FableTLC docs/formats/MESH.md sections 3/8/9/10 and addon +mesh_rw.py clone_skeleton, _write_skeleton and encode_skin; locally available +EgoCore-git MeshParser.h confirms five u16 optional-header counts and raw bone +block shapes. Current preview reader combines packed-name/volume counts and +does not walk volume/generator blocks; do not extend donor acceptance to those +headers without resolving this known layout gap. Accepted head donors exercise +the existing supported path. Next: complete skinned composer using raw skeleton +spans, authored influences and bounded multi-block palettes, then geometry import +and owned bank/definition publication. Runtime compatibility remains unproven. diff --git a/libs/forgecore/include/forge/meshpreview.hpp b/libs/forgecore/include/forge/meshpreview.hpp index 51b128b0..7cb47fc3 100644 --- a/libs/forgecore/include/forge/meshpreview.hpp +++ b/libs/forgecore/include/forge/meshpreview.hpp @@ -63,6 +63,16 @@ struct Bone { // fourth row is padding; the reader sets it to (0,0,0,1). std::array inverseBind{}; }; +// Decoded donor spans, before preview normalization. A skinned composer must +// retain the full records (including matrix padding and bind-pose TRS), not +// reconstruct them from the rendering-oriented Bone structure. +struct SkeletonBlocks { + std::vector indices; // u16 global IDs, 2 bytes per bone + std::vector names; // counted NUL-terminated names + std::vector bones; // C3DBone records, 60 bytes per bone + std::vector keyframes; // bind-pose TRS, 48 bytes per bone + std::vector inverseBinds; // raw matrices, 64 bytes per bone +}; struct PrimitiveInfo { uint32_t vertexCount = 0; uint32_t indexCount = 0; @@ -84,6 +94,7 @@ struct Geometry { std::vector materials; std::vector helpers; std::vector bones; + std::optional skeletonBlocks; // absent when BoneCount is zero std::vector primitives; uint32_t boneCount = 0; uint32_t primitiveCount = 0; diff --git a/libs/forgecore/src/meshpreview.cpp b/libs/forgecore/src/meshpreview.cpp index 98ab0f8a..afa5dcce 100644 --- a/libs/forgecore/src/meshpreview.cpp +++ b/libs/forgecore/src/meshpreview.cpp @@ -200,11 +200,12 @@ Geometry decodeLod0(const std::vector& payload, uint32_t meshType) { if(bones) { const size_t count=size_t(bones); c.need(count*2); + const std::vector indicesRaw(c.b.begin()+c.p,c.b.begin()+c.p+count*2); std::vector globalIds(count); for(auto& id:globalIds) id=c.u16(); const auto namesRaw=fableLzo(c,size_t(boneNames)); const auto bonesRaw=fableLzo(c,count*60); - fableLzo(c,count*48); // bind-pose TRS; retained by the source archive + const auto keyframesRaw=fableLzo(c,count*48); const auto ibmsRaw=fableLzo(c,count*64); result.bones.reserve(count); size_t namePos=0; @@ -231,6 +232,7 @@ Geometry decodeLod0(const std::vector& payload, uint32_t meshType) { throw std::runtime_error("meshpreview: nonfinite inverse bind matrix"); result.bones.push_back(std::move(bone)); } + result.skeletonBlocks=SkeletonBlocks{indicesRaw,namesRaw,bonesRaw,keyframesRaw,ibmsRaw}; } c.skip(48); for(int32_t i=0;i cases={{{4521,27,"EYE_SET_L",0x12e44699u,1131,0x0a120a63u}, - {5121,28,"EYE_SET_R",0xd8ebec00u,1012,0xe8945f52u}, - {5150,28,"EYE_SET_R",0x480a3c2fu,846,0x9a81b8a7u}, - {3985,17,"COLD_BREATH",0x60d20858u,757,0x6398ee17u}, - {5110,27,"EYE_SET_R",0x22d4bb4eu,822,0xfc4567b6u}}}; + // Raw CRC concatenates clone_skeleton's indices/names/bones/TRS/IBMs + // without normalization or padding changes (independent Python oracle). + constexpr std::array cases={{{4521,27,"EYE_SET_L",0x12e44699u,1131,0x0a120a63u,0x23ca606au}, + {5121,28,"EYE_SET_R",0xd8ebec00u,1012,0xe8945f52u,0xf62e93d9u}, + {5150,28,"EYE_SET_R",0x480a3c2fu,846,0x9a81b8a7u,0x9d6a7aaau}, + {3985,17,"COLD_BREATH",0x60d20858u,757,0x6398ee17u,0xcaf34886u}, + {5110,27,"EYE_SET_R",0x22d4bb4eu,822,0xfc4567b6u,0x8bc2ee81u}}}; for(const auto& expected:cases) { const auto geometry=forge::meshpreview::readLod0( std::filesystem::path(argv[1]),expected.id); @@ -49,6 +51,18 @@ int main(int argc,char** argv) { for(float value:bone.inverseBind) u32(std::bit_cast(value)); } crc=~crc; + require(geometry.skeletonBlocks.has_value(),"missing raw donor skeleton blocks"); + const auto& raw=*geometry.skeletonBlocks; + require(raw.indices.size()==expected.bones*2 && raw.bones.size()==expected.bones*60 && + raw.keyframes.size()==expected.bones*48 && raw.inverseBinds.size()==expected.bones*64, + "raw donor skeleton span lengths"); + uint32_t rawCrc=0xffffffffu; + for(const auto* block:{&raw.indices,&raw.names,&raw.bones,&raw.keyframes,&raw.inverseBinds}) + for(uint8_t value:*block) { + rawCrc^=value; + for(int bit=0;bit<8;++bit) rawCrc=(rawCrc>>1)^((rawCrc&1)?0xedb88320u:0); + } + require(~rawCrc==expected.rawCrc,"raw donor skeleton differs from Python clone_skeleton"); if(crc!=expected.crc) std::cerr << "mesh " << expected.id << " CRC " << std::hex << crc << " expected " << expected.crc << std::dec << '\n'; From a799b7433ca2aac80a091e2574c4d346a6ad79bd Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:51:59 -0600 Subject: [PATCH 37/75] feat(mesh): compose skinned geometry with bounded donor palettes --- CMakeLists.txt | 3 + docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 42 ++++ libs/forgecore/include/forge/meshcompose.hpp | 20 ++ libs/forgecore/src/meshcompose.cpp | 193 ++++++++++++++++++- tests/test_skinnedcompose.cpp | 142 ++++++++++++++ tools/test_skinned_composition.py | 76 ++++++++ 6 files changed, 474 insertions(+), 2 deletions(-) create mode 100644 tests/test_skinnedcompose.cpp create mode 100644 tools/test_skinned_composition.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 34006586..e38477e0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,6 +128,9 @@ target_link_libraries(fableforge_meshbounds_tests PRIVATE forgecore_slice) add_test(NAME fableforge_meshbounds_tests COMMAND fableforge_meshbounds_tests) add_executable(fableforge_meshskeleton_tests tests/test_meshskeleton.cpp) target_link_libraries(fableforge_meshskeleton_tests PRIVATE forgecore_slice) +add_executable(fableforge_skinnedcompose_tests tests/test_skinnedcompose.cpp) +target_link_libraries(fableforge_skinnedcompose_tests PRIVATE forgecore_slice) +add_test(NAME fableforge_skinnedcompose_tests COMMAND fableforge_skinnedcompose_tests) add_executable(fableforge_animation_tests tests/test_animation.cpp) target_link_libraries(fableforge_animation_tests PRIVATE forgecore_slice) add_test(NAME fableforge_animation_tests COMMAND fableforge_animation_tests) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 0e8886f9..add67bf3 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -866,3 +866,45 @@ headers without resolving this known layout gap. Accepted head donors exercise the existing supported path. Next: complete skinned composer using raw skeleton spans, authored influences and bounded multi-block palettes, then geometry import and owned bank/definition publication. Runtime compatibility remains unproven. + +Canonical composeSkinned API now emits new type-5 payload/Info bytes from authored +geometry, per-vertex local-slot influence lists, materials and retained donor raw +skeleton spans. Float stride 28, triangle lists, no degenerate sentinel, <=16-bone +greedy animated-block palettes. Splitting regroups vertices in face-first order +and duplicates shared vertices across blocks; post-duplication u16 capacity is +checked. Top three positive influences normalize via the largest first (finite +large sums cannot overflow); byte weights sum exactly 255. Explicit empty lists +bind to slot zero; missing per-vertex lists refuse. Duplicate slots, negative/ +nonfinite weights, out-of-range bones and nonfinite geometry refuse. Public API +documents top-three reduction; future import UI must expose this authoring limit. +Every raw donor span survives exactly in LOD0 and ghost, including ignored IBM/TRS +padding. Validates shape/counts, meaningful bind floats and parent cycles. Donor +support: 1-256 local slots; name pool 4 bytes to 1 MiB. Tiny <=3-byte pools are +outside the current chunked reader contract. PhysicsIndex zero; no bank writes. + +New registered fableforge_skinnedcompose_tests passes stored/compressed single/ +multi-palette face/skin checks, raw donor+ghost equality, top-three quantization, +empty-root and huge finite weights, malformed spans/influences/geometry refusals, +multiple primitives, sentinel/name refusals, and the specific overflow caused +by palette duplication at 65,535 input vertices. Rebuilt static meshbounds and +headpose regressions pass; shared static writer changes only the default-zero +header animation parameter. Shipping applications not rebuilt or repackaged. + +Independent read-only addon oracle tools/test_skinned_composition.py gate passes +12 payloads in `skinned-composition-dtgr8apx`: compressed/stored authored triangles +against five native head skeletons, and synthetic multi-palette fans. Python +checks exact cloned donor and ghost spans, descriptor excluding ghost, positions/ +winding, <=16 palettes, block vertex ownership, per-corner bone mapping, joint +byte divisibility and <=3 nonzero weights summing to 255. Native graphics SHA256 +unchanged; no archive copies. First f63lgdws packet completed the C++ retail gate +but Python assertion referenced nonexistent 'sbc' key; corrected to sblocks. +Initial C++ harness compile failure was a lambda default argument referencing a +local; corrected test harness. No production validation weakened for either. +Evidence source paths explicitly supplied (--oracle-root), not checkout-assumed. + +This completes the byte composer, not the custom-body workflow or runtime proof. +Next: import authored joint names/weights and bind transforms (glTF), resolve to +the donor local skeleton without silently stripping skin, then owned graphics / +texture and private-definition publication with GUI selection and readback. +Optional-header volume/generator decoder gap remains as previously recorded; +do not claim all native donors supported. Existing validated packages unchanged. diff --git a/libs/forgecore/include/forge/meshcompose.hpp b/libs/forgecore/include/forge/meshcompose.hpp index 982879c4..0a064a8f 100644 --- a/libs/forgecore/include/forge/meshcompose.hpp +++ b/libs/forgecore/include/forge/meshcompose.hpp @@ -17,6 +17,8 @@ #include #include +namespace forge::meshpreview { struct SkeletonBlocks; } + namespace forge::meshcompose { // Current static writer capacity per primitive. @@ -56,6 +58,24 @@ struct Composed { size_t vertices = 0, triangles = 0; }; +struct Influence { uint16_t bone = 0; double weight = 0; }; // local donor slot, not global ID +struct SkinnedPrimitive { + Primitive geometry; + std::vector> skin; // exactly one list per vertex +}; +// New type-5 payload: float positions (stride 28), triangle lists and greedy +// <=16-bone palettes with shared vertices duplicated across animated blocks. +// Uses the top three positive influences, normalized to bytes summing to 255; +// an explicitly empty influence list binds to donor slot 0. Rejects nonfinite, +// negative, duplicate-slot or out-of-range influences and malformed donor spans. +// Donors have 1-256 local slots and a 4-byte to 1-MiB terminated name pool; +// smaller name blocks are outside the existing chunked-LZO readback contract. +// All five decoded donor skeleton spans survive exactly in LOD0 and ghost LOD. +// This composes bytes only; it does not publish banks or prove runtime compatibility. +Composed composeSkinned(const std::string& name,const std::vector& prims, + const std::vector& materials, + const meshpreview::SkeletonBlocks& skeleton,bool compress=true); + // `name` must start with "MESH_" (the payload classifier). Throws std::invalid_argument on bad input // (no prims / materials, > 65535 primitives or vertices in one primitive, a face index out of range, // or embedded NUL in a serialized mesh/material name). diff --git a/libs/forgecore/src/meshcompose.cpp b/libs/forgecore/src/meshcompose.cpp index 1b9e60eb..65ad48bc 100644 --- a/libs/forgecore/src/meshcompose.cpp +++ b/libs/forgecore/src/meshcompose.cpp @@ -9,6 +9,7 @@ #include "forge/lzo.hpp" #include "forge/texturewrite.hpp" +#include "forge/meshpreview.hpp" namespace forge::meshcompose { @@ -55,9 +56,9 @@ void writeMaterial(std::vector& out, int32_t id, const std::string& nam } } -void meshHeader(std::vector& out, const std::string& name, const Vec3& bsc, float bsr, const Vec3& bbMin, const Vec3& bbMax) { +void meshHeader(std::vector& out, const std::string& name, const Vec3& bsc, float bsr, const Vec3& bbMin, const Vec3& bbMax,bool animated=false) { putStr(out, name); - putU8(out, 0); // AnimatedFlag + putU8(out, animated?1:0); putF32(out, bsc.x); putF32(out, bsc.y); putF32(out, bsc.z); putF32(out, bsr); putF32(out, bbMin.x); putF32(out, bbMin.y); putF32(out, bbMin.z); @@ -240,6 +241,194 @@ Composed composeStatic(const std::string& name, const std::vector& pr namespace { +size_t validateSkeleton(const meshpreview::SkeletonBlocks& s) { + const size_t count=s.indices.size()/2; + if(s.indices.size()%2 || !count || count>256 || s.bones.size()!=60*count || + s.keyframes.size()!=48*count || s.inverseBinds.size()!=64*count || + s.names.size()<=3 || s.names.size()>1048576 || std::count(s.names.begin(),s.names.end(),uint8_t(0))!=count || + s.names.empty() || s.names.back()!=0) + throw std::invalid_argument("invalid donor skeleton spans"); + std::vector parents(count); + const auto finite=[](const std::vector& bytes,size_t offset,size_t floats) { + for(size_t i=0;i=int32_t(count)) throw std::invalid_argument("invalid donor bone parent"); + finite(s.bones,60*i+12,12);finite(s.inverseBinds,64*i,12); + finite(s.keyframes,48*i,7);finite(s.keyframes,48*i+32,3); // raw pad words are retained, not interpreted + } + for(size_t i=0;icount) throw std::invalid_argument("cyclic donor skeleton");p=parents[size_t(p)];} + } + return count; +} +void writeSkeleton(std::vector& out,const meshpreview::SkeletonBlocks& s,bool compress) { + out.insert(out.end(),s.indices.begin(),s.indices.end()); + for(const auto* span:{&s.names,&s.bones,&s.keyframes,&s.inverseBinds}) lzoBlock(out,*span,compress); +} +struct SkinBlock { + std::vector palette; + std::vector> faces; + size_t vertices=0; +}; +struct PreparedSkin { + Primitive geometry; + std::vector> skin; + std::vector blocks; +}; +void addBone(std::vector& palette,uint16_t bone) { + const auto slot=uint8_t(bone); + if(std::find(palette.begin(),palette.end(),slot)==palette.end()) palette.push_back(slot); +} +PreparedSkin prepareSkin(const SkinnedPrimitive& input,size_t boneCount,size_t materials) { + const auto& p=input.geometry; + if(p.verts.empty() || p.faces.empty() || p.verts.size()>maxPrimitiveVertices || + input.skin.size()!=p.verts.size() || p.faces.size()>meshpreview::maxPrimitiveIndices/3 || + p.material<0 || size_t(p.material)>=materials || + (!p.uvs.empty() && p.uvs.size()!=p.verts.size()) || + (!p.normals.empty() && p.normals.size()!=p.verts.size())) + throw std::invalid_argument("invalid skinned primitive counts or material"); + for(const auto& v:p.verts) if(!std::isfinite(v.x) || !std::isfinite(v.y) || !std::isfinite(v.z)) + throw std::invalid_argument("non-finite skinned position"); + for(const auto& n:p.normals) (void)packNormal(n); + for(const auto& uv:p.uvs) {(void)compressUv(uv.u);(void)compressUv(1.f-uv.v);} + for(const auto& f:p.faces) for(uint32_t index:f) if(index>=p.verts.size()) throw std::invalid_argument("skinned face index out of range"); + std::vector> skin=input.skin; + std::vector palette; + for(auto& influences:skin) { + std::vector used; + for(const auto& i:influences) { + if(i.bone>=boneCount || !std::isfinite(i.weight) || i.weight<0 || + std::find(used.begin(),used.end(),i.bone)!=used.end()) throw std::invalid_argument("invalid skin influence"); + used.push_back(i.bone); + } + std::erase_if(influences,[](const auto& i){return i.weight==0;}); + std::stable_sort(influences.begin(),influences.end(),[](const auto& a,const auto& b){return a.weight>b.weight;}); + if(influences.size()>3) influences.resize(3); + if(influences.empty()) influences.push_back({0,1}); + // Divide by the largest first to avoid overflowing sums of finite weights. + const double largest=influences.front().weight;double sum=0; + for(auto& i:influences) {i.weight/=largest;sum+=i.weight;} + for(auto& i:influences) {i.weight/=sum;addBone(palette,i.bone);} + } + PreparedSkin out; + const auto normals=p.normals.empty()?faceNormals(p.verts,p.faces):p.normals; + if(palette.size()<=16) { + out.geometry=p;out.geometry.normals=normals;out.skin=std::move(skin); + out.blocks.push_back({palette,p.faces,p.verts.size()});return out; + } + std::vector groups(1); + for(const auto& face:p.faces) { + std::vector needed; + for(auto index:face) for(const auto& i:skin[index]) addBone(needed,i.bone); + auto expanded=groups.back().palette; + for(auto bone:needed) addBone(expanded,bone); + if(expanded.size()>16 && !groups.back().faces.empty()) groups.emplace_back(SkinBlock{needed,{},{}}); + else groups.back().palette=std::move(expanded); + groups.back().faces.push_back(face); + } + out.geometry.material=p.material; + for(auto& group:groups) { + std::map remap; + for(auto& face:group.faces) for(auto& index:face) { + auto found=remap.find(index); + if(found==remap.end()) { + if(out.geometry.verts.size()>=maxPrimitiveVertices) throw std::invalid_argument("skinned vertex overflow after palette partition"); + const auto next=uint32_t(out.geometry.verts.size());remap.emplace(index,next); + out.geometry.verts.push_back(p.verts[index]);out.geometry.normals.push_back(normals[index]); + out.geometry.uvs.push_back(p.uvs.empty()?Vec2{}:p.uvs[index]);out.skin.push_back(skin[index]);index=next; + }else index=found->second; + } + group.vertices=remap.size(); + out.geometry.faces.insert(out.geometry.faces.end(),group.faces.begin(),group.faces.end());out.blocks.push_back(std::move(group)); + } + return out; +} +void skinBytes(std::vector& out,const std::vector& skin,const std::vector& palette) { + std::array joints{};std::array weights{};int total=0; + for(size_t k=0;k& prims, + const std::vector& materials,const meshpreview::SkeletonBlocks& skeleton,bool compress) { + const size_t boneCount=validateSkeleton(skeleton); + if(name.rfind("MESH_",0)!=0 || prims.empty() || prims.size()>10000 || materials.empty() || materials.size()>10000) + throw std::invalid_argument("invalid skinned mesh name, primitive or material count"); + std::vector prepared;size_t blockCount=0; + for(const auto& p:prims) { + prepared.push_back(prepareSkin(p,boneCount,materials.size()));blockCount+=prepared.back().blocks.size(); + if(blockCount>65535) throw std::invalid_argument("skinned animated block overflow"); + } + for(const auto& m:materials) if(m.nativeFlags && (*m.nativeFlags)[3]) throw std::invalid_argument("skinned materials cannot contain a degenerate sentinel"); + std::vector all;for(const auto& p:prepared) all.push_back(&p.geometry); + Composed result;bounds(all,result.bbMin,result.bbMax,result.sphereCentre,result.sphereRadius); + const auto header=[&](std::vector& out,bool ghost) { + meshHeader(out,name,result.sphereCentre,result.sphereRadius,result.bbMin,result.bbMax,true); + putI32(out,ghost?0:int32_t(materials.size()));putI32(out,ghost?0:int32_t(prims.size())); + putI32(out,int32_t(boneCount));putI32(out,int32_t(skeleton.names.size())); + putU8(out,0);putU16(out,0);putU16(out,ghost?0:uint16_t(blockCount));writeSkeleton(out,skeleton,compress);identityRoot(out); + }; + auto& lod=result.payload;header(lod,false); + for(size_t slot=0;slot vb;vb.reserve(p.verts.size()*28);size_t block=0,within=0; + for(size_t i=0;i=blocks[block].vertices && block+1 ib;ib.reserve(p.faces.size()*6); + for(const auto& f:p.faces) {putU16(ib,uint16_t(f[0]));putU16(ib,uint16_t(f[2]));putU16(ib,uint16_t(f[1]));} + lzoBlock(lod,ib,compress);putU32(lod,0);result.vertices+=p.verts.size();result.triangles+=p.faces.size(); + } + const size_t lodSize=lod.size();if(lodSize>UINT32_MAX) throw std::invalid_argument("skinned LOD size overflow"); + header(lod,true); + auto& info=result.info;putU32(info,0); + for(float f:{result.sphereCentre.x,result.sphereCentre.y,result.sphereCentre.z,result.sphereRadius, + result.bbMin.x,result.bbMin.y,result.bbMin.z,result.bbMax.x,result.bbMax.y,result.bbMax.z}) putF32(info,f); + putU32(info,1);putU32(info,uint32_t(lodSize));putF32(info,0); + std::vector textures; + for(const auto& m:materials) for(uint32_t id:{m.diffuseId,m.bumpId,m.reflectId,m.illumId,m.decalId>0?uint32_t(m.decalId):0u}) + if(id && std::find(textures.begin(),textures.end(),id)==textures.end()) textures.push_back(id); + putU32(info,uint32_t(textures.size()));for(auto id:textures) putU32(info,id); + return result; +} + +namespace { + // EgoCore's ChunkWriter: TAG, u32 size placeholder, body, size patched on pop struct ChunkWriter { std::vector buf; diff --git a/tests/test_skinnedcompose.cpp b/tests/test_skinnedcompose.cpp new file mode 100644 index 00000000..19bbd648 --- /dev/null +++ b/tests/test_skinnedcompose.cpp @@ -0,0 +1,142 @@ +#include "forge/meshcompose.hpp" +#include "forge/meshpreview.hpp" +#include "forge/big.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mc=forge::meshcompose; +namespace mp=forge::meshpreview; +static void require(bool value,const char* message) {if(!value) throw std::runtime_error(message);} +static void word(std::vector& out,uint32_t value) {for(unsigned i=0;i<4;++i) out.push_back(uint8_t(value>>(8*i)));} +static void real(std::vector& out,float value) {word(out,std::bit_cast(value));} +static mp::SkeletonBlocks donor(size_t count=32) { + mp::SkeletonBlocks s; + for(size_t i=0;i=8 && k<=10)?1.f:0.f);} + for(size_t k=0;k<16;++k) {if(k>=12) word(s.inverseBinds,0x7fc05678u);else real(s.inverseBinds,k==0 || k==5 || k==10?1.f:0.f);} + } + return s; +} +static bool same(const mp::SkeletonBlocks& a,const mp::SkeletonBlocks& b) { + return a.indices==b.indices && a.names==b.names && a.bones==b.bones && a.keyframes==b.keyframes && a.inverseBinds==b.inverseBinds; +} +static mc::SkinnedPrimitive triangle() { + mc::SkinnedPrimitive p;p.geometry.verts={{0,0,0},{1,0,0},{0,1,0}};p.geometry.faces={{0,1,2}}; + p.skin={{{0,5},{1,3},{2,2},{3,1}},{{2,1}},{{1,1}}};return p; +} +static mc::SkinnedPrimitive fan() { + mc::SkinnedPrimitive p;p.geometry.verts.push_back({0,0,0});p.skin.push_back({{0,1}}); + for(uint16_t i=1;i<=20;++i) { + const float angle=float(i)*.2f;p.geometry.verts.push_back({std::cos(angle),std::sin(angle),float(i)/20});p.skin.push_back({{i,1}}); + if(i>1) p.geometry.faces.push_back({0,uint32_t(i-1),i}); + } + return p; +} +static void checkMesh(const mc::Composed& mesh,const mc::SkinnedPrimitive& source,const mp::SkeletonBlocks& skeleton,bool multi) { + const auto g=mp::decodeLod0(mesh.payload,5); + require(g.skeletonBlocks && same(*g.skeletonBlocks,skeleton),"donor spans changed in LOD0"); + require(g.materials.size()==1 && g.materials[0].diffuseTexture==123,"skinned material/sentinel mismatch"); + require(g.triangles.size()==source.geometry.faces.size(),"triangle count mismatch"); + require(multi?g.animatedBlockCount>1:g.animatedBlockCount==1,"animated block partition mismatch"); + if(multi) require(g.vertices.size()>source.geometry.verts.size(),"shared palette-boundary vertices not duplicated"); + for(size_t i=0;i actual={t.a,t.b,t.c}; + for(size_t k=0;k<3;++k) { + const auto index=source.geometry.faces[i][k];const auto& want=source.geometry.verts[index];const auto& v=g.vertices.at(actual[k]); + require(v.x==want.x && v.y==want.y && v.z==want.z && v.skinned,"face position/winding/skin changed"); + float sum=0;for(float w:v.weights) sum+=w;require(std::abs(sum-1)<1e-6,"weights do not sum to one"); + if(multi) require(v.joints[0]==source.skin[index][0].bone && v.weights[0]==1,"partition changed per-corner bone"); + } + } + if(!multi) { + const auto& v=g.vertices[0];require(v.joints[0]==0 && v.joints[1]==1 && v.joints[2]==2 && v.weights[3]==0,"top-three influences changed"); + for(size_t k=0;k<3;++k) require(std::abs(v.weights[k]-float(source.skin[0][k].weight/10))<=1.f/255,"weight quantization changed"); + } + uint32_t size=0;std::memcpy(&size,mesh.info.data()+48,4);require(size ghost(mesh.payload.begin()+size,mesh.payload.end());const auto ghostGeometry=mp::decodeLod0(ghost,5); + require(ghostGeometry.vertices.empty() && ghostGeometry.skeletonBlocks && same(*ghostGeometry.skeletonBlocks,skeleton),"ghost skeleton changed"); + require(mp::decodeLods(mesh.payload,mesh.info,5).size()==1,"descriptor included ghost LOD"); +} +template static void refuses(F action) { + bool refused=false;try {action();}catch(const std::invalid_argument&) {refused=true;}require(refused,"invalid authored mesh accepted"); +} +static void write(const std::filesystem::path& path,const std::vector& bytes) { + std::ofstream out(path,std::ios::binary);out.write(reinterpret_cast(bytes.data()),std::streamsize(bytes.size())); + require(bool(out),"cannot write isolated composed fixture"); +} +int main(int argc,char** argv) { + try { + const auto skeleton=donor();mc::Material material;material.diffuseId=123; + for(bool compress:{false,true}) { + checkMesh(mc::composeSkinned("MESH_TRIANGLE",{triangle()},{material},skeleton,compress),triangle(),skeleton,false); + checkMesh(mc::composeSkinned("MESH_FAN",{fan()},{material},skeleton,compress),fan(),skeleton,true); + } + const auto invoke=[&](const mc::SkinnedPrimitive& p,const std::optional& s=std::nullopt) {return mc::composeSkinned("MESH_BAD",{p},{material},s?*s:skeleton);}; + auto p=triangle();p.skin.pop_back();refuses([&]{invoke(p);}); + p=triangle();p.skin[0][0].bone=32;refuses([&]{invoke(p);}); + p=triangle();p.skin[0][0].weight=-1;refuses([&]{invoke(p);}); + p=triangle();p.skin[0][0].weight=std::numeric_limits::quiet_NaN();refuses([&]{invoke(p);}); + p=triangle();p.skin[0][0].weight=std::numeric_limits::infinity();refuses([&]{invoke(p);}); + p=triangle();p.skin[0][1].bone=0;refuses([&]{invoke(p);}); + p=triangle();p.geometry.faces[0][0]=3;refuses([&]{invoke(p);}); + p=triangle();p.geometry.material=1;refuses([&]{invoke(p);}); + p=triangle();p.geometry.uvs={{0,0}};refuses([&]{invoke(p);}); + p=triangle();p.geometry.normals={{0,0,1}};refuses([&]{invoke(p);}); + p=triangle();p.geometry.verts[0].x=std::numeric_limits::infinity();refuses([&]{invoke(p);}); + auto s=skeleton;s.indices.pop_back();refuses([&]{invoke(triangle(),s);}); + s=skeleton;s.bones.pop_back();refuses([&]{invoke(triangle(),s);}); + s=skeleton;s.keyframes.pop_back();refuses([&]{invoke(triangle(),s);}); + s=skeleton;s.inverseBinds.pop_back();refuses([&]{invoke(triangle(),s);}); + s=skeleton;s.names.back()='x';refuses([&]{invoke(triangle(),s);}); + s=skeleton;std::fill(s.bones.begin()+4,s.bones.begin()+8,0);refuses([&]{invoke(triangle(),s);}); // self cycle + s=skeleton;const uint32_t nan=0x7fc00000;std::memcpy(s.inverseBinds.data(),&nan,4);refuses([&]{invoke(triangle(),s);}); + p=triangle();p.skin[0].clear();const auto root=mp::decodeLod0(invoke(p).payload,5);require(root.vertices[0].joints[0]==0 && root.vertices[0].weights[0]==1,"empty authored list did not bind root"); + p=triangle();p.skin[0]={{0,std::numeric_limits::max()},{1,std::numeric_limits::max()}}; + const auto huge=mp::decodeLod0(invoke(p).payload,5);require(huge.vertices[0].weights[0]+huge.vertices[0].weights[1]==1,"finite large weights overflowed"); + mc::SkinnedPrimitive overflow; + for(uint32_t i=0;i1) overflow.geometry.faces.push_back({0,i-1,i}); + } + bool partitionOverflow=false; + try {invoke(overflow);}catch(const std::invalid_argument& e) {partitionOverflow=std::string(e.what()).find("after palette partition")!=std::string::npos;} + require(partitionOverflow,"palette duplication overflow not specifically refused"); + const auto two=mc::composeSkinned("MESH_TWO",{triangle(),fan()},{material},skeleton); + const auto twoGeometry=mp::decodeLod0(two.payload,5); + require(twoGeometry.primitiveCount==2 && twoGeometry.triangles.size()==20 && twoGeometry.vertices.size()==two.vertices, + "multiple skinned primitives lost geometry"); + auto badMaterial=material;badMaterial.nativeFlags=std::array{0,0,0,1,0}; + refuses([&]{mc::composeSkinned("MESH_BAD",{triangle()},{badMaterial},skeleton);}); + refuses([&]{mc::composeSkinned("NOT_A_MESH",{triangle()},{material},skeleton);}); + refuses([&]{mc::composeSkinned(std::string("MESH_\0BAD",9),{triangle()},{material},skeleton);}); + if(argc==4 && std::string(argv[1])=="--retail") { + const auto archive=forge::big::File::open(argv[2]);const auto* bank=archive.findBank("MBANK_ALLMESHES");require(bank,"missing meshes bank"); + const auto output=std::filesystem::absolute(argv[3]);require(!std::filesystem::exists(output),"retail output must be fresh"); + std::filesystem::create_directories(output); + for(uint32_t id:{4521u,5121u,5150u,3985u,5110u}) { + const auto e=std::find_if(bank->entries.begin(),bank->entries.end(),[&](const auto& e){return e.id==id;});require(e!=bank->entries.end() && e->type==5,"missing donor"); + const auto geometry=mp::decodeLod0(archive.entryData(*e),5);require(bool(geometry.skeletonBlocks),"missing donor spans"); + for(bool compressed:{false,true}) { + const auto stem=std::to_string(id)+(compressed?"-compressed":"-stored"); + const auto mesh=mc::composeSkinned("MESH_CUSTOM_"+std::to_string(id),{triangle()},{material},*geometry.skeletonBlocks,compressed); + checkMesh(mesh,triangle(),*geometry.skeletonBlocks,false);write(output/(stem+".mesh"),mesh.payload);write(output/(stem+".info"),mesh.info); + } + } + for(bool compressed:{false,true}) {const auto mesh=mc::composeSkinned("MESH_CUSTOM_FAN",{fan()},{material},skeleton,compressed); + const auto stem=compressed?"fan-compressed":"fan-stored";write(output/(std::string(stem)+".mesh"),mesh.payload);write(output/(std::string(stem)+".info"),mesh.info);} + }else require(argc==1,"expected --retail graphics.big fresh-output-directory"); + std::cout<<"skinned composition: compressed/stored single/multi palettes, raw skeleton/ghost, authored weights and refusal gates PASS\n"; + return 0; + }catch(const std::exception& e) {std::cerr<<"skinned composition: "<0 + assert all(len(ab[4])<=16 for ab in p['ablocks']) + assert sum(ab[3] for ab in p['ablocks'])==len(vertices) + for i in range(len(vertices)): + joints=p['vb'][28*i+12:28*i+16];weights=p['vb'][28*i+16:28*i+20] + assert sum(weights)==255 and sum(w>0 for w in weights)<=3 + assert all(joint%3==0 for joint in joints) + if path.stem.startswith('fan-'): + assert len(vertices)>21 and len(faces)==19 and p['abc']>=2 + for face_index,face in enumerate(faces): + for k,index in enumerate(face): + expected_bone=(0,face_index+1,face_index+2)[k] + assert skin[index]==[(expected_bone,1.0)],'face corner palette remap changed' + if expected_bone==0:assert vertices[index]==(0.0,0.0,0.0) + else:assert abs(vertices[index][2]-expected_bone/20)<1e-6 + else: + donor_id=int(path.stem.split('-')[0]);entry=next(e for e in entries if e['id']==donor_id) + source=mesh_rw.clone_skeleton(buf[entry['offset']:entry['offset']+entry['size']],5) + assert all(skeleton[k]==source[k] for k in keys),'raw donor changed' + assert vertices==[(0.0,0.0,0.0),(1.0,0.0,0.0),(0.0,1.0,0.0)] and faces==[(0,1,2)] + assert dict(skin[1])=={2:1.0} and dict(skin[2])=={1:1.0} + assert set(dict(skin[0]))=={0,1,2} + assert all(abs(dict(skin[0])[i]-w)<=1/255 for i,w in enumerate((.5,.3,.2))) + checks.append(dict(payload=path.name,bones=skeleton['count'],vertices=len(vertices),faces=len(faces),blocks=p['abc'],sha256=digest(path))) + print(path.name,'independent skeleton, geometry, skin and ghost: PASS',flush=True) + assert len(checks)==12 and digest(graphics)==before,'missing cases or native graphics changed' + (work/'result.json').write_text(json.dumps(dict(passed=True,graphics_sha256=before,checks=checks),indent=2)+'\n',encoding='utf-8') + + +if __name__=='__main__':main() From 373a892bd2285beb450ce58f46e9948fc9ef4469 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:00:47 -0600 Subject: [PATCH 38/75] feat(mesh): retain authored glTF skin and bind metadata --- CMakeLists.txt | 3 + docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 45 ++++++ src/meshimport.cpp | 160 ++++++++++++++++++---- src/meshimport.hpp | 20 +++ tests/test_skinnedinput.cpp | 120 ++++++++++++++++ 5 files changed, 325 insertions(+), 23 deletions(-) create mode 100644 tests/test_skinnedinput.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e38477e0..18d3e879 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -161,6 +161,9 @@ add_test(NAME fableforge_tests COMMAND fableforge_tests) add_executable(fableforge_meshinput_tests tests/test_meshinput.cpp) target_link_libraries(fableforge_meshinput_tests PRIVATE fableforge) add_test(NAME fableforge_meshinput_tests COMMAND fableforge_meshinput_tests) +add_executable(fableforge_skinnedinput_tests tests/test_skinnedinput.cpp) +target_link_libraries(fableforge_skinnedinput_tests PRIVATE fableforge) +add_test(NAME fableforge_skinnedinput_tests COMMAND fableforge_skinnedinput_tests) add_executable(fableforge_stage_tests tests/test_stage.cpp) target_link_libraries(fableforge_stage_tests PRIVATE forgecore_slice) add_test(NAME fableforge_stage_tests COMMAND fableforge_stage_tests) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index add67bf3..f3886c2a 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -908,3 +908,48 @@ the donor local skeleton without silently stripping skin, then owned graphics / texture and private-definition publication with GUI selection and readback. Optional-header volume/generator decoder gap remains as previously recorded; do not claim all native donors supported. Existing validated packages unchanged. + +Authored glTF skin input now has a distinct read-only loadSkinnedModel API, +reusing the existing GLB/sidecar buffers, topology and sparse accessor reader. +Model.skins parallels geometry primitives and retains source skin/mesh-node IDs, +joint-array order, source-node/parent identity, names, default scene-world and +inverse-bind matrices in original glTF metres/column-major form. Geometry is +converted to Fable coordinates without baking the skinned mesh node transform. +Khronos glTF 2.0 sections 3.7.3/3.7.2 explicitly require this distinction from the +static importer (https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html#skins). +Static loadModel still refuses selected active skins; uninstantiated skin +declarations keep prior static behavior. No caller is switched to the new loader. + +Reads paired consecutive JOINTS_n/WEIGHTS_n sets, VEC4/count-matched byte/ushort +joints and float or normalized byte/ushort weights. Retains every positive +influence, including higher sets; notes identify vertices exceeding the +composer's strongest-three capacity. Missing/out-of-scene/repeated joint nodes, +missing common/declared root, invalid matrices, attribute encodings/indices, +negative/nonfinite/duplicate positive weights and invalid total weights refuse. +Integer totals checked exactly in a common 65535 denominator; float tolerance +1e-5. Undefined IBMs become identity. Meaningful source matrices are retained, +not claimed to match donor bind poses. Source joint cap 65536; IBM accessor cap +1048576 matrices. Selected mesh instances currently must all be skinned; mixed +rigid attachments require intentional handling, never silent stripping. Active +morphs remain refused, animation tracks not imported (explicit note). Shared UV +clipping notes now apply to both readers, with static messages unchanged. + +New registered fableforge_skinnedinput_tests passes 36 checks: six joint/weight +encoding combinations, ignored mesh-node pose/winding, exact joint order and +world/IBM translation, default IBM identity, malformed scenes/spans/encodings/ +weights, static boundary, all-positive higher-set preservation/reduction note, +attribute-index/duplicate guards, repeated mesh instances, sparse joints, +quantized total rejection, IBM stride rejection and sidecar glTF. Existing +meshinput and skinnedcompose CTests pass against rebuilt libraries (three focused +tests total). Initial test fixture compilation had one missing JSON brace; +corrected fixture, no production validation change. Temporary fixtures tiny; +test-created directory validated within the resolved temp root and removed on +success. No retail input changes, full asset copies, shipping GUI rebuild or new +archive. No full glTF conformance or runtime compatibility claim. + +Next: resolve used source joint names uniquely to donor local slots and validate +coordinate-converted inverse binds/hierarchy before feeding the composer. Then +owned graphics/texture and private definition publication, creature import card +and placement/runtime evidence. Reading metadata is not compatibility proof; +the complete custom-body workflow remains unfinished. Existing checkpoints stay +as previously validated. diff --git a/src/meshimport.cpp b/src/meshimport.cpp index 86fb7298..689c5f34 100644 --- a/src/meshimport.cpp +++ b/src/meshimport.cpp @@ -298,7 +298,7 @@ template std::vector readAccessor(const GltfBuffers& g, size_t index, int& comps) { const auto& acc=g.doc.at("accessors").at(index); const std::string type=acc.at("type").get(); - comps=type=="SCALAR" ? 1 : type=="VEC2" ? 2 : type=="VEC3" ? 3 : type=="VEC4" ? 4 : 0; + comps=type=="SCALAR" ? 1 : type=="VEC2" ? 2 : type=="VEC3" ? 3 : type=="VEC4" ? 4 : type=="MAT4" ? 16 : 0; if(!comps) throw std::runtime_error("unsupported accessor type "+type); const size_t count=gltfSize(acc,"count"),ct=gltfInteger(acc.at("componentType"),"componentType"); const bool norm=acc.value("normalized",false); @@ -414,9 +414,94 @@ struct NormalTransform { } }; -Model loadGltf(const fs::path& path) { +SkinBinding readSkin(const GltfBuffers& g,size_t skinIndex,size_t meshNode, + const std::vector>& worlds, + const std::vector>& parents) { + const auto& skin=g.doc.at("skins").at(skinIndex);const auto& nodes=g.doc.at("nodes"); + const auto& joints=skin.at("joints"); + if(!joints.is_array() || joints.empty() || joints.size()>65536) throw std::runtime_error("invalid glTF skin joint count"); + SkinBinding binding;binding.sourceSkin=skinIndex;binding.meshNode=meshNode; + std::vector used(worlds.size(),false);std::optional commonRoot; + if(skin.contains("skeleton")) binding.skeletonRoot=gltfSize(skin,"skeleton"); + for(const auto& joint:joints) { + const size_t ni=gltfInteger(joint,"joint node"); + if(ni>=worlds.size() || !worlds[ni] || used[ni]) + throw std::runtime_error("skin joint is missing, repeated or outside the selected scene"); + used[ni]=true;size_t root=ni;bool foundSkeleton=!binding.skeletonRoot; + for(std::optional ancestor=ni;ancestor;ancestor=parents[*ancestor]) { + root=*ancestor;if(binding.skeletonRoot==ancestor) foundSkeleton=true; + } + if(!foundSkeleton || (commonRoot && *commonRoot!=root)) throw std::runtime_error("skin joints lack the declared/common scene root"); + commonRoot=root; + binding.joints.push_back({ni,parents[ni],nodes.at(ni).value("name",std::string()),*worlds[ni],identity()}); + } + if(skin.contains("inverseBindMatrices")) { + const size_t index=gltfSize(skin,"inverseBindMatrices");const auto& acc=g.doc.at("accessors").at(index); + if(acc.at("type")!="MAT4" || gltfSize(acc,"componentType")!=5126 || acc.value("normalized",false) || + gltfSize(acc,"count")1048576) + throw std::runtime_error("invalid glTF inverse-bind accessor"); + if(acc.contains("bufferView") && g.doc.at("bufferViews").at(gltfSize(acc,"bufferView")).contains("byteStride")) + throw std::runtime_error("inverse-bind buffer view must not define byteStride"); + int comps=0;const auto values=readAccessor(g,index,comps); + for(size_t i=0;i joints,weights;size_t weightType;}; + std::map sets; + for(const auto& [name,index]:attributes.items()) if(name.starts_with("JOINTS_") || name.starts_with("WEIGHTS_")) { + const size_t prefix=name.starts_with("JOINTS_")?7:8;const std::string_view suffix(name.data()+prefix,name.size()-prefix); + size_t set=0;const auto parsed=std::from_chars(suffix.data(),suffix.data()+suffix.size(),set); + if(suffix.empty() || parsed.ec!=std::errc{} || parsed.ptr!=suffix.data()+suffix.size() || std::to_string(set)!=suffix) + throw std::runtime_error("invalid glTF skin attribute set index"); + sets.try_emplace(set); + } + if(sets.empty() || !sets.contains(0)) throw std::runtime_error("skinned primitive requires JOINTS_0 and WEIGHTS_0"); + size_t next=0; + for(auto& [set,data]:sets) { + if(set!=next++) throw std::runtime_error("glTF skin attribute sets must be consecutive"); + for(bool joints:{true,false}) { + const auto name=std::string(joints?"JOINTS_":"WEIGHTS_")+std::to_string(set); + if(!attributes.contains(name)) throw std::runtime_error("unpaired glTF skin attribute set"); + const size_t index=gltfSize(attributes,name.c_str());const auto& acc=g.doc.at("accessors").at(index); + const size_t component=gltfSize(acc,"componentType");const bool norm=acc.value("normalized",false); + if(acc.at("type")!="VEC4" || gltfSize(acc,"count")!=count || + (joints? (norm || (component!=5121 && component!=5123)): + !((component==5126 && !norm) || ((component==5121 || component==5123) && norm)))) + throw std::runtime_error("invalid glTF joint/weight accessor shape or encoding"); + int comps=0;auto values=readAccessor(g,index,comps); + if(joints) data.joints=std::move(values);else {data.weights=std::move(values);data.weightType=component;} + } + } + binding.influences.resize(count); + for(size_t v=0;v=binding.joints.size() || weight<0 || weight>1) throw std::runtime_error("glTF joint index or weight is out of range"); + total+=weight; + if(data.weightType==5126) integerOnly=false; + else exactTotal+=uint64_t(std::lround(double(weight)*(data.weightType==5121?255:65535)))*(data.weightType==5121?257:1); + if(weight==0) continue; + auto& influences=binding.influences[v]; + if(std::any_of(influences.begin(),influences.end(),[&](const auto& i){return i.bone==ordinal;})) + throw std::runtime_error("joint has repeated positive weights for one vertex"); + influences.push_back({uint16_t(ordinal),weight}); + } + if(integerOnly?exactTotal!=65535:std::abs(total-1)>1e-5) throw std::runtime_error("glTF vertex weights must sum to one"); + } +} + +Model loadGltf(const fs::path& path,bool skinned=false) { Model model; const GltfBuffers g = openGltf(path); + if(skinned && g.doc.contains("animations") && !g.doc.at("animations").empty()) + model.notes.push_back("glTF animation tracks are not imported; skin metadata retains the default joint transforms and inverse binds"); for (const auto& extension : g.doc.value("extensionsUsed", json::array())) model.notes.push_back("ignored optional glTF extension: " + extension.get()); if (!g.doc.contains("meshes") || !g.doc.at("meshes").is_array() || g.doc.at("meshes").empty()) @@ -442,43 +527,53 @@ Model loadGltf(const fs::path& path) { }; // Traverse the selected scene iteratively: malformed cycles and deep valid // hierarchies must not exhaust the process stack. Preserve depth-first order. - struct MeshInstance {size_t mesh;Mat4 world;json weights;}; + struct MeshInstance {size_t mesh;Mat4 world;json weights;std::optional skin;size_t node;}; std::vector instances; + const auto nodes=g.doc.value("nodes",json::array()); + if(!nodes.is_array()) throw std::runtime_error("nodes is not an array"); + std::vector> worlds(nodes.size()); + std::vector> parents(nodes.size()); if (g.doc.contains("scenes") && !g.doc["scenes"].empty()) { const size_t sceneIndex = gltfSize(g.doc, "scene"); if (!g.doc["scenes"].is_array() || sceneIndex >= g.doc["scenes"].size()) throw std::runtime_error("invalid default scene index"); const auto& scene = g.doc["scenes"][sceneIndex]; - const auto nodes = g.doc.value("nodes", json::array()); - if (!nodes.is_array()) throw std::runtime_error("nodes is not an array"); std::vector visited(nodes.size(), false); - std::vector> pending; - auto enqueue = [&](const json& children, const Mat4& parent) { + struct Pending {size_t node;Mat4 world;std::optional parent;}; + std::vector pending; + auto enqueue = [&](const json& children, const Mat4& parent,std::optional parentNode) { if (!children.is_array()) throw std::runtime_error("node list is not an array"); for (auto it = children.rbegin(); it != children.rend(); ++it) - pending.emplace_back(gltfInteger(*it, "node index"), parent); + pending.push_back({gltfInteger(*it, "node index"), parent,parentNode}); }; - enqueue(scene.value("nodes", json::array()), identity()); + enqueue(scene.value("nodes", json::array()), identity(),std::nullopt); while (!pending.empty()) { - const auto [ni, parent] = pending.back(); pending.pop_back(); + const auto [ni, parent,parentNode] = pending.back(); pending.pop_back(); if (ni >= nodes.size()) throw std::runtime_error("scene references a missing node"); const auto& n = nodes[ni]; if (visited[ni]) throw std::runtime_error("scene contains a cycle or repeated node"); visited[ni] = true; - if(n.contains("mesh") && n.contains("skin")) + if(!skinned && n.contains("mesh") && n.contains("skin")) throw std::runtime_error("active glTF skin is unsupported; bake the posed shape into a static mesh before export"); const Mat4 world = mul(parent, nodeMatrix(n)); if (!std::all_of(world.begin(), world.end(), [](float v) { return std::isfinite(v); })) throw std::runtime_error("node world transform is not finite"); - if (n.contains("mesh")) instances.push_back({gltfSize(n, "mesh"),world,n.value("weights",json{})}); - enqueue(n.value("children", json::array()), world); + worlds[ni]=world;parents[ni]=parentNode; + if (n.contains("mesh")) { + if(skinned && !n.contains("skin")) throw std::runtime_error("skinned import requires skin on every selected mesh instance"); + instances.push_back({gltfSize(n, "mesh"),skinned?identity():world,n.value("weights",json{}), + n.contains("skin")?std::optional(gltfSize(n,"skin")):std::nullopt,ni}); + } + enqueue(n.value("children", json::array()), world,ni); } } else { + if(skinned) throw std::runtime_error("skinned glTF requires a selected scene and joint hierarchy"); // With no scenes, import the asset as a mesh library. - for (size_t mi = 0; mi < g.doc["meshes"].size(); ++mi) instances.push_back({mi,identity(),json{}}); + for (size_t mi = 0; mi < g.doc["meshes"].size(); ++mi) instances.push_back({mi,identity(),json{},std::nullopt,0}); } - for (const auto& [mi, world, nodeWeights] : instances) { + for (const auto& [mi, world, nodeWeights,skinIndex,meshNode] : instances) { + const auto binding=skinned?std::optional(readSkin(g,*skinIndex,meshNode,worlds,parents)):std::nullopt; const NormalTransform normalTransform(world); const auto& mesh = g.doc["meshes"].at(size_t(mi)); if (!mesh.is_object() || !mesh.contains("primitives") || !mesh.at("primitives").is_array() || mesh.at("primitives").empty()) @@ -584,7 +679,10 @@ Model loadGltf(const fs::path& path) { } // a mirrored node transform flips the winding if (normalTransform.determinant < 0) for (auto& f : p.faces) std::swap(f[1], f[2]); - if (!p.faces.empty()) model.prims.push_back(std::move(p)); + if (!p.faces.empty()) { + if(binding) {auto skin=*binding;readSkinAttributes(g,attrs,nv,skin);model.skins.push_back(std::move(skin));} + model.prims.push_back(std::move(p)); + } } catch (const std::exception& error) { throw std::runtime_error("glTF mesh " + std::to_string(mi) + " primitive " + std::to_string(pi) + ": " + error.what()); } @@ -806,13 +904,7 @@ bool validName(const std::string& n) { } // namespace -Model loadModel(const fs::path& path) { - std::string ext = path.extension().string(); - std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return char(std::tolower(c)); }); - Model model; - if (ext == ".glb" || ext == ".gltf") model=loadGltf(path); - else if (ext == ".obj") model=loadObj(path); - else throw std::runtime_error("unsupported model format " + ext + " (.glb, .gltf or .obj)"); +static void appendUvNotes(Model& model) { for (size_t part=0;part3) ++reduced; + if(reduced) model.notes.push_back("model part "+std::to_string(part+1)+": "+std::to_string(reduced)+" vertices have more than three influences; Fable composition keeps the strongest three"); + } return model; } diff --git a/src/meshimport.hpp b/src/meshimport.hpp index 4393cfa2..1843dc4b 100644 --- a/src/meshimport.hpp +++ b/src/meshimport.hpp @@ -16,15 +16,30 @@ #include #include #include +#include #include "forge/meshcompose.hpp" namespace albion::meshimport { +struct SkinJoint { + size_t sourceNode=0; + std::optional parentNode; + std::string name; + // Original glTF coordinates/metres, column-major. Not donor transforms. + std::array world{},inverseBind{}; +}; +struct SkinBinding { + size_t sourceSkin=0,meshNode=0; + std::optional skeletonRoot; + std::vector joints; // skin.joints order; influence indices refer here + std::vector> influences; +}; struct Model { std::vector prims; // one per glTF primitive / OBJ material group, in Fable space std::vector materialNames; // per slot std::vector notes; + std::vector skins; // loadSkinnedModel only, parallel to prims }; // .glb / .gltf (embedded or sidecar buffers; POSITION, NORMAL, TEXCOORD_0, indices; @@ -36,6 +51,11 @@ struct Model { // Notes identify texture coordinates that will clip in the static writer. Model loadModel(const std::filesystem::path& path); +// Selected-scene glTF/GLB skinned instances only. Retains every positive weight +// set and joint/bind metadata for subsequent donor compatibility validation. +// Does not apply pose or skinned mesh-node transforms to the authored vertices, +// map joints to a donor, compose bytes or publish banks. Static loader unchanged. +Model loadSkinnedModel(const std::filesystem::path& path); struct ImportRequest { std::filesystem::path model; diff --git a/tests/test_skinnedinput.cpp b/tests/test_skinnedinput.cpp new file mode 100644 index 00000000..b410c479 --- /dev/null +++ b/tests/test_skinnedinput.cpp @@ -0,0 +1,120 @@ +#include "meshimport.hpp" +#include "nlohmann/json.hpp" +#include +#include +#include +#include +#include +#include +#include + +using json=nlohmann::json; +using Bytes=std::vector; +namespace fs=std::filesystem; +static void require(bool condition,const char* message) {if(!condition) throw std::runtime_error(message);} +static void word(Bytes& b,uint32_t v) {for(unsigned i=0;i<4;++i) b.push_back(uint8_t(v>>(8*i)));} +static void real(Bytes& b,float v) {word(b,std::bit_cast(v));} +static void setReal(Bytes& b,size_t at,float v) {const auto bits=std::bit_cast(v);for(unsigned i=0;i<4;++i) b.at(at+i)=uint8_t(bits>>(8*i));} +static void chunk(Bytes& b,uint32_t type,Bytes data) {while(data.size()%4) data.push_back(type==0x4e4f534a?' ':0);word(b,uint32_t(data.size()));word(b,type);b.insert(b.end(),data.begin(),data.end());} +struct Fixture {json doc;Bytes data;}; +static Fixture fixture(bool shortJoints=false,size_t weightType=5126) { + Fixture f;for(float p:{0.f,0.f,0.f,1.f,0.f,0.f,0.f,1.f,0.f}) real(f.data,p); + for(uint16_t j:{0,0,0,0,1,0,0,0,0,1,0,0}) {f.data.push_back(uint8_t(j));if(shortJoints) f.data.push_back(uint8_t(j>>8));} + const size_t weightOffset=f.data.size(); + if(weightType==5126) for(float w:{1.f,0.f,0.f,0.f,1.f,0.f,0.f,0.f,.5f,.5f,0.f,0.f}) real(f.data,w); + else if(weightType==5121) for(uint8_t w:{255,0,0,0,255,0,0,0,128,127,0,0}) f.data.push_back(w); + else for(uint16_t w:{65535,0,0,0,65535,0,0,0,32768,32767,0,0}) {f.data.push_back(uint8_t(w));f.data.push_back(uint8_t(w>>8));} + const size_t matrixOffset=f.data.size(); + for(size_t joint=0;joint<4;++joint) for(size_t k=0;k<16;++k) real(f.data,k==12 && joint==0?-1.f:k==0 || k==5 || k==10 || k==15?1.f:0.f); + f.doc={{"asset",{{"version","2.0"}}},{"buffers",{{{"byteLength",f.data.size()}}}}, + {"bufferViews",{{{"buffer",0},{"byteOffset",0},{"byteLength",36}}, + {{"buffer",0},{"byteOffset",36},{"byteLength",weightOffset-36}}, + {{"buffer",0},{"byteOffset",weightOffset},{"byteLength",matrixOffset-weightOffset}}, + {{"buffer",0},{"byteOffset",matrixOffset},{"byteLength",256}}}}, + {"accessors",{{{"bufferView",0},{"componentType",5126},{"count",3},{"type","VEC3"}}, + {{"bufferView",1},{"componentType",shortJoints?5123:5121},{"count",3},{"type","VEC4"}}, + {{"bufferView",2},{"componentType",weightType},{"count",3},{"type","VEC4"},{"normalized",weightType!=5126}}, + {{"bufferView",3},{"componentType",5126},{"count",4},{"type","MAT4"}}}}, + {"meshes",{{{"primitives",{{{"attributes",{{"POSITION",0},{"JOINTS_0",1},{"WEIGHTS_0",2}}}}}}}}}, + {"nodes",{{{"mesh",0},{"skin",0},{"translation",{8,0,0}},{"scale",{-1,2,1}}}, + {{"name","ROOT"},{"children",{2,3,4}}},{{"name","CHILD"},{"translation",{1,0,0}}},{{"name","OTHER"}},{{"name","LAST"}}}}, + {"skins",{{{"joints",{2,1,3,4}},{"skeleton",1},{"inverseBindMatrices",3}}}}, + {"scenes",{{{"nodes",{0,1}}}}},{"scene",0}}; + return f; +} +static void save(const fs::path& path,const Fixture& f) { + const auto text=f.doc.dump();Bytes bytes;word(bytes,0x46546c67);word(bytes,2);word(bytes,0); + chunk(bytes,0x4e4f534a,Bytes(text.begin(),text.end()));chunk(bytes,0x004e4942,f.data); + const auto size=uint32_t(bytes.size());for(unsigned i=0;i<4;++i) bytes[8+i]=uint8_t(size>>(8*i)); + std::ofstream out(path,std::ios::binary);out.write(reinterpret_cast(bytes.data()),std::streamsize(bytes.size()));require(bool(out),"fixture write failed"); +} +int main() { + const auto dir=fs::temp_directory_path()/("FableForgeSkinInput-"+std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + try { + require(fs::create_directory(dir),"fixture directory already exists"); + require(fs::canonical(dir).parent_path()==fs::canonical(fs::temp_directory_path()),"fixture escaped temporary root"); + const auto path=dir/"skin.glb";size_t checks=0; + const auto load=[&](const Fixture& f){save(path,f);return albion::meshimport::loadSkinnedModel(path);}; + const auto refuses=[&](const Fixture& f,const char* label) {bool failed=false;try {load(f);}catch(const std::exception&) {failed=true;}require(failed,label);++checks;}; + for(bool shorts:{false,true}) for(size_t weights:{5126u,5121u,5123u}) { + const auto m=load(fixture(shorts,weights));require(m.prims.size()==1 && m.skins.size()==1,"skin/primitive alignment"); + const auto& p=m.prims[0];const auto& s=m.skins[0]; + require(p.verts[1].x==100 && p.verts[2].z==100 && p.faces[0]==std::array{0,1,2},"skinned node pose baked or winding flipped"); + require(s.sourceSkin==0 && s.meshNode==0 && s.skeletonRoot==1 && s.joints.size()==4,"skin identity lost"); + require(s.joints[0].name=="CHILD" && s.joints[0].sourceNode==2 && s.joints[0].parentNode==1 && s.joints[0].world[12]==1 && s.joints[0].inverseBind[12]==-1,"joint order/world/bind changed"); + require(s.influences[0].size()==1 && s.influences[0][0].bone==0 && s.influences[1][0].bone==1 && s.influences[2].size()==2,"joint ordinal/weights lost"); + require(std::abs(s.influences[2][0].weight+s.influences[2][1].weight-1)<1e-6,"normalized weights changed");++checks; + } + auto f=fixture();f.doc["skins"][0].erase("inverseBindMatrices");const auto identity=load(f);require(identity.skins[0].joints[0].inverseBind[12]==0 && identity.skins[0].joints[0].inverseBind[15]==1,"omitted inverse bind not identity");++checks; + f=fixture();f.doc["skins"][0]["joints"]={2,2,3,4};refuses(f,"duplicate skin joint accepted"); + f=fixture();f.doc["skins"][0]["joints"][0]=99;refuses(f,"missing skin joint accepted"); + f=fixture();f.doc["scenes"][0]["nodes"]={0};refuses(f,"out-of-scene joints accepted"); + f=fixture();f.doc["nodes"][1].erase("children");f.doc["scenes"][0]["nodes"]={0,1,2,3,4};refuses(f,"disconnected joint roots accepted"); + f=fixture();f.doc["skins"][0]["skeleton"]=2;refuses(f,"non-common skeleton root accepted"); + f=fixture();f.doc["accessors"][3]["count"]=3;refuses(f,"short inverse bind accessor accepted"); + f=fixture();f.doc["accessors"][3]["type"]="VEC4";refuses(f,"wrong inverse bind shape accepted"); + f=fixture();setReal(f.data,96+12,1);refuses(f,"projective inverse bind accepted"); + f=fixture();f.data[36]=4;refuses(f,"joint outside skin.joints accepted"); + f=fixture();setReal(f.data,48,-1);refuses(f,"negative skin weight accepted"); + f=fixture();setReal(f.data,48,.5f);refuses(f,"weight sum error accepted"); + f=fixture();setReal(f.data,48,std::nanf(""));refuses(f,"nonfinite skin weight accepted"); + f=fixture();setReal(f.data,48,.5f);setReal(f.data,52,.5f);refuses(f,"duplicate positive joint influence accepted"); + f=fixture();f.doc["accessors"][1]["normalized"]=true;refuses(f,"normalized joints accepted"); + f=fixture();f.doc["accessors"][2]["count"]=2;refuses(f,"short weight accessor accepted"); + f=fixture();f.doc["meshes"][0]["primitives"][0]["attributes"].erase("WEIGHTS_0");refuses(f,"unpaired skin attributes accepted"); + f=fixture();f.doc["nodes"][0].erase("skin");refuses(f,"unskinned selected mesh accepted"); + f=fixture();f.doc.erase("scenes");refuses(f,"uninstantiated skin accepted"); + f=fixture();save(path,f);bool staticRefused=false;try {albion::meshimport::loadModel(path);}catch(const std::exception&) {staticRefused=true;}require(staticRefused,"static path silently stripped skin");++checks; + f=fixture();f.doc["nodes"][0].erase("skin");save(path,f);require(albion::meshimport::loadModel(path).skins.empty(),"static path gained skin metadata");++checks; + f=fixture();setReal(f.data,48,.25f);setReal(f.data,52,.25f);f.data[37]=1; + const size_t jointOffset=f.data.size();f.data.insert(f.data.end(),{2,3,0,0,0,0,0,0,0,0,0,0}); + const size_t weightOffset=f.data.size();for(float v:{.25f,.25f,0.f,0.f,0.f,0.f,0.f,0.f,0.f,0.f,0.f,0.f}) real(f.data,v); + f.doc["bufferViews"].push_back({{"buffer",0},{"byteOffset",jointOffset},{"byteLength",12}}); + f.doc["bufferViews"].push_back({{"buffer",0},{"byteOffset",weightOffset},{"byteLength",48}}); + f.doc["accessors"].push_back({{"bufferView",4},{"componentType",5121},{"count",3},{"type","VEC4"}}); + f.doc["accessors"].push_back({{"bufferView",5},{"componentType",5126},{"count",3},{"type","VEC4"}}); + f.doc["buffers"][0]["byteLength"]=f.data.size(); + auto& a=f.doc["meshes"][0]["primitives"][0]["attributes"];a["JOINTS_1"]=4;a["WEIGHTS_1"]=5; + const auto multi=load(f);require(multi.skins[0].influences[0].size()==4 && !multi.notes.empty(),"higher weight set silently dropped");++checks; + auto gap=f;auto& gapAttrs=gap.doc["meshes"][0]["primitives"][0]["attributes"]; + gapAttrs["JOINTS_2"]=4;gapAttrs["WEIGHTS_2"]=5;gapAttrs.erase("JOINTS_1");gapAttrs.erase("WEIGHTS_1");refuses(gap,"nonconsecutive skin sets accepted"); + auto duplicate=f;duplicate.data[jointOffset]=0;refuses(duplicate,"positive duplicate across weight sets accepted"); + auto malformed=f;malformed.doc["meshes"][0]["primitives"][0]["attributes"]["JOINTS_01"]=4;refuses(malformed,"leading-zero skin set accepted"); + f=fixture();f.doc["nodes"].push_back({{"mesh",0},{"skin",0},{"translation",{20,0,0}}});f.doc["scenes"][0]["nodes"].push_back(5); + const auto repeated=load(f);require(repeated.prims.size()==2 && repeated.skins.size()==2 && repeated.skins[1].meshNode==5 && repeated.prims[1].verts[1].x==100,"repeated skinned instances lost identity or baked pose");++checks; + f=fixture();const size_t sparseIndices=f.data.size();f.data.insert(f.data.end(),{1,2,0,0});const size_t sparseValues=f.data.size();f.data.insert(f.data.end(),{1,0,0,0,0,1,0,0}); + f.doc["buffers"][0]["byteLength"]=f.data.size(); + f.doc["bufferViews"].push_back({{"buffer",0},{"byteOffset",sparseIndices},{"byteLength",2}}); + f.doc["bufferViews"].push_back({{"buffer",0},{"byteOffset",sparseValues},{"byteLength",8}}); + f.doc["accessors"][1].erase("bufferView");f.doc["accessors"][1]["sparse"]={{"count",2},{"indices",{{"bufferView",4},{"componentType",5121}}},{"values",{{"bufferView",5}}}}; + const auto sparse=load(f);require(sparse.skins[0].influences[1][0].bone==1 && sparse.skins[0].influences[2][1].bone==1,"sparse joint attributes lost");++checks; + f=fixture(false,5121);f.data[48]=254;refuses(f,"quantized weight sum error accepted"); + f=fixture();f.doc["bufferViews"][3]["byteStride"]=64;refuses(f,"interleaved inverse binds accepted"); + f=fixture();f.doc["buffers"][0]["uri"]="skin.bin"; + const auto external=dir/"skin.gltf"; + {std::ofstream out(dir/"skin.bin",std::ios::binary);out.write(reinterpret_cast(f.data.data()),std::streamsize(f.data.size()));} + {std::ofstream out(external);out< Date: Sat, 10 Oct 2026 14:04:38 -0600 Subject: [PATCH 39/75] feat(mesh): convert authored inverse binds to donor coordinates --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 13 +++++++++++ src/meshimport.cpp | 19 ++++++++++++++++ src/meshimport.hpp | 4 ++++ tests/test_skinnedinput.cpp | 27 ++++++++++++++++++++++- 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index f3886c2a..1bb33edf 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -953,3 +953,16 @@ owned graphics/texture and private definition publication, creature import card and placement/runtime evidence. Reading metadata is not compatibility proof; the complete custom-body workflow remains unfinished. Existing checkpoints stay as previously validated. + +Donor comparison coordinate foundation: inverseBindInFable converts original +column-major glTF matrices to row-major Fable matrices by C*M*inverse(C), where +C maps metres (x,y,z) to centimetres (x,-z,y)*100. Linear scale cancels in this +conjugation; translation retains the factor 100. Source nonfinite/projective +matrices refuse. No donor bytes, source pose or weights are modified. Independent +point-action tests use asymmetric linear terms and four points to catch transpose, +axis/sign and unit errors, plus five malformed matrix refusals. Fixture POSITION +now includes its actual min/max bounds. This is coordinate conversion only, not +joint-name resolution, hierarchy validation or donor compatibility proof; those +remain the next implementation step. Shipping executables/checkpoints unchanged. +Rebuilt skin/static input targets; all three focused skin-input, static-input and +skinned-composition CTests pass. Skin-input executable reports 45 checks PASS. diff --git a/src/meshimport.cpp b/src/meshimport.cpp index 689c5f34..f3e5bb46 100644 --- a/src/meshimport.cpp +++ b/src/meshimport.cpp @@ -915,6 +915,25 @@ static void appendUvNotes(Model& model) { } } +std::array inverseBindInFable(const std::array& source) { + for(float value:source) + if(!std::isfinite(value)) throw std::runtime_error("non-finite source inverse bind matrix"); + if(source[3]!=0 || source[7]!=0 || source[11]!=0 || source[15]!=1) + throw std::runtime_error("non-affine source inverse bind matrix"); + // Signed permutation plus uniform unit scale. Linear terms lose the + // uniform scale in conjugation; translation retains metres-to-centimetres. + constexpr std::array axes{0,2,1}; + constexpr std::array signs{1,-1,1}; + std::array result{}; + for(size_t row=0;row<3;++row) { + for(size_t column=0;column<3;++column) + result[row*4+column]=signs[row]*signs[column]*source[axes[column]*4+axes[row]]; + result[row*4+3]=100.0*signs[row]*source[12+axes[row]]; + } + result[15]=1; + return result; +} + Model loadModel(const fs::path& path) { std::string ext = path.extension().string(); std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return char(std::tolower(c)); }); diff --git a/src/meshimport.hpp b/src/meshimport.hpp index 1843dc4b..ead62137 100644 --- a/src/meshimport.hpp +++ b/src/meshimport.hpp @@ -56,6 +56,10 @@ Model loadModel(const std::filesystem::path& path); // Does not apply pose or skinned mesh-node transforms to the authored vertices, // map joints to a donor, compose bytes or publish banks. Static loader unchanged. Model loadSkinnedModel(const std::filesystem::path& path); +// Coordinate-only conversion for donor bind comparison: C * M * inverse(C), +// C maps glTF metres (x,y,z) to Fable centimetres (x,-z,y)*100. +// Returns a row-major affine matrix. Does not establish donor compatibility. +std::array inverseBindInFable(const std::array& gltfMatrix); struct ImportRequest { std::filesystem::path model; diff --git a/tests/test_skinnedinput.cpp b/tests/test_skinnedinput.cpp index b410c479..94806f04 100644 --- a/tests/test_skinnedinput.cpp +++ b/tests/test_skinnedinput.cpp @@ -31,7 +31,7 @@ static Fixture fixture(bool shortJoints=false,size_t weightType=5126) { {{"buffer",0},{"byteOffset",36},{"byteLength",weightOffset-36}}, {{"buffer",0},{"byteOffset",weightOffset},{"byteLength",matrixOffset-weightOffset}}, {{"buffer",0},{"byteOffset",matrixOffset},{"byteLength",256}}}}, - {"accessors",{{{"bufferView",0},{"componentType",5126},{"count",3},{"type","VEC3"}}, + {"accessors",{{{"bufferView",0},{"componentType",5126},{"count",3},{"type","VEC3"},{"min",{0,0,0}},{"max",{1,1,0}}}, {{"bufferView",1},{"componentType",shortJoints?5123:5121},{"count",3},{"type","VEC4"}}, {{"bufferView",2},{"componentType",weightType},{"count",3},{"type","VEC4"},{"normalized",weightType!=5126}}, {{"bufferView",3},{"componentType",5126},{"count",4},{"type","MAT4"}}}}, @@ -54,6 +54,31 @@ int main() { require(fs::create_directory(dir),"fixture directory already exists"); require(fs::canonical(dir).parent_path()==fs::canonical(fs::temp_directory_path()),"fixture escaped temporary root"); const auto path=dir/"skin.glb";size_t checks=0; + // Independent point oracle: converting after applying M must agree + // with applying the converted matrix to the converted point. Use an + // asymmetric linear matrix to expose transpose/sign/axis mistakes. + const std::array source{2,3,5,0,7,11,13,0,17,19,23,0,29,31,37,1}; + const auto converted=albion::meshimport::inverseBindInFable(source); + for(const std::array point:std::array,4>{{{0,0,0},{1,0,0},{0,1,0},{2,-3,4}}}) { + std::array transformed{}; + for(size_t row=0;row<3;++row) { + transformed[row]=source[12+row]; + for(size_t column=0;column<3;++column) transformed[row]+=source[column*4+row]*point[column]; + } + const std::array fablePoint{100*point[0],-100*point[2],100*point[1]}; + const std::array expected{100*transformed[0],-100*transformed[2],100*transformed[1]}; + for(size_t row=0;row<3;++row) { + double actual=converted[row*4+3]; + for(size_t column=0;column<3;++column) actual+=converted[row*4+column]*fablePoint[column]; + require(actual==expected[row],"inverse bind coordinate conjugation mismatch"); + } + ++checks; + } + for(size_t bad:{3u,7u,11u,15u,0u}) { + auto invalid=source;invalid[bad]=bad==0?std::nanf(""):2.f; + bool rejected=false;try {albion::meshimport::inverseBindInFable(invalid);}catch(const std::exception&) {rejected=true;} + require(rejected,"invalid inverse bind conversion accepted");++checks; + } const auto load=[&](const Fixture& f){save(path,f);return albion::meshimport::loadSkinnedModel(path);}; const auto refuses=[&](const Fixture& f,const char* label) {bool failed=false;try {load(f);}catch(const std::exception&) {failed=true;}require(failed,label);++checks;}; for(bool shorts:{false,true}) for(size_t weights:{5126u,5121u,5123u}) { From fc486e5ce1488cae99b12e6110a97f0e9c29e67a Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:09:14 -0600 Subject: [PATCH 40/75] feat(mesh): validate and remap authored skin to donor joints --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 29 ++++++++ src/meshimport.cpp | 80 +++++++++++++++++++++++ src/meshimport.hpp | 9 +++ tests/test_skinnedinput.cpp | 50 ++++++++++++++ 4 files changed, 168 insertions(+) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 1bb33edf..99c6eaca 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -966,3 +966,32 @@ joint-name resolution, hierarchy validation or donor compatibility proof; those remain the next implementation step. Shipping executables/checkpoints unchanged. Rebuilt skin/static input targets; all three focused skin-input, static-input and skinned-composition CTests pass. Skin-input executable reports 45 checks PASS. + +Read-only mapSkinToDonor now resolves every positive-weight source joint by +exact unique name to a 1-256-slot decoded donor. Required donor ancestors also +resolve and validate binds, even if unweighted. SkinJoint.parentJoint retains +nearest skin-joint ancestry through non-joint scene nodes; parent chains must +match donor names. Partial donor roots may have extra source ancestors above +them; unrelated unweighted source joints need not exist in the donor. Empty, +missing or ambiguous required names refuse, as do invalid donor parents/cycles, +nonfinite/projective donor binds, bad source influences/weight totals and parent +or bind mismatches. Source default pose is not used as a bind pose. All positive +influences survive unchanged except their local slot indices; strongest-three +reduction remains exclusively the composer's job. Geometry is copied unchanged. + +Comparison uses converted inverse binds with absolute tolerance 1e-5 for linear +terms, 0.001 cm for translation, plus 1e-5 relative. This is an explicit import +policy for float authoring jitter, not a recovered engine tolerance. Native donor +export round-trip evidence is still required before promoting the import card. +API takes decoded bones; future publication must derive both decoded bones and +raw skeleton spans from the same freshly validated donor payload, not mix inputs. +No bank writer, shipping app rebuild, package update or runtime proof in this step. +Validation complete: final affected-target build succeeded. The same initially +delayed CTest invocation (exec session 77476) completed terminal exit zero: all +three focused skin-input, static-input and skinned-composition tests PASS. Direct +skin-input run reports 66 checks PASS, including remapping, intermediate non-joint +ancestry, unused joints, partial donor roots, unweighted required ancestor binds, +default-pose independence, float jitter and retention of four positive influences. +No test restart or acceptance from an old log. Next: exercise authored glTF input +and mapped composition against native donor exports with independent readback, +then owned-bank/private-definition publication and the creature import card. diff --git a/src/meshimport.cpp b/src/meshimport.cpp index f3e5bb46..7c26bfc5 100644 --- a/src/meshimport.cpp +++ b/src/meshimport.cpp @@ -435,6 +435,11 @@ SkinBinding readSkin(const GltfBuffers& g,size_t skinIndex,size_t meshNode, commonRoot=root; binding.joints.push_back({ni,parents[ni],nodes.at(ni).value("name",std::string()),*worlds[ni],identity()}); } + std::map ordinals; + for(size_t i=0;isecond;break;} if(skin.contains("inverseBindMatrices")) { const size_t index=gltfSize(skin,"inverseBindMatrices");const auto& acc=g.doc.at("accessors").at(index); if(acc.at("type")!="MAT4" || gltfSize(acc,"componentType")!=5126 || acc.value("normalized",false) || @@ -934,6 +939,81 @@ std::array inverseBindInFable(const std::array& source) { return result; } +std::vector mapSkinToDonor( + const Model& model,const std::vector& donor) { + const auto fail=[](const std::string& reason) {throw std::runtime_error("donor skin mapping: "+reason);}; + if(donor.empty() || donor.size()>256) fail("donor must have 1-256 local bones"); + if(model.prims.empty() || model.skins.size()!=model.prims.size()) fail("missing aligned skin primitives"); + std::map> donorNames; + for(size_t i=0;i=int32_t(donor.size())) fail("invalid donor parent"); + for(float value:donor[i].inverseBind) if(!std::isfinite(value)) fail("nonfinite donor inverse bind"); + const auto& m=donor[i].inverseBind; + if(m[12]!=0 || m[13]!=0 || m[14]!=0 || m[15]!=1) fail("non-affine donor inverse bind"); + } + for(size_t i=0;i=0;p=donor[size_t(p)].parent) + if(++depth>donor.size()) fail("cyclic donor hierarchy"); + } + std::vector result; + for(size_t part=0;part> sourceNames; + for(size_t i=0;i=skin.joints.size()) fail("invalid source parent joint"); + } + std::vector> mapped(skin.joints.size()); + std::vector required(skin.joints.size(),false); + for(const auto& vertex:skin.influences) { + double total=0;std::vector seen(skin.joints.size(),false); + for(const auto& influence:vertex) { + if(influence.bone>=skin.joints.size() || !std::isfinite(influence.weight) || influence.weight<0) fail("invalid source influence"); + if(influence.weight==0) continue; + if(seen[influence.bone]) fail("duplicate positive source influence"); + seen[influence.bone]=true;required[influence.bone]=true;total+=influence.weight; + } + if(!std::isfinite(total) || std::abs(total-1)>1e-5) fail("source weights must sum to one"); + } + // Validate donor ancestor closure as well as directly weighted joints. + // A partial donor root can have additional source ancestors above it. + for(size_t first=0;firstskin.joints.size()) fail("cyclic source hierarchy"); + const auto& joint=skin.joints[current];const auto& name=joint.name; + const auto found=donorNames.find(name); + if(name.empty() || sourceNames.at(name).size()!=1 || found==donorNames.end() || found->second.size()!=1) + fail("missing or ambiguous joint name '"+name+"'"); + const size_t slot=found->second[0];const auto converted=inverseBindInFable(joint.inverseBind); + for(size_t k=0;k<16;++k) { + const double expected=donor[slot].inverseBind[k]; + const double tolerance=(k<12 && k%4==3?0.001:1e-5)+1e-5*std::max(std::abs(expected),std::abs(converted[k])); + if(std::abs(converted[k]-expected)>tolerance) fail("inverse bind mismatch for '"+name+"'"); + } + mapped[current]=slot; + if(donor[slot].parent<0) break; + if(!joint.parentJoint) fail("missing source parent for '"+name+"'"); + const size_t parent=*joint.parentJoint; + if(skin.joints[parent].name!=donor[size_t(donor[slot].parent)].name) fail("parent mismatch for '"+name+"'"); + current=parent; + } + } + forge::meshcompose::SkinnedPrimitive primitive;primitive.geometry=model.prims[part]; + for(const auto& vertex:skin.influences) { + std::vector weights; + for(const auto& influence:vertex) if(influence.weight>0) + weights.push_back({uint16_t(*mapped[influence.bone]),influence.weight}); + primitive.skin.push_back(std::move(weights)); + } + result.push_back(std::move(primitive)); + } + return result; +} + Model loadModel(const fs::path& path) { std::string ext = path.extension().string(); std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return char(std::tolower(c)); }); diff --git a/src/meshimport.hpp b/src/meshimport.hpp index ead62137..903835bc 100644 --- a/src/meshimport.hpp +++ b/src/meshimport.hpp @@ -19,6 +19,7 @@ #include #include "forge/meshcompose.hpp" +namespace forge::meshpreview { struct Bone; } namespace albion::meshimport { @@ -28,6 +29,7 @@ struct SkinJoint { std::string name; // Original glTF coordinates/metres, column-major. Not donor transforms. std::array world{},inverseBind{}; + std::optional parentJoint; // nearest ancestor in skin.joints, through non-joint nodes }; struct SkinBinding { size_t sourceSkin=0,meshNode=0; @@ -60,6 +62,13 @@ Model loadSkinnedModel(const std::filesystem::path& path); // C maps glTF metres (x,y,z) to Fable centimetres (x,-z,y)*100. // Returns a row-major affine matrix. Does not establish donor compatibility. std::array inverseBindInFable(const std::array& gltfMatrix); +// Read-only donor remapping. Exact unique names for positive-weight joints and +// donor ancestors; checks nearest-joint parent chains and converted inverse binds. +// Bind tolerance: 1e-5 linear / 0.001 cm translation absolute + 1e-5 relative. +// Retains geometry and every positive weight; does not compose/publish or prove +// animation/runtime compatibility. Donor roots may be partial source subtrees. +std::vector mapSkinToDonor( + const Model& model,const std::vector& donor); struct ImportRequest { std::filesystem::path model; diff --git a/tests/test_skinnedinput.cpp b/tests/test_skinnedinput.cpp index 94806f04..9f9775b2 100644 --- a/tests/test_skinnedinput.cpp +++ b/tests/test_skinnedinput.cpp @@ -1,4 +1,5 @@ #include "meshimport.hpp" +#include "forge/meshpreview.hpp" #include "nlohmann/json.hpp" #include #include @@ -81,6 +82,50 @@ int main() { } const auto load=[&](const Fixture& f){save(path,f);return albion::meshimport::loadSkinnedModel(path);}; const auto refuses=[&](const Fixture& f,const char* label) {bool failed=false;try {load(f);}catch(const std::exception&) {failed=true;}require(failed,label);++checks;}; + const auto authored=load(fixture()); + std::vector donor(2); + donor[0].name="ROOT";donor[0].parent=-1;donor[1].name="CHILD";donor[1].parent=0; + for(auto& bone:donor) bone.inverseBind={1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1}; + donor[1].inverseBind[3]=-100; + const auto mapped=albion::meshimport::mapSkinToDonor(authored,donor); + require(mapped.size()==1 && mapped[0].skin[0][0].bone==1 && mapped[0].skin[1][0].bone==0 && + mapped[0].skin[2][0].weight==.5 && mapped[0].geometry.verts[1].x==100, + "donor remap changed geometry, weights or used source order");++checks; + require(authored.skins[0].joints[0].parentJoint==1 && !authored.skins[0].joints[1].parentJoint, + "nearest joint hierarchy not retained");++checks; + const auto mappingRefuses=[&](const albion::meshimport::Model& input,const std::vector& bones,const char* label) { + bool rejected=false;try {albion::meshimport::mapSkinToDonor(input,bones);}catch(const std::exception&) {rejected=true;} + require(rejected,label);++checks; + }; + auto badDonor=donor;badDonor[1].name="child";mappingRefuses(authored,badDonor,"case-folded joint accepted"); + badDonor=donor;badDonor.push_back(donor[1]);mappingRefuses(authored,badDonor,"ambiguous donor joint accepted"); + badDonor=donor;badDonor[0].parent=1;mappingRefuses(authored,badDonor,"cyclic donor accepted"); + badDonor=donor;badDonor[1].parent=2;mappingRefuses(authored,badDonor,"invalid donor parent accepted"); + badDonor=donor;badDonor[1].inverseBind[3]=-99;mappingRefuses(authored,badDonor,"wrong donor bind accepted"); + badDonor=donor;badDonor[1].inverseBind[0]=std::nanf("");mappingRefuses(authored,badDonor,"nonfinite donor bind accepted"); + badDonor=donor;badDonor[1].inverseBind[15]=0;mappingRefuses(authored,badDonor,"projective donor bind accepted"); + auto badSource=authored;badSource.skins[0].joints[2].name="CHILD";mappingRefuses(badSource,donor,"ambiguous source joint accepted"); + badSource=authored;badSource.skins[0].joints[0].parentJoint.reset();mappingRefuses(badSource,donor,"missing parent chain accepted"); + badSource=authored;badSource.skins[0].joints[0].parentJoint=2;mappingRefuses(badSource,donor,"wrong parent chain accepted"); + badSource=authored;badSource.skins[0].influences[0].clear();mappingRefuses(badSource,donor,"empty source weights silently root-bound"); + badSource=authored;badSource.skins[0].influences[0][0].bone=4;mappingRefuses(badSource,donor,"invalid source joint accepted"); + badSource=authored;badSource.skins[0].influences[0].push_back({0,.5});mappingRefuses(badSource,donor,"duplicate source influence accepted"); + // Non-joint scene nodes may sit between joints without changing the + // effective joint hierarchy. Unused unmatched source joints are retained. + auto throughNode=fixture();throughNode.doc["nodes"].push_back({{"children",{2}}}); + throughNode.doc["nodes"][1]["children"]={5,3,4}; + const auto throughModel=load(throughNode); + require(throughModel.skins[0].joints[0].parentNode==5 && throughModel.skins[0].joints[0].parentJoint==1 && + albion::meshimport::mapSkinToDonor(throughModel,donor)[0].skin[0][0].bone==1, + "non-joint ancestor broke donor mapping");++checks; + auto posed=authored;posed.skins[0].joints[0].world[12]=900; + require(albion::meshimport::mapSkinToDonor(posed,donor)[0].geometry.verts[1].x==100,"default pose mistaken for bind pose");++checks; + auto childOnly=authored;for(auto& weights:childOnly.skins[0].influences) weights={{0,1}}; + auto partial=std::vector{donor[1]};partial[0].parent=-1; + require(albion::meshimport::mapSkinToDonor(childOnly,partial)[0].skin[0][0].bone==0,"partial donor root rejected");++checks; + badDonor=donor;badDonor[0].inverseBind[3]=10;mappingRefuses(childOnly,badDonor,"unweighted required ancestor bind not checked"); + badDonor=donor;badDonor[1].inverseBind[3]-=.0001f; + require(albion::meshimport::mapSkinToDonor(authored,badDonor)[0].skin[0][0].bone==1,"float bind jitter rejected");++checks; for(bool shorts:{false,true}) for(size_t weights:{5126u,5121u,5123u}) { const auto m=load(fixture(shorts,weights));require(m.prims.size()==1 && m.skins.size()==1,"skin/primitive alignment"); const auto& p=m.prims[0];const auto& s=m.skins[0]; @@ -124,6 +169,11 @@ int main() { auto gap=f;auto& gapAttrs=gap.doc["meshes"][0]["primitives"][0]["attributes"]; gapAttrs["JOINTS_2"]=4;gapAttrs["WEIGHTS_2"]=5;gapAttrs.erase("JOINTS_1");gapAttrs.erase("WEIGHTS_1");refuses(gap,"nonconsecutive skin sets accepted"); auto duplicate=f;duplicate.data[jointOffset]=0;refuses(duplicate,"positive duplicate across weight sets accepted"); + auto fullDonor=donor; + for(const auto& name:{"OTHER","LAST"}) {auto bone=donor[0];bone.name=name;bone.parent=0;fullDonor.push_back(bone);} + const auto fourWeights=albion::meshimport::mapSkinToDonor(load(f),fullDonor); + require(fourWeights[0].skin[0].size()==4 && fourWeights[0].skin[0][2].bone==2 && fourWeights[0].skin[0][3].weight==.25, + "donor mapping prematurely reduced positive weights");++checks; auto malformed=f;malformed.doc["meshes"][0]["primitives"][0]["attributes"]["JOINTS_01"]=4;refuses(malformed,"leading-zero skin set accepted"); f=fixture();f.doc["nodes"].push_back({{"mesh",0},{"skin",0},{"translation",{20,0,0}}});f.doc["scenes"][0]["nodes"].push_back(5); const auto repeated=load(f);require(repeated.prims.size()==2 && repeated.skins.size()==2 && repeated.skins[1].meshNode==5 && repeated.prims[1].verts[1].x==100,"repeated skinned instances lost identity or baked pose");++checks; From fb74041e36cfdde553a932f92a18003340af4fcf Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:11:31 -0600 Subject: [PATCH 41/75] test(mesh): verify authored skin import against native donors --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 27 +++++ tests/test_skinnedinput.cpp | 43 ++++++- tools/test_skinned_input_donors.py | 134 ++++++++++++++++++++++ 3 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 tools/test_skinned_input_donors.py diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 99c6eaca..41b3092c 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -995,3 +995,30 @@ default-pose independence, float jitter and retention of four positive influence No test restart or acceptance from an old log. Next: exercise authored glTF input and mapped composition against native donor exports with independent readback, then owned-bank/private-definition publication and the creature import card. + +Native donor input/composition gate added: test_skinnedinput --donor graphics.big +ID source.gltf fresh-output-directory reads one donor from the archive, uses the +production skin loader/remapper, composes stored/compressed outputs and verifies +all five donor raw spans on C++ readback. tools/test_skinned_input_donors.py builds +tiny sidecar glTF triangles using independent addon-decoded native names, parents +and IBMs. Generic Python matrix multiplication reverses the coordinate basis; +skin.joints order is reversed to ensure mapping is not an ordinal passthrough. +Mesh-node translation/nonuniform mirror must not bake into authored geometry. +Default node transforms are identity, not a Blender-exported native bind scene. +This proves this authored donor-metadata route, not a full Blender or native +geometry export round trip, all donor headers, animation or gameplay behavior. + +Five donors (4521/5121/5150/3985/5110, 17-28 bones) pass ten stored/compressed +independent Python readbacks: exact donor spans in LOD0 and ghost, intended +triangle positions/winding and per-vertex mapped weights within byte quantization. +Five required-name mismatch inputs refuse before output-directory creation. +Evidence: skinned-input-donors-fl8uxu0_ under external 20261010 evidence root; +result.json records each payload hash and unchanged native graphics SHA256 +8b909c4036a89840413f66828d855ee3ccf29796feaeb1f05f36727a5086f617. +Initial 18ojta_v packet passed donor 4521 output checks but its refusal assertion +expected a name error; actual parent-mismatch refusal was equally correct. +Test assertion now recognizes both specific diagnostic paths; no production +validation changed. Rebuilt test harness and normal 66-case CTest also PASS. +No copied banks, native writes, shipping rebuild or checkpoint replacement. +Next: owned graphics/texture and private-definition transactional publication, +then creature import card, placement and runtime evidence. diff --git a/tests/test_skinnedinput.cpp b/tests/test_skinnedinput.cpp index 9f9775b2..3bb19cce 100644 --- a/tests/test_skinnedinput.cpp +++ b/tests/test_skinnedinput.cpp @@ -1,7 +1,10 @@ #include "meshimport.hpp" #include "forge/meshpreview.hpp" +#include "forge/big.hpp" #include "nlohmann/json.hpp" #include +#include +#include #include #include #include @@ -49,7 +52,45 @@ static void save(const fs::path& path,const Fixture& f) { const auto size=uint32_t(bytes.size());for(unsigned i=0;i<4;++i) bytes[8+i]=uint8_t(size>>(8*i)); std::ofstream out(path,std::ios::binary);out.write(reinterpret_cast(bytes.data()),std::streamsize(bytes.size()));require(bool(out),"fixture write failed"); } -int main() { +static int donorGate(char** argv) { + uint32_t id=0;const std::string idText=argv[3]; + const auto parsed=std::from_chars(idText.data(),idText.data()+idText.size(),id); + require(parsed.ec==std::errc{} && parsed.ptr==idText.data()+idText.size(),"invalid donor ID"); + const auto archive=forge::big::File::open(argv[2]);const auto* bank=archive.findBank("MBANK_ALLMESHES"); + require(bank,"missing donor bank"); + const auto entry=std::find_if(bank->entries.begin(),bank->entries.end(),[&](const auto& e){return e.id==id;}); + require(entry!=bank->entries.end() && entry->type==5,"missing type-5 donor"); + const auto donor=forge::meshpreview::decodeLod0(archive.entryData(*entry),5); + require(bool(donor.skeletonBlocks),"missing donor raw skeleton"); + const auto model=albion::meshimport::loadSkinnedModel(argv[4]); + const auto mapped=albion::meshimport::mapSkinToDonor(model,donor.bones); + const auto output=fs::absolute(argv[5]);require(!fs::exists(output),"donor output must be fresh"); + forge::meshcompose::Material material;material.name="native donor import test"; + std::vector meshes; + for(bool compressed:{false,true}) { + auto mesh=forge::meshcompose::composeSkinned("MESH_IMPORTED_"+idText,mapped,{material},*donor.skeletonBlocks,compressed); + const auto readback=forge::meshpreview::decodeLod0(mesh.payload,5); + require(readback.skeletonBlocks && readback.skeletonBlocks->indices==donor.skeletonBlocks->indices && + readback.skeletonBlocks->names==donor.skeletonBlocks->names && readback.skeletonBlocks->bones==donor.skeletonBlocks->bones && + readback.skeletonBlocks->keyframes==donor.skeletonBlocks->keyframes && readback.skeletonBlocks->inverseBinds==donor.skeletonBlocks->inverseBinds, + "imported donor spans changed"); + meshes.push_back(std::move(mesh)); + } + require(fs::create_directories(output),"cannot create fresh donor output"); + for(size_t i=0;i,2>{{{".mesh",&meshes[i].payload},{".info",&meshes[i].info}}}) { + std::ofstream stream(output/(std::string(stem)+suffix),std::ios::binary); + stream.write(reinterpret_cast(bytes->data()),std::streamsize(bytes->size()));require(bool(stream),"donor output write failed"); + } + } + std::cout<<"authored glTF -> donor remap -> stored/compressed readback PASS: "<= 3 + # Independent generic matrix multiplication, not the C++ signed-permutation + # implementation. Undo Fable coordinates before writing column-major glTF. + c = [[100,0,0,0],[0,0,-100,0],[0,100,0,0],[0,0,0,1]] + ci = [[.01,0,0,0],[0,0,.01,0],[0,-.01,0,0],[0,0,0,1]] + order = list(reversed(range(count))) + nodes = [{'name': name} for name in skeleton['names']] + roots = [] + for i in range(count): + parent = struct.unpack_from(' Date: Sat, 10 Oct 2026 14:13:50 -0600 Subject: [PATCH 42/75] feat(mesh): prepare guarded donor-skinned bank entries --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 21 +++++++++++++ src/meshimport.cpp | 37 +++++++++++++++++++++++ src/meshimport.hpp | 9 ++++++ tests/test_skinnedinput.cpp | 25 ++++++++++++--- 4 files changed, 87 insertions(+), 5 deletions(-) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 41b3092c..3b6ed158 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -1022,3 +1022,24 @@ validation changed. Rebuilt test harness and normal 66-case CTest also PASS. No copied banks, native writes, shipping rebuild or checkpoint replacement. Next: owned graphics/texture and private-definition transactional publication, then creature import card, placement and runtime evidence. + +Production prepareSkinnedMesh now prepares a new BIGB type-5 entry from a loaded +authored Model, unique donor ID, mesh name and material vector. It reads a single +donor payload and derives both decoded bones and raw skeleton spans from that +same decode, closing the mixed-donor gap in ad-hoc composition. Exactly one +MBANK_ALLMESHES is required; names and IDs are checked against every entry type. +Duplicate selected donor IDs, existing/new-invalid names, wrong container/type, +missing donor/bank and exhausted ID capacity refuse. Entry gets next bank-local +ID, native new-record magic 42, donor devFileType, composed Info and payload. +Mandatory readback verifies all five retained donor spans. Archive is const and +never appended/serialized/published here. Publication still must guard sources, +destination namespace, recovery/Restore and game-running state transactionally. + +Native gate now routes all ten outputs through this production preparation API, +checks the entry metadata and adds six preflight refusal cases per donor (30 +total). Independent Python readback and five authored-name mismatch refusals +again PASS for all five donors, packet skinned-input-donors-e2hokgxi. Native +graphics hash unchanged; no full-bank copies. Rebuilt affected targets and all +three focused CTests PASS. No shipping executable/package rebuild or runtime +claim. Coordinated graphics/texture/private-definition publication remains next; +existing clone publisher's exact private-clone contract is deliberately unchanged. diff --git a/src/meshimport.cpp b/src/meshimport.cpp index 7c26bfc5..615381a8 100644 --- a/src/meshimport.cpp +++ b/src/meshimport.cpp @@ -1014,6 +1014,43 @@ std::vector mapSkinToDonor( return result; } +forge::big::Entry prepareSkinnedMesh(const Model& model,const forge::big::File& graphics, + uint32_t donorId,const std::string& meshName, + const std::vector& materials,bool compress) { + const auto fail=[](const std::string& reason) {throw std::runtime_error("skinned mesh preparation: "+reason);}; + if(graphics.magic()!="BIGB") fail("requires BIGB graphics"); + if(!meshName.starts_with("MESH_") || !validName(meshName.substr(5))) fail("invalid new mesh name"); + const forge::big::Bank* bank=nullptr; + for(const auto& candidate:graphics.banks()) if(candidate.name=="MBANK_ALLMESHES") { + if(bank) fail("ambiguous MBANK_ALLMESHES");bank=&candidate; + } + if(!bank) fail("missing MBANK_ALLMESHES"); + const forge::big::Entry* source=nullptr;uint32_t maximum=0; + for(const auto& entry:bank->entries) { + maximum=std::max(maximum,entry.id); + if(entry.name==meshName) fail("new mesh name already exists"); + if(entry.id==donorId) {if(source) fail("ambiguous donor ID");source=&entry;} + } + if(!source || source->type!=5) fail("requires unique type-5 donor"); + if(maximum==UINT32_MAX) fail("no free mesh entry ID"); + const auto donor=forge::meshpreview::decodeLod0(graphics.entryData(*source),5); + if(!donor.skeletonBlocks) fail("donor has no raw skeleton"); + const auto prims=mapSkinToDonor(model,donor.bones); + auto composed=forge::meshcompose::composeSkinned(meshName,prims,materials,*donor.skeletonBlocks,compress); + const auto verified=forge::meshpreview::decodeLod0(composed.payload,5); + const auto& original=*donor.skeletonBlocks; + if(!verified.skeletonBlocks) fail("readback lost donor skeleton"); + const auto& actual=*verified.skeletonBlocks; + if(actual.indices!=original.indices || actual.names!=original.names || actual.bones!=original.bones || + actual.keyframes!=original.keyframes || actual.inverseBinds!=original.inverseBinds) + fail("readback changed donor skeleton"); + if(composed.payload.size()>UINT32_MAX) fail("composed payload exceeds BIG capacity"); + forge::big::Entry result;result.name=meshName;result.id=maximum+1;result.type=5; + result.magic=42;result.devFileType=source->devFileType; + result.length=uint32_t(composed.payload.size());result.data=std::move(composed.payload); + result.subHeader=std::move(composed.info);return result; +} + Model loadModel(const fs::path& path) { std::string ext = path.extension().string(); std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return char(std::tolower(c)); }); diff --git a/src/meshimport.hpp b/src/meshimport.hpp index 903835bc..a53fc3ba 100644 --- a/src/meshimport.hpp +++ b/src/meshimport.hpp @@ -20,6 +20,7 @@ #include "forge/meshcompose.hpp" namespace forge::meshpreview { struct Bone; } +namespace forge::big { class File; struct Entry; } namespace albion::meshimport { @@ -69,6 +70,14 @@ std::array inverseBindInFable(const std::array& gltfMatrix) // animation/runtime compatibility. Donor roots may be partial source subtrees. std::vector mapSkinToDonor( const Model& model,const std::vector& donor); +// Prepare one new type-5 BIGB entry without mutating/publishing the archive. +// A unique donor payload supplies both decoded bones and retained raw skeleton. +// Name and next bank-local ID are checked against all MBANK_ALLMESHES types. +// Readback verifies retained skeleton spans; caller must revalidate source and +// destination state in its coordinated bank/definition publication transaction. +forge::big::Entry prepareSkinnedMesh(const Model& model,const forge::big::File& graphics, + uint32_t donorId,const std::string& meshName, + const std::vector& materials,bool compress=true); struct ImportRequest { std::filesystem::path model; diff --git a/tests/test_skinnedinput.cpp b/tests/test_skinnedinput.cpp index 3bb19cce..5eb8cfe1 100644 --- a/tests/test_skinnedinput.cpp +++ b/tests/test_skinnedinput.cpp @@ -63,23 +63,38 @@ static int donorGate(char** argv) { const auto donor=forge::meshpreview::decodeLod0(archive.entryData(*entry),5); require(bool(donor.skeletonBlocks),"missing donor raw skeleton"); const auto model=albion::meshimport::loadSkinnedModel(argv[4]); - const auto mapped=albion::meshimport::mapSkinToDonor(model,donor.bones); const auto output=fs::absolute(argv[5]);require(!fs::exists(output),"donor output must be fresh"); forge::meshcompose::Material material;material.name="native donor import test"; - std::vector meshes; + std::vector meshes; for(bool compressed:{false,true}) { - auto mesh=forge::meshcompose::composeSkinned("MESH_IMPORTED_"+idText,mapped,{material},*donor.skeletonBlocks,compressed); - const auto readback=forge::meshpreview::decodeLod0(mesh.payload,5); + auto mesh=albion::meshimport::prepareSkinnedMesh(model,archive,id,"MESH_IMPORTED_"+idText,{material},compressed); + uint32_t maximum=0;for(const auto& existing:bank->entries) maximum=std::max(maximum,existing.id); + require(mesh.id==maximum+1 && mesh.type==5 && mesh.magic==42 && mesh.devFileType==entry->devFileType && + mesh.length==mesh.data.size() && mesh.name=="MESH_IMPORTED_"+idText,"prepared entry metadata invalid"); + const auto readback=forge::meshpreview::decodeLod0(mesh.data,5); require(readback.skeletonBlocks && readback.skeletonBlocks->indices==donor.skeletonBlocks->indices && readback.skeletonBlocks->names==donor.skeletonBlocks->names && readback.skeletonBlocks->bones==donor.skeletonBlocks->bones && readback.skeletonBlocks->keyframes==donor.skeletonBlocks->keyframes && readback.skeletonBlocks->inverseBinds==donor.skeletonBlocks->inverseBinds, "imported donor spans changed"); meshes.push_back(std::move(mesh)); } + const auto refuses=[&](const forge::big::File& candidate,const std::string& name,const char* reason) { + bool rejected=false; + try {albion::meshimport::prepareSkinnedMesh(model,candidate,id,name,{material});} + catch(const std::exception& e) {rejected=std::string(e.what()).find(reason)!=std::string::npos;} + require(rejected,"preparation guard did not refuse with expected reason"); + }; + refuses(archive,entry->name,"new mesh name already exists"); + refuses(archive,"MESH_bad","invalid new mesh name"); + auto bad=archive;bad.setMagic(std::string("B\0\0\0",4));refuses(bad,"MESH_TEST","requires BIGB"); + bad=archive;bad.banks().push_back(*bank);refuses(bad,"MESH_TEST","ambiguous MBANK_ALLMESHES"); + bad=archive;bad.findBank("MBANK_ALLMESHES")->entries.push_back(*entry);refuses(bad,"MESH_TEST","ambiguous donor ID"); + bad=archive;forge::big::Entry exhausted;exhausted.id=UINT32_MAX; + bad.findBank("MBANK_ALLMESHES")->entries.push_back(exhausted);refuses(bad,"MESH_TEST","no free mesh entry ID"); require(fs::create_directories(output),"cannot create fresh donor output"); for(size_t i=0;i,2>{{{".mesh",&meshes[i].payload},{".info",&meshes[i].info}}}) { + for(const auto& [suffix,bytes]:std::array,2>{{{".mesh",&meshes[i].data},{".info",&meshes[i].subHeader}}}) { std::ofstream stream(output/(std::string(stem)+suffix),std::ios::binary); stream.write(reinterpret_cast(bytes->data()),std::streamsize(bytes->size()));require(bool(stream),"donor output write failed"); } From 65a490d9505985592e715dfbb5c22f0c8f2d70ba Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:17:09 -0600 Subject: [PATCH 43/75] feat(creature): prepare coordinated private skinned-mesh plans --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 27 +++++++++ src/creatureimport.hpp | 55 ++++++++++++++++++ tests/test_skinnedinput.cpp | 70 ++++++++++++++++++++++- 3 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 src/creatureimport.hpp diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 3b6ed158..7dea657c 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -1043,3 +1043,30 @@ graphics hash unchanged; no full-bank copies. Rebuilt affected targets and all three focused CTests PASS. No shipping executable/package rebuild or runtime claim. Coordinated graphics/texture/private-definition publication remains next; existing clone publisher's exact private-clone contract is deliberately unchanged. + +creatureimport::prepareMesh adds an all-or-nothing in-memory coordinated plan. +Const source definition/graphics banks produce disposable copies: canonical +private root/body/appearance clone, selected body-slot lookup, skinned preparation +using that slot's current model as the donor, one new type-5 entry and exactly one +new-body model word changed. It does not allow an unrelated caller-selected rig. +Returns both proposed banks plus clone/binding indices; source graph, gameplay +ownership, original visual components and raw donor assets are unchanged. No +filesystem write or publication. Materials can carry prepared texture IDs, but +the future publisher must validate them against the coordinated texture bank. + +Native gate now exercises ten complete in-memory plans with tiny independent +six-record definition fixtures and five native skeleton donors. All nine planned +records compare exactly with canonical private cloning plus the specific four- +byte word at body offset 27; expected append indices 6/7/8, original donor ID and +next mesh ID checked. Ten mesh outputs again pass independent Python geometry, +weights and raw LOD0/ghost skeleton checks. Three plan refusals per donor (15) +cover missing body part, conflicting root name and existing mesh name. Thirty +mesh-preparation guards and five authored-name mismatch refusals remain exercised. +Final packet skinned-input-donors-x5qkf0vu passes, native graphics hash unchanged. +Private-clone and 66-case skin-input CTests PASS; final rebuilt harness also passes +the donor gate. Initial k8gb9tf4 fixture used default addEntry ordinals and was +correctly refused as noncanonical. Tiny fixture now saves/reopens before planning, +matching the core test setup; no production ordinal validation weakened. No native +definition corpus claim for this combined plan, no full archive copies, shipping +app rebuild or package update. Next: coordinated filesystem publication with fresh +source/destination proofs, texture validation, recovery/Restore and rollback tests. diff --git a/src/creatureimport.hpp b/src/creatureimport.hpp new file mode 100644 index 00000000..d7cc5171 --- /dev/null +++ b/src/creatureimport.hpp @@ -0,0 +1,55 @@ +#pragma once +#include "meshimport.hpp" +#include "forge/big.hpp" +#include "forge/defedit.hpp" +#include "forge/defdecode.hpp" +#include "forge/fmplinks.hpp" +#include +#include + +namespace albion::creatureimport { +struct MeshRequest { + size_t donorIndex=0,bodyPart=0,alternative=0; + std::string rootName,bodyName,appearanceName,meshName; + bool compress=true; +}; +struct MeshPlan { + forge::bin::File definitions; + forge::big::File graphics; + forge::defedit::CreatureVisualCloneResult clones; + forge::defedit::CreatureBodyModelResult binding; +}; +// All-or-nothing in-memory plan. Derives the mesh donor from the selected body +// slot in the privately cloned CCreatureDef, not an unrelated caller-chosen rig. +// Three definitions and one mesh append; only the selected new body model word +// differs from canonical private cloning. All source records remain const. +// Materials may refer to prepared texture IDs; coordinated publication must +// validate those IDs and all fresh source/destination/recovery/offline state. +// This neither publishes files nor establishes animation/runtime compatibility. +inline MeshPlan prepareMesh(const forge::bin::File& sourceDefinitions, + const forge::big::File& sourceGraphics,const forge::defschema::Schema& schema, + const MeshRequest& request,const meshimport::Model& model, + const std::vector& materials) { + auto definitions=sourceDefinitions; + const auto clones=forge::defedit::cloneCreatureVisuals(definitions,schema,request.donorIndex, + request.rootName,request.bodyName,request.appearanceName); + const auto& body=definitions.entries().at(clones.bodyIndex); + const auto decoded=forge::defdecode::decode(body,schema); + if(!decoded.clean() || forge::defdecode::encode(decoded)!=body.data) + throw std::runtime_error("creature mesh plan: body requires clean exact decode"); + const auto layout=forge::fmplinks::creaturePrefixLayout(decoded.prefix); + const auto slot=std::find_if(layout.bodyModels.begin(),layout.bodyModels.end(),[&](const auto& value){ + return value.bodyPart==request.bodyPart && value.alternative==request.alternative; + }); + if(slot==layout.bodyModels.end()) throw std::runtime_error("creature mesh plan: absent body part or alternative"); + uint32_t donorId=0; + for(size_t byte=0;byte<4;++byte) donorId|=uint32_t(body.data.at(slot->offset+byte))<<(8*byte); + auto mesh=meshimport::prepareSkinnedMesh(model,sourceGraphics,donorId,request.meshName,materials,request.compress); + const auto meshId=mesh.id; + auto graphics=sourceGraphics; + graphics.findBank("MBANK_ALLMESHES")->entries.push_back(std::move(mesh)); + const auto binding=forge::defedit::setCreatureBodyModel(definitions,schema,clones.bodyIndex, + request.bodyPart,request.alternative,meshId,graphics); + return {std::move(definitions),std::move(graphics),clones,binding}; +} +} // namespace albion::creatureimport diff --git a/tests/test_skinnedinput.cpp b/tests/test_skinnedinput.cpp index 5eb8cfe1..63190d98 100644 --- a/tests/test_skinnedinput.cpp +++ b/tests/test_skinnedinput.cpp @@ -1,6 +1,8 @@ #include "meshimport.hpp" #include "forge/meshpreview.hpp" #include "forge/big.hpp" +#include "creatureimport.hpp" +#include "forge/temporarydirectory.hpp" #include "nlohmann/json.hpp" #include #include @@ -63,11 +65,65 @@ static int donorGate(char** argv) { const auto donor=forge::meshpreview::decodeLod0(archive.entryData(*entry),5); require(bool(donor.skeletonBlocks),"missing donor raw skeleton"); const auto model=albion::meshimport::loadSkinnedModel(argv[4]); + // Small independent definition graph; the donor's real skeleton/payload + // remains lazy/read-only, with no full archive serialization or copies. + const auto schema=forge::defschema::Schema::loadText(R"({ + "CREATURE":{"fields":[{"name":"Components","type":"uint32"},{"name":"Self","type":"CDefIndex"},{"name":"Other","type":"int32"}]}, + "CCreatureDef":{"fields":[{"name":"Link","type":"CDefIndex"},{"name":"Other","type":"int32"}]}, + "CAppearanceDef":{"fields":[{"name":"Link","type":"CDefIndex"},{"name":"Other","type":"int32"}]} + })"); + const auto field=[](Bytes& bytes,const char* name,uint32_t value){word(bytes,forge::defdecode::fieldTag(name));word(bytes,value);}; + const auto root=[&](bool donorRoot) { + Bytes bytes{1,0,1,3,0}; + for(const auto& [name,index]:std::array,3>{{{"CCreatureDef",3},{"CAppearanceDef",4},{"CBrainDef",5}}}) { + word(bytes,forge::defdecode::fieldTag(name));word(bytes,index);word(bytes,donorRoot?1:0); + } + field(bytes,"Components",0);field(bytes,"Self",donorRoot?1:0);field(bytes,"Other",99);return bytes; + }; + Bytes body{1,0,1};field(body,"WoundMorphs",0);word(body,0x01ab6cee); + field(body,"RandomAppearanceMorph",1);word(body,1); + for(uint32_t value:{id,0u,0u,0u}) word(body,value);body.push_back(0); + body.insert(body.end(),27,0);word(body,0);body.insert(body.end(),13,0); + field(body,"Expressions",0);word(body,0x01ab6d32);field(body,"Stats",0); + field(body,"Link",1);field(body,"Other",99); + Bytes appearance{1,0,1};field(appearance,"Link",1);field(appearance,"Other",99); + forge::TemporaryDirectory definitionFixture(fs::temp_directory_path(),"creature-mesh-plan-"); + const auto namesPath=definitionFixture.path()/"names.bin",bankPath=definitionFixture.path()/"game.bin"; + const auto write=[&](const fs::path& path,const Bytes& bytes) { + std::ofstream stream(path,std::ios::binary);stream.write(reinterpret_cast(bytes.data()),std::streamsize(bytes.size())); + require(bool(stream),"definition plan fixture write failed"); + }; + Bytes emptyNames;for(uint32_t value:{0u,0xa8e36c34u,0u,4u,0u})word(emptyNames,value); + Bytes emptyBank(9);for(uint32_t value:{0u,1u,0u,0u,0u})word(emptyBank,value); + write(namesPath,emptyNames);write(bankPath,emptyBank); + auto definitions=forge::bin::File::open(namesPath,bankPath); + definitions.addEntry("CREATURE","ANCESTOR",root(false));definitions.addEntry("CREATURE","DONOR",root(true)); + definitions.addEntry("OTHER","OTHER",{});definitions.addEntry("CCreatureDef","",body); + definitions.addEntry("CAppearanceDef","",appearance);definitions.addEntry("CBrainDef","SHARED_BRAIN",{5,7}); + definitions.save(namesPath,bankPath);definitions=forge::bin::File::open(namesPath,bankPath); + albion::creatureimport::MeshRequest request{1,0,0,"CREATURE_IMPORTED","IMPORTED_BODY","IMPORTED_APPEARANCE","MESH_IMPORTED_"+idText,true}; const auto output=fs::absolute(argv[5]);require(!fs::exists(output),"donor output must be fresh"); forge::meshcompose::Material material;material.name="native donor import test"; std::vector meshes; for(bool compressed:{false,true}) { - auto mesh=albion::meshimport::prepareSkinnedMesh(model,archive,id,"MESH_IMPORTED_"+idText,{material},compressed); + request.compress=compressed; + const auto plan=albion::creatureimport::prepareMesh(definitions,archive,schema,request,model,{material}); + const auto& mesh=plan.graphics.findBank("MBANK_ALLMESHES")->entries.back(); + require(plan.definitions.entries().size()==9 && plan.clones.rootIndex==6 && plan.clones.bodyIndex==7 && plan.clones.appearanceIndex==8, + "private mesh graph append order changed"); + auto expectedDefinitions=definitions; + forge::defedit::cloneCreatureVisuals(expectedDefinitions,schema,1,request.rootName,request.bodyName,request.appearanceName); + auto expectedBody=expectedDefinitions.entries()[7].data; + for(size_t byte=0;byte<4;++byte) expectedBody.at(plan.binding.offset+byte)=uint8_t(mesh.id>>(8*byte)); + expectedDefinitions.setEntryData(7,std::move(expectedBody)); + for(size_t i=0;i<9;++i) { + const auto& expected=expectedDefinitions.entries()[i];const auto& actual=plan.definitions.entries()[i]; + require(expected.data==actual.data && expected.name==actual.name && expected.definition==actual.definition && + expected.indexInDefinition==actual.indexInDefinition,"private mesh plan changed unrelated definition bytes"); + } + require(plan.binding.oldModelId==id && plan.binding.newModelId==mesh.id && plan.binding.offset==27 && + plan.graphics.findBank("MBANK_ALLMESHES")->entries.size()==bank->entries.size()+1 && definitions.entries().size()==6, + "private mesh plan donor selection or source preservation changed"); uint32_t maximum=0;for(const auto& existing:bank->entries) maximum=std::max(maximum,existing.id); require(mesh.id==maximum+1 && mesh.type==5 && mesh.magic==42 && mesh.devFileType==entry->devFileType && mesh.length==mesh.data.size() && mesh.name=="MESH_IMPORTED_"+idText,"prepared entry metadata invalid"); @@ -76,8 +132,18 @@ static int donorGate(char** argv) { readback.skeletonBlocks->names==donor.skeletonBlocks->names && readback.skeletonBlocks->bones==donor.skeletonBlocks->bones && readback.skeletonBlocks->keyframes==donor.skeletonBlocks->keyframes && readback.skeletonBlocks->inverseBinds==donor.skeletonBlocks->inverseBinds, "imported donor spans changed"); - meshes.push_back(std::move(mesh)); + meshes.push_back(mesh); } + const auto planRefuses=[&](const albion::creatureimport::MeshRequest& invalid,const albion::meshimport::Model& authored,const char* reason) { + bool rejected=false; + try {albion::creatureimport::prepareMesh(definitions,archive,schema,invalid,authored,{material});} + catch(const std::exception& e) {rejected=std::string(e.what()).find(reason)!=std::string::npos;} + require(rejected && definitions.entries().size()==6 && archive.findBank("MBANK_ALLMESHES")->entries.size()==bank->entries.size(), + "private mesh refusal did not preserve source graph/bank"); + }; + auto invalidRequest=request;invalidRequest.bodyPart=99;planRefuses(invalidRequest,model,"absent body part or alternative"); + invalidRequest=request;invalidRequest.rootName="DONOR";planRefuses(invalidRequest,model,"name"); + invalidRequest=request;invalidRequest.meshName=entry->name;planRefuses(invalidRequest,model,"new mesh name already exists"); const auto refuses=[&](const forge::big::File& candidate,const std::string& name,const char* reason) { bool rejected=false; try {albion::meshimport::prepareSkinnedMesh(model,candidate,id,name,{material});} From f0a65690330187272d502c75314ebe94c16b30ac Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:23:31 -0600 Subject: [PATCH 44/75] feat(creature): publish guarded private mesh transactions --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 41 +++++++ src/creaturepublish.hpp | 126 ++++++++++++++++++++++ tests/test_skinnedinput.cpp | 67 ++++++++++++ tools/test_skinned_input_donors.py | 8 +- 4 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 src/creaturepublish.hpp diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 7dea657c..a72ed6c5 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -1070,3 +1070,44 @@ matching the core test setup; no production ordinal validation weakened. No nati definition corpus claim for this combined plan, no full archive copies, shipping app rebuild or package update. Next: coordinated filesystem publication with fresh source/destination proofs, texture validation, recovery/Restore and rollback tests. + +creatureimport::publishMesh implements coordinated names/game/graphics filesystem +publication for the existing-texture path. SourceProof captures resolved paths +and streaming SHA256 of all four input files (including textures); fresh hashes +and path identities must match before planning and immediately before install. +The publisher recomputes the canonical private mesh plan from source files, not +caller-provided modified banks. Explicit graphics output permits only the two +supported graphics.big layouts. Destination canonical paths stay inside the +resolved install root, cannot alias each other or hardlink/unrelated sources, +and existing outputs must equal the captured loaded inputs. Destination presence, +resolution and bytes are checked again before install. Running destination game +refuses. Missing outputs with recovery records refuse; creation markers and +original backups use the existing rollback-capable PendingBanks transaction. + +All nonzero material diffuse/bump/reflection/illumination/positive-decal IDs must +resolve uniquely in GBANK_MAIN_PC before writes. New texture append is explicitly +unfinished and must join this same transaction, not publish separately first. +Prepared definitions read back with every planned record exact and original raw +name-pool prefix preserved. Graphics readback compares every bank/entry identity, +native metadata and payload (physical offsets/container size may move). Staged +graphics fully reopened before install, so returned in-place results cannot retain +stale lazy source offsets and no fallible read occurs after the commit. This adds +RAM cost during serialization/readback, not repeated disk scratch bank copies. + +Final packet skinned-input-donors-wlpok87m PASS: ten fresh publications + exact +created-file Restore, five in-place publications + hash-exact baseline Restore, +five source-change and five destination-change barriers, five missing-texture +refusals, five Windows locked-third-file rollback cases. The first two targets +are replaced before graphics rename is forced to fail; all live destination bytes +return to their captured baselines. Original backups may remain after refusal; +no claim that backup creation is rolled back. Publication sources are tiny private +definition fixtures and one-entry graphics archives, not full retail banks. Native +graphics SHA256 unchanged; ten mesh payloads still pass independent addon checks. +Result manifest records publication counts and runner checks publication-stage +messages to refuse an older reader lacking these gates. Private visual clone and +66-case skin-input CTests also PASS. Initial srppv6oi rollback fixture compared +proofs from different resolved trees; its snapshot now uses the actual destination +tree and was rebuilt. Production source diagnostics identify input index and +path/content mismatch without relaxing checks. No release/shipping rebuild or +gameplay claim. Next: optional texture append in the same transaction, native +definition graph cases, CLI/GUI card wiring and placement/runtime validation. diff --git a/src/creaturepublish.hpp b/src/creaturepublish.hpp new file mode 100644 index 00000000..744330bb --- /dev/null +++ b/src/creaturepublish.hpp @@ -0,0 +1,126 @@ +#pragma once +#include "creatureimport.hpp" +#include "definitionclone.hpp" +#include "forge/modorder.hpp" +#include +#include + +namespace albion::creatureimport { +namespace fs=std::filesystem; +struct Sources {fs::path names,definitions,graphics,textures;}; +struct FileProof {fs::path resolved;std::string sha256;}; +using SourceProof=std::array; +inline SourceProof captureSources(const Sources& source) { + SourceProof proof;size_t index=0; + for(const auto& path:{source.names,source.definitions,source.graphics,source.textures}) { + if(!fs::is_regular_file(path)) throw std::runtime_error("creature publication: source is not a file"); + proof[index++]={paths::resolvedPath(path),forge::modorder::sha256Of(path)}; + } + return proof; +} +inline void requireSourceProof(const Sources& source,const SourceProof& expected) { + const auto current=captureSources(source); + for(size_t i=0;i& materials, + bool trackRecovery=true,const std::function& beforeInstall={}) { + const auto fail=[](const char* reason){throw std::runtime_error(std::string("creature publication: ")+reason);}; + if(source.names.filename()!="names.bin" || source.definitions.filename()!="game.bin" || + paths::resolvedPath(source.names.parent_path())!=paths::resolvedPath(source.definitions.parent_path())) + fail("source definitions require paired game.bin/names.bin"); + if(graphicsRelative!=fs::path("data/graphics/graphics.big") && graphicsRelative!=fs::path("data/graphics/pc/graphics.big")) + fail("unsupported graphics destination"); + const auto root=fs::absolute(destinationRoot).lexically_normal(),resolvedRoot=paths::resolvedPath(root); + const std::array targets{root/"data/CompiledDefs/names.bin",root/"data/CompiledDefs/game.bin", + root/graphicsRelative,root/"data/graphics/pc/textures.big"}; + std::array destinationProof;std::array present{}; + requireSourceProof(source,proof); + for(size_t i=0;i inputPaths{source.names,source.definitions,source.graphics,source.textures}; + for(size_t j=0;j0?uint32_t(material.decalId):0u}) if(id) { + if(!textureBank || std::count_if(textureBank->entries.begin(),textureBank->entries.end(),[&](const auto& e){return e.id==id;})!=1) + fail("missing or ambiguous material texture ID"); + } + } + auto plan=prepareMesh(forge::bin::File::open(source.names,source.definitions),forge::big::File::open(source.graphics),schema,request,model,materials); + detail::PendingBanks pending(root,".forge-creature-import-"); + const auto preparedNames=pending.prepare("data/CompiledDefs/names.bin"),preparedBank=pending.prepare("data/CompiledDefs/game.bin"); + plan.definitions.save(preparedNames,preparedBank); + definitionwrite::verifyCloneRecords(plan.definitions,forge::bin::File::open(preparedNames,preparedBank),plan.definitions.entries().size()); + const auto oldNames=definitionwrite::cloneBytes(source.names),newNames=definitionwrite::cloneBytes(preparedNames); + if(oldNames.size()<20 || newNames.size()(bytes.data()),std::streamsize(bytes.size())); + stream.close();if(!stream) fail("cannot prepare graphics bank");} + auto publishedGraphics=forge::big::File::openFully(preparedGraphics); + verifyGraphics(plan.graphics,publishedGraphics); + auto publishedDefinitions=forge::bin::File::open(preparedNames,preparedBank); + if(trackRecovery) for(size_t i=0;i<3;++i) if(!present[i]) { + const auto marker=pending.prepare(fs::path(targets[i].lexically_relative(root).string()+backups::kCreatedSuffix)); + std::ofstream stream(marker,std::ios::binary);stream<<"created by FableForge\n";stream.close();if(!stream) fail("cannot prepare creation marker"); + } + if(beforeInstall) beforeInstall();validate(); + std::string error;if(!pending.install(trackRecovery,error)) throw std::runtime_error("creature publication: "+error); + // No fallible read after commit, and no stale lazy offsets in an in-place + // result. Staged output was fully reopened and verified before installation. + plan.definitions=std::move(publishedDefinitions);plan.graphics=std::move(publishedGraphics); + return plan; +} +} // namespace albion::creatureimport diff --git a/tests/test_skinnedinput.cpp b/tests/test_skinnedinput.cpp index 63190d98..f17de828 100644 --- a/tests/test_skinnedinput.cpp +++ b/tests/test_skinnedinput.cpp @@ -2,6 +2,7 @@ #include "forge/meshpreview.hpp" #include "forge/big.hpp" #include "creatureimport.hpp" +#include "creaturepublish.hpp" #include "forge/temporarydirectory.hpp" #include "nlohmann/json.hpp" #include @@ -101,6 +102,14 @@ static int donorGate(char** argv) { definitions.addEntry("OTHER","OTHER",{});definitions.addEntry("CCreatureDef","",body); definitions.addEntry("CAppearanceDef","",appearance);definitions.addEntry("CBrainDef","SHARED_BRAIN",{5,7}); definitions.save(namesPath,bankPath);definitions=forge::bin::File::open(namesPath,bankPath); + forge::big::File tinyGraphics;tinyGraphics.setMagic("BIGB");tinyGraphics.setVersion(archive.version()); + auto tinyEntry=*entry;tinyEntry.data=archive.entryData(*entry); + tinyGraphics.addBank("MBANK_ALLMESHES",bank->id).entries.push_back(std::move(tinyEntry)); + forge::big::File tinyTextures;tinyTextures.setMagic("BIGB");tinyTextures.addBank("GBANK_MAIN_PC",0); + const auto graphicsPath=definitionFixture.path()/"graphics.big",texturesPath=definitionFixture.path()/"textures.big"; + write(graphicsPath,tinyGraphics.serialize());write(texturesPath,tinyTextures.serialize()); + const albion::creatureimport::Sources sources{namesPath,bankPath,graphicsPath,texturesPath}; + const auto proof=albion::creatureimport::captureSources(sources); albion::creatureimport::MeshRequest request{1,0,0,"CREATURE_IMPORTED","IMPORTED_BODY","IMPORTED_APPEARANCE","MESH_IMPORTED_"+idText,true}; const auto output=fs::absolute(argv[5]);require(!fs::exists(output),"donor output must be fresh"); forge::meshcompose::Material material;material.name="native donor import test"; @@ -133,7 +142,65 @@ static int donorGate(char** argv) { readback.skeletonBlocks->keyframes==donor.skeletonBlocks->keyframes && readback.skeletonBlocks->inverseBinds==donor.skeletonBlocks->inverseBinds, "imported donor spans changed"); meshes.push_back(mesh); + const auto publishRoot=definitionFixture.path()/(compressed?"compressed-publish":"stored-publish"); + const auto published=albion::creatureimport::publishMesh(sources,proof,publishRoot,"data/graphics/graphics.big",schema,request,model,{material}); + require(published.definitions.entries().size()==9 && published.binding.oldModelId==id && published.binding.newModelId==id+1 && + published.graphics.findBank("MBANK_ALLMESHES")->entries.size()==2,"coordinated publication result differs"); + albion::creatureimport::requireSourceProof(sources,proof); + for(const auto& path:std::array{publishRoot/"data/CompiledDefs/names.bin",publishRoot/"data/CompiledDefs/game.bin",publishRoot/"data/graphics/graphics.big"}) + require(fs::is_regular_file(fs::path(path.string()+albion::backups::kCreatedSuffix)),"coordinated publication missing creation marker"); + std::vector restoreNotes;std::string restoreError; + require(albion::backups::restoreAll(publishRoot,true,restoreNotes,restoreError)==3 && restoreError.empty(),"coordinated created outputs did not Restore"); + require(!fs::exists(publishRoot/"data/CompiledDefs/game.bin") && !fs::exists(publishRoot/"data/graphics/graphics.big"),"Restore kept created creature files"); } + const auto originalNames=albion::definitionwrite::cloneBytes(namesPath); + const auto staleRoot=definitionFixture.path()/"stale-publish"; + bool staleRefused=false; + try {albion::creatureimport::publishMesh(sources,proof,staleRoot,"data/graphics/graphics.big",schema,request,model,{material},true,[&]{ + std::ofstream stream(namesPath,std::ios::binary|std::ios::app);stream.put('x');stream.close();require(bool(stream),"stale fixture update failed"); + });}catch(const std::exception& e){staleRefused=std::string(e.what()).find("source changed")!=std::string::npos;} + write(namesPath,originalNames); + require(staleRefused && !fs::exists(staleRoot/"data/CompiledDefs/game.bin") && !fs::exists(staleRoot/"data/graphics/graphics.big"),"stale coordinated publication installed files"); + albion::creatureimport::requireSourceProof(sources,proof); + const auto destinationStaleRoot=definitionFixture.path()/"destination-stale-publish"; + bool destinationRefused=false; + try {albion::creatureimport::publishMesh(sources,proof,destinationStaleRoot,"data/graphics/graphics.big",schema,request,model,{material},true,[&]{ + fs::create_directories(destinationStaleRoot/"data/CompiledDefs");write(destinationStaleRoot/"data/CompiledDefs/game.bin",Bytes{7,8,9}); + });}catch(const std::exception& e){destinationRefused=std::string(e.what()).find("destination changed")!=std::string::npos;} + require(destinationRefused && albion::definitionwrite::cloneBytes(destinationStaleRoot/"data/CompiledDefs/game.bin")==Bytes{7,8,9} && + !fs::exists(destinationStaleRoot/"data/graphics/graphics.big"),"stale destination overwritten during creature publication"); + auto missingTexture=material;missingTexture.diffuseId=99;bool textureRefused=false; + const auto textureRefusalRoot=definitionFixture.path()/"missing-texture-publish"; + try {albion::creatureimport::publishMesh(sources,proof,textureRefusalRoot,"data/graphics/graphics.big",schema,request,model,{missingTexture});} + catch(const std::exception& e){textureRefused=std::string(e.what()).find("missing or ambiguous material texture ID")!=std::string::npos;} + require(textureRefused && !fs::exists(textureRefusalRoot),"invalid material texture published"); + const auto existingTree=[&](const fs::path& root) { + fs::create_directories(root/"data/CompiledDefs");fs::create_directories(root/"data/graphics/pc"); + fs::copy_file(namesPath,root/"data/CompiledDefs/names.bin");fs::copy_file(bankPath,root/"data/CompiledDefs/game.bin"); + fs::copy_file(graphicsPath,root/"data/graphics/graphics.big");fs::copy_file(texturesPath,root/"data/graphics/pc/textures.big"); + return albion::creatureimport::Sources{root/"data/CompiledDefs/names.bin",root/"data/CompiledDefs/game.bin",root/"data/graphics/graphics.big",root/"data/graphics/pc/textures.big"}; + }; + const auto inPlaceRoot=definitionFixture.path()/"in-place";const auto inPlaceSources=existingTree(inPlaceRoot); + std::cout<<"publication in-place and Restore: "<entries.front())==archive.entryData(*entry),"in-place result retained stale lazy offsets"); + std::vector restoreNotes;std::string restoreError; + require(albion::backups::restoreAll(inPlaceRoot,true,restoreNotes,restoreError)==3 && restoreError.empty(),"in-place creature Restore failed"); + albion::creatureimport::requireSourceProof(inPlaceSources,inPlaceProof); +#ifdef _WIN32 + const auto lockedRoot=definitionFixture.path()/"locked-output";const auto lockedSources=existingTree(lockedRoot); + std::cout<<"publication locked-third-file rollback: "< Date: Sat, 10 Oct 2026 14:30:11 -0600 Subject: [PATCH 45/75] feat(creature): append diffuse textures in coordinated imports --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 39 ++++++++++++++ src/creatureimport.hpp | 2 + src/creaturepublish.hpp | 57 +++++++++++++++++--- tests/test_skinnedinput.cpp | 64 ++++++++++++++++++++++- tools/test_skinned_input_donors.py | 50 +++++++++++++++++- 5 files changed, 203 insertions(+), 9 deletions(-) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index a72ed6c5..af438905 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -1111,3 +1111,42 @@ tree and was rebuilt. Production source diagnostics identify input index and path/content mismatch without relaxing checks. No release/shipping rebuild or gameplay claim. Next: optional texture append in the same transaction, native definition graph cases, CLI/GUI card wiring and placement/runtime validation. + +Optional texture append now participates in publishMesh's same transaction. +captureTexture binds the selected image path/hash to a draft; commit revalidates +its resolved identity and bytes alongside all source/destination proofs. A unique +uppercase/number/underscore symbol (<80 bytes) appends to nonempty BIGB +GBANK_MAIN_PC through native terraintex::importPng. Reserves the next bank-local +ID before composing; every ordinary material diffuse uses that ID, while other +maps must resolve in the original texture bank. DXT1, DXT3 and ARGB encoding use +the existing native writer. Imported ID/type/name/count and texture validation +checked; all prior bank/entry metadata and payloads must survive exactly. New +texture bytes are staged first, then names/game/graphics; one PendingBanks install +handles all four, including backups/creation markers and rollback. No separate +texture publication precedes creature publication. Returned plan owns fully read +texture bytes and the new ID, avoiding stale in-place offsets. Exactly the selected +new-body slot/alternative changes; other body alternatives remain donor-derived. + +Final _m23l9ws native-donor packet PASS: fifteen fresh four-file publications and +Restore (three formats x five donors), five in-place four-file Restore/hash checks, +five image-change barriers and five existing-name refusals. Five Windows locked +fourth-file cases roll back earlier texture/names/game replacements to their exact +baseline hashes. Prior ten mesh outputs, three-file publication/Restore, stale +source/destination and missing-existing-ID guards remain PASS. Publication texture +sources are one-entry 4x4 blue fixtures, not native texture-bank copies; tiny +published archives retained per format for independent Python checks. Native +graphics SHA unchanged; private visual and 66-case skin-input CTests PASS. + +Independent Python pixel/descriptor checks decode preserved blue and appended +red/green stripe images for all fifteen outputs and verify material texture ID 8. +First 2h00g95a packet exposed an addon TextureBank.decode issue: it pads the LZO +result with zeros rather than consuming Fable's three raw tail bytes. The test now +uses that addon's correctly framed _lzo_fable reader followed by its independent +parse_texture pixel decoder; exact tail handling checked against mip-0 span size. +No external oracle files or production texture encoder changed, no pixel tolerance +relaxed (0.01 normalized channel tolerance for DXT quantization). Runner manifests +now include texture publication counts and stage-presence guards against an older +harness. Existing texture assets remain unchanged except the single appended slot. +No shipping/release rebuild, full native definition corpus or runtime claim. Next: +CLI wiring and native definition-graph cases, then the donor/mesh/texture GUI card, +placement and runtime validation. New-asset flow is not complete without that UI. diff --git a/src/creatureimport.hpp b/src/creatureimport.hpp index d7cc5171..b4e2ec8e 100644 --- a/src/creatureimport.hpp +++ b/src/creatureimport.hpp @@ -18,6 +18,8 @@ struct MeshPlan { forge::big::File graphics; forge::defedit::CreatureVisualCloneResult clones; forge::defedit::CreatureBodyModelResult binding; + std::optional textures; // publication with texture append only + uint32_t textureId=0; }; // All-or-nothing in-memory plan. Derives the mesh donor from the selected body // slot in the privately cloned CCreatureDef, not an unrelated caller-chosen rig. diff --git a/src/creaturepublish.hpp b/src/creaturepublish.hpp index 744330bb..3e170fa4 100644 --- a/src/creaturepublish.hpp +++ b/src/creaturepublish.hpp @@ -2,6 +2,7 @@ #include "creatureimport.hpp" #include "definitionclone.hpp" #include "forge/modorder.hpp" +#include "forge/terraintex.hpp" #include #include @@ -9,6 +10,11 @@ namespace albion::creatureimport { namespace fs=std::filesystem; struct Sources {fs::path names,definitions,graphics,textures;}; struct FileProof {fs::path resolved;std::string sha256;}; +struct TextureAppend {fs::path image;std::string name,format;FileProof proof;}; +inline TextureAppend captureTexture(const fs::path& image,std::string name,std::string format="dxt1") { + if(!fs::is_regular_file(image)) throw std::runtime_error("creature publication: texture image is not a file"); + return {image,std::move(name),std::move(format),{paths::resolvedPath(image),forge::modorder::sha256Of(image)}}; +} using SourceProof=std::array; inline SourceProof captureSources(const Sources& source) { SourceProof proof;size_t index=0; @@ -40,14 +46,14 @@ inline void verifyGraphics(const forge::big::File& expected,const forge::big::Fi } } } -// Existing-texture path of the coordinated import. New texture append remains -// to be added to this same transaction, not published as an earlier operation. +// Optional diffuse texture is appended inside the same staged transaction. +// Other material maps continue to refer to unique existing texture IDs. // Expected source proof is captured when the caller selects its inputs/draft. inline MeshPlan publishMesh(const Sources& source,const SourceProof& proof, const fs::path& destinationRoot,const fs::path& graphicsRelative, const forge::defschema::Schema& schema,const MeshRequest& request, const meshimport::Model& model,const std::vector& materials, - bool trackRecovery=true,const std::function& beforeInstall={}) { + bool trackRecovery=true,const std::function& beforeInstall={},const TextureAppend* textureAppend=nullptr) { const auto fail=[](const char* reason){throw std::runtime_error(std::string("creature publication: ")+reason);}; if(source.names.filename()!="names.bin" || source.definitions.filename()!="game.bin" || paths::resolvedPath(source.names.parent_path())!=paths::resolvedPath(source.definitions.parent_path())) @@ -83,6 +89,8 @@ inline MeshPlan publishMesh(const Sources& source,const SourceProof& proof, const auto validate=[&] { if(backups::gameRunningIn(root) || backups::gameRunningIn(resolvedRoot)) fail("close Fable before publishing"); requireSourceProof(source,proof); + if(textureAppend && (!fs::is_regular_file(textureAppend->image) || paths::resolvedPath(textureAppend->image)!=textureAppend->proof.resolved || + forge::modorder::sha256Of(textureAppend->image)!=textureAppend->proof.sha256)) fail("texture source changed"); for(size_t i=0;ientries.empty()) fail("texture append requires nonempty BIGB GBANK_MAIN_PC"); + if(textureAppend->name.empty() || textureAppend->name.size()>=80 || + !std::all_of(textureAppend->name.begin(),textureAppend->name.end(),[](char c){return (c>='A' && c<='Z') || (c>='0' && c<='9') || c=='_';})) + fail("invalid new texture name"); + for(const auto& bank:textures.banks()) for(const auto& entry:bank.entries) + if(entry.name==textureAppend->name) fail("new texture name already exists"); + for(const auto& entry:textureBank->entries) appendedTextureId=std::max(appendedTextureId,entry.id); + if(appendedTextureId==UINT32_MAX) fail("no free texture entry ID"); + ++appendedTextureId; + for(auto& material:effectiveMaterials) material.diffuseId=appendedTextureId; + } for(const auto& material:materials) { - for(uint32_t id:{material.diffuseId,material.bumpId,material.reflectId,material.illumId, + for(uint32_t id:{textureAppend?0u:material.diffuseId,material.bumpId,material.reflectId,material.illumId, material.decalId>0?uint32_t(material.decalId):0u}) if(id) { if(!textureBank || std::count_if(textureBank->entries.begin(),textureBank->entries.end(),[&](const auto& e){return e.id==id;})!=1) fail("missing or ambiguous material texture ID"); } } - auto plan=prepareMesh(forge::bin::File::open(source.names,source.definitions),forge::big::File::open(source.graphics),schema,request,model,materials); + auto plan=prepareMesh(forge::bin::File::open(source.names,source.definitions),forge::big::File::open(source.graphics),schema,request,model,effectiveMaterials); detail::PendingBanks pending(root,".forge-creature-import-"); + std::optional publishedTextures; + if(textureAppend) { + const auto preparedTexture=pending.prepare("data/graphics/pc/textures.big"); + forge::terraintex::ImportRequest textureRequest; + textureRequest.png=textureAppend->image;textureRequest.srcBig=source.textures;textureRequest.outBig=preparedTexture; + textureRequest.entryName=textureAppend->name;textureRequest.format=textureAppend->format;textureRequest.add=true; + const auto imported=forge::terraintex::importPng(textureRequest); + if(!imported.ok) { + std::string error="creature publication: texture import failed"; + for(const auto& reason:imported.validation.errors) error+="; "+reason; + throw std::runtime_error(error); + } + publishedTextures=forge::big::File::openFully(preparedTexture); + const auto* preparedBank=publishedTextures->findBank("GBANK_MAIN_PC"); + if(!preparedBank || preparedBank->entries.size()!=textureBank->entries.size()+1 || imported.entryId!=appendedTextureId || + preparedBank->entries.back().id!=appendedTextureId || preparedBank->entries.back().name!=textureAppend->name || preparedBank->entries.back().type!=0) + fail("prepared texture append differs"); + auto expectedTextures=textures; + expectedTextures.findBank("GBANK_MAIN_PC")->entries.push_back(preparedBank->entries.back()); + // Own payload for the added record, rather than its staged file offsets. + auto& added=expectedTextures.findBank("GBANK_MAIN_PC")->entries.back();added.data=publishedTextures->entryData(added); + verifyGraphics(expectedTextures,*publishedTextures); + } const auto preparedNames=pending.prepare("data/CompiledDefs/names.bin"),preparedBank=pending.prepare("data/CompiledDefs/game.bin"); plan.definitions.save(preparedNames,preparedBank); definitionwrite::verifyCloneRecords(plan.definitions,forge::bin::File::open(preparedNames,preparedBank),plan.definitions.entries().size()); @@ -112,7 +156,7 @@ inline MeshPlan publishMesh(const Sources& source,const SourceProof& proof, auto publishedGraphics=forge::big::File::openFully(preparedGraphics); verifyGraphics(plan.graphics,publishedGraphics); auto publishedDefinitions=forge::bin::File::open(preparedNames,preparedBank); - if(trackRecovery) for(size_t i=0;i<3;++i) if(!present[i]) { + if(trackRecovery) for(size_t i=0;i<(textureAppend?4u:3u);++i) if(!present[i]) { const auto marker=pending.prepare(fs::path(targets[i].lexically_relative(root).string()+backups::kCreatedSuffix)); std::ofstream stream(marker,std::ios::binary);stream<<"created by FableForge\n";stream.close();if(!stream) fail("cannot prepare creation marker"); } @@ -121,6 +165,7 @@ inline MeshPlan publishMesh(const Sources& source,const SourceProof& proof, // No fallible read after commit, and no stale lazy offsets in an in-place // result. Staged output was fully reopened and verified before installation. plan.definitions=std::move(publishedDefinitions);plan.graphics=std::move(publishedGraphics); + plan.textures=std::move(publishedTextures);plan.textureId=appendedTextureId; return plan; } } // namespace albion::creatureimport diff --git a/tests/test_skinnedinput.cpp b/tests/test_skinnedinput.cpp index f17de828..7a1261e7 100644 --- a/tests/test_skinnedinput.cpp +++ b/tests/test_skinnedinput.cpp @@ -4,6 +4,7 @@ #include "creatureimport.hpp" #include "creaturepublish.hpp" #include "forge/temporarydirectory.hpp" +#include "forge/texturewrite.hpp" #include "nlohmann/json.hpp" #include #include @@ -105,7 +106,13 @@ static int donorGate(char** argv) { forge::big::File tinyGraphics;tinyGraphics.setMagic("BIGB");tinyGraphics.setVersion(archive.version()); auto tinyEntry=*entry;tinyEntry.data=archive.entryData(*entry); tinyGraphics.addBank("MBANK_ALLMESHES",bank->id).entries.push_back(std::move(tinyEntry)); - forge::big::File tinyTextures;tinyTextures.setMagic("BIGB");tinyTextures.addBank("GBANK_MAIN_PC",0); + forge::big::File tinyTextures;tinyTextures.setMagic("BIGB"); + forge::texturewrite::Image blue;blue.width=blue.height=4; + for(size_t pixel=0;pixel<16;++pixel) blue.rgba.insert(blue.rgba.end(),{0,0,255,255}); + const auto builtBlue=forge::texturewrite::buildEntry(blue,forge::texturewrite::kFormatARGB); + forge::big::Entry blueEntry;blueEntry.id=7;blueEntry.name="EXISTING";blueEntry.magic=42;blueEntry.devCrc=123; + blueEntry.devSources={"retained source"};blueEntry.data=builtBlue.payload;blueEntry.subHeader=builtBlue.info;blueEntry.length=uint32_t(blueEntry.data.size()); + tinyTextures.addBank("GBANK_MAIN_PC",9).entries.push_back(std::move(blueEntry)); const auto graphicsPath=definitionFixture.path()/"graphics.big",texturesPath=definitionFixture.path()/"textures.big"; write(graphicsPath,tinyGraphics.serialize());write(texturesPath,tinyTextures.serialize()); const albion::creatureimport::Sources sources{namesPath,bankPath,graphicsPath,texturesPath}; @@ -201,6 +208,61 @@ static int donorGate(char** argv) { require(rollbackRefused,"locked third publication target accepted"); albion::creatureimport::requireSourceProof(lockedSources,lockedProof); #endif + const auto imagePath=fs::path(argv[4]).parent_path()/"diffuse.png"; + std::cout<<"publication texture append, Restore and refusal: "<findBank("GBANK_MAIN_PC")->entries.size()==2, + "texture append result differs"); + const auto& texture=published.textures->findBank("GBANK_MAIN_PC")->entries.back(); + require(forge::terraintex::validateTextureEntry(texture.subHeader,published.textures->entryData(texture)).ok,"appended texture validation failed"); + const auto textureMesh=forge::meshpreview::decodeLod0(published.graphics.entryData(published.graphics.findBank("MBANK_ALLMESHES")->entries.back()),5); + require(textureMesh.materials.size()==1 && textureMesh.materials[0].diffuseTexture==8,"new diffuse ID not bound to composed mesh"); + // Retain tiny published files for independent Python archive/texture + // decoding before Restore removes the actual publication tree. + const auto retained=fs::path(argv[4]).parent_path()/(std::string("texture-output-")+format); + require(fs::create_directory(retained),"texture evidence output must be fresh"); + fs::copy_file(textureRoot/"data/graphics/pc/textures.big",retained/"textures.big"); + fs::copy_file(textureRoot/"data/graphics/graphics.big",retained/"graphics.big"); + restoreNotes.clear();restoreError.clear(); + require(albion::backups::restoreAll(textureRoot,true,restoreNotes,restoreError)==4 && restoreError.empty(),"four-file texture publication Restore failed"); + require(!fs::exists(textureRoot/"data/graphics/pc/textures.big"),"Restore kept newly created textures bank"); + albion::creatureimport::requireSourceProof(sources,proof); + } + auto append=albion::creatureimport::captureTexture(imagePath,"IMPORTED_DIFFUSE","argb8888"); + const auto pngBytes=albion::definitionwrite::cloneBytes(imagePath);bool imageStaleRefused=false; + const auto imageStaleRoot=definitionFixture.path()/"image-stale"; + try {albion::creatureimport::publishMesh(sources,proof,imageStaleRoot,"data/graphics/graphics.big",schema,request,model,{material},true,[&]{ + std::ofstream stream(imagePath,std::ios::binary|std::ios::app);stream.put('x');stream.close();require(bool(stream),"image stale update failed"); + },&append);}catch(const std::exception& e){imageStaleRefused=std::string(e.what()).find("texture source changed")!=std::string::npos;} + write(imagePath,pngBytes);require(imageStaleRefused && !fs::exists(imageStaleRoot/"data/graphics/pc/textures.big"),"stale image published a texture bank"); + const auto textureInPlaceRoot=definitionFixture.path()/"texture-in-place"; + const auto textureInPlaceSources=existingTree(textureInPlaceRoot);const auto textureInPlaceProof=albion::creatureimport::captureSources(textureInPlaceSources); + const auto textureInPlace=albion::creatureimport::publishMesh(textureInPlaceSources,textureInPlaceProof,textureInPlaceRoot,"data/graphics/graphics.big",schema,request,model,{material},true,{},&append); + require(textureInPlace.textures && textureInPlace.textures->entryData(textureInPlace.textures->findBank("GBANK_MAIN_PC")->entries.front())==tinyTextures.entryData(tinyTextures.findBank("GBANK_MAIN_PC")->entries.front()), + "in-place texture return lost unchanged payload"); + restoreNotes.clear();restoreError.clear(); + require(albion::backups::restoreAll(textureInPlaceRoot,true,restoreNotes,restoreError)==4 && restoreError.empty(),"in-place four-file Restore failed"); + albion::creatureimport::requireSourceProof(textureInPlaceSources,textureInPlaceProof); +#ifdef _WIN32 + std::cout<<"publication texture fourth-file rollback: "<I', len(payload))+kind+payload+struct.pack('>I', zlib.crc32(kind+payload)) + row = b'\0'+bytes([255,0,0,255]*2+[0,255,0,255]*2) + png = b'\x89PNG\r\n\x1a\n'+png_chunk(b'IHDR', struct.pack('>2I5B',4,4,8,6,0,0,0)) + png += png_chunk(b'IDAT', zlib.compress(row*4))+png_chunk(b'IEND', b'') + (folder/'diffuse.png').write_bytes(png) return doc @@ -99,6 +106,43 @@ def run(path, output): assert 'publication in-place and Restore:' in result.stdout, 'reader lacks coordinated publication gate' if os.name=='nt': assert 'publication locked-third-file rollback:' in result.stdout, 'reader lacks Windows rollback gate' + assert 'publication texture append, Restore and refusal:' in result.stdout, 'reader lacks texture publication gate' + if os.name=='nt': + assert 'publication texture fourth-file rollback:' in result.stdout, 'reader lacks four-file rollback gate' + for format_name, fmt in (('dxt1',0x1f),('dxt3',0x20),('argb8888',1)): + texture_output = folder/('texture-output-'+format_name) + texture_bank = fc.TextureBank([str(texture_output/'textures.big')]) + assert texture_bank.count()==2 and texture_bank.entry_name(8)=='IMPORTED_DIFFUSE' + info = fc.parse_texture.parse_info(texture_bank._by_id[8][2]['info']) + assert info['fmt']==fmt + def pixels(tex_id): + # TextureBank.decode pads the LZO result with zeros and drops + # Fable's three raw tail bytes. Use the addon's correct framed + # block reader, then its independent pixel-format decoder. + buf, _, entry, _ = texture_bank._by_id[tex_id] + desc = fc.parse_texture.parse_info(entry['info']) + assert desc['alloc_w']==4 and desc['alloc_h']==4 + pay = buf[entry['offset']:entry['offset']+entry['size']] + raw_size = fc._base_mip_raw_len(desc['fmt'], 4, 4) + if desc['first_block_field']: + raw, cursor = fc._lzo_fable(pay,0,raw_size) + assert cursor==desc['first_block_field'] + else: + raw = pay[:raw_size] + assert len(raw)==raw_size + return fc.rgba_to_blender(fc.parse_texture.decode_mip(desc['fmt'],raw,4,4)) + old = pixels(7) + new = pixels(8) + assert all(abs(value-[0,0,1,1][i%4])<.01 for i,value in enumerate(old)) + expected_row = [1,0,0,1]*2+[0,1,0,1]*2 + assert all(abs(value-expected_row[i%16])<.01 for i,value in enumerate(new)), 'independent texture pixels differ' + graphics_bytes, graphics_entries = fc.open_graphics_big(str(texture_output/'graphics.big')) + composed_entry = max(graphics_entries,key=lambda e:e['id']) + pay = graphics_bytes[composed_entry['offset']:composed_entry['offset']+composed_entry['size']] + _, texture_prims, _ = fc._parse_mesh_primitives(pay,5) + assert len(texture_prims)==1 + mesh_model = fc.build_model(graphics_bytes, composed_entry, texture_bank, decode_textures=False) + assert len(mesh_model.materials)==1 and mesh_model.materials[0].base_tex_id==8 for mode in ('stored','compressed'): payload_path = output/(mode+'.mesh') payload = payload_path.read_bytes() @@ -133,7 +177,9 @@ def run(path, output): assert len(checks)==10 and digest(graphics)==before (work/'result.json').write_text(json.dumps({'passed': True, 'graphics_sha256': before, 'checks': checks, 'refusals': 5, 'publication': {'fresh_restore': 10, 'in_place_restore': 5, 'source_stale': 5, 'destination_stale': 5, - 'missing_texture': 5, 'locked_third_file_rollback': 5 if os.name=='nt' else 0}}, indent=2)+'\n', encoding='utf-8') + 'missing_texture': 5, 'locked_third_file_rollback': 5 if os.name=='nt' else 0, + 'texture_fresh_restore': 15, 'texture_in_place_restore': 5, 'texture_image_stale': 5, + 'texture_name_refusal': 5, 'texture_fourth_file_rollback': 5 if os.name=='nt' else 0}}, indent=2)+'\n', encoding='utf-8') if __name__=='__main__': From 50bc962c9d6bec98a0f5044047bfbf40bd697d3e Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:37:02 -0600 Subject: [PATCH 46/75] Expose coordinated creature import through explicit CLI selectors --- docs/CLI.md | 2 + docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 14 ++++ src/cli/common.cpp | 2 + src/cli/common.hpp | 1 + src/cli/creatures.cpp | 83 +++++++++++++++++++++++ src/main.cpp | 1 + tools/test_creature_import_arguments.py | 57 ++++++++++++++++ 7 files changed, 160 insertions(+) create mode 100644 src/cli/creatures.cpp create mode 100644 tools/test_creature_import_arguments.py diff --git a/docs/CLI.md b/docs/CLI.md index 2cfa46af..bcafd1c4 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -26,6 +26,8 @@ usage: (a ground theme from your own texture: appended to textures.big + a new ENGINE_THEME in game.bin; paint it from the editor) forge mesh-import [--texture | --texture-id ] [--donor ] [--stored] [--no-collision] [--install ] forge mesh-replace [--stored] [--install ] (static geometry; preserves materials and collision) + forge creature-import --donor-index --body --alternative --schema --install --out [--texture | --texture-id ] [--texture-format dxt1|dxt3|argb8888] [--stored] [--json] + (private root/body/appearance clone + one selected body mesh; other body slots and animations remain shared; runtime compatibility unverified) (a custom static object: MESH_ appended to graphics.big, its diffuse to textures.big, a collision hull from its own triangles, an OBJECT_ def copied from the donor; Y-up model space) forge textures [filter] [--bank ] (list textures.big entries: id, size, format, bank) forge texture-export | texture-replace | texture-add [--bank B] [--format dxt1|dxt3|argb8888] diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index af438905..696bc0e8 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -1150,3 +1150,17 @@ harness. Existing texture assets remain unchanged except the single appended slo No shipping/release rebuild, full native definition corpus or runtime claim. Next: CLI wiring and native definition-graph cases, then the donor/mesh/texture GUI card, placement and runtime validation. New-asset flow is not complete without that UI. + +CLI continuation: forge creature-import now dispatches explicit donor-index, +body-part/alternative, schema, install and output selections to the coordinated +publisher. Optional image/format or existing texture ID uses the same transaction; +JSON and human reports explicitly retain shared slots/animations and unverified +runtime compatibility. Strict decimal selectors, duplicate/unknown/missing options, +texture conflicts and ASCII names refuse. No extra mesh decode occurs after commit. +Owned forge target builds successfully. tools/test_creature_import_arguments.py +passes 22 argument refusals plus help from an unrelated temporary cwd, with no +output creation. Initial help check exposed usage's nonzero status; command-specific +help now returns zero and the rerun passes. Generated docs/CLI.md refreshed. +These are argument-only checks, not positive CLI import acceptance; next retain +tiny donor source fixtures and independently test successful CLI publications and +Restore before GUI wiring. Native graph, package and runtime gates remain open. diff --git a/src/cli/common.cpp b/src/cli/common.cpp index 20fe8cd7..a79c9c96 100644 --- a/src/cli/common.cpp +++ b/src/cli/common.cpp @@ -69,6 +69,8 @@ int usage() { " (a ground theme from your own texture: appended to textures.big + a new ENGINE_THEME in game.bin; paint it from the editor)\n" " forge mesh-import [--texture | --texture-id ] [--donor ] [--stored] [--no-collision] [--install ]\n" " forge mesh-replace [--stored] [--install ] (static geometry; preserves materials and collision)\n" + " forge creature-import --donor-index --body --alternative --schema --install --out [--texture | --texture-id ] [--texture-format dxt1|dxt3|argb8888] [--stored] [--json]\n" + " (private root/body/appearance clone + one selected body mesh; other body slots and animations remain shared; runtime compatibility unverified)\n" " (a custom static object: MESH_ appended to graphics.big, its diffuse to textures.big, a collision hull from its own triangles, an OBJECT_ def copied from the donor; Y-up model space)\n" " forge textures [filter] [--bank ] (list textures.big entries: id, size, format, bank)\n" " forge texture-export | texture-replace | texture-add [--bank B] [--format dxt1|dxt3|argb8888]\n" diff --git a/src/cli/common.hpp b/src/cli/common.hpp index e6e2d0ea..5c6fd98d 100644 --- a/src/cli/common.hpp +++ b/src/cli/common.hpp @@ -33,6 +33,7 @@ using Args = std::vector; std::optional runLevels(const std::string& cmd, const Args& args); // new levels and region entrances std::optional runTextures(const std::string& cmd, const Args& args); // textures.big and ground themes std::optional runInstall(const std::string& cmd, const Args& args); // backups and restore +std::optional runCreatures(const std::string& cmd,const Args& args); std::optional runWorld(const std::string& cmd, const Args& args); // the overworld (WLD/BWD) and regions std::optional runNav(const std::string& cmd, const Args& args); // navigation diagnostics std::optional runChunks(const std::string& cmd, const Args& args); // STB terrain chunk diagnostics and bakes diff --git a/src/cli/creatures.cpp b/src/cli/creatures.cpp new file mode 100644 index 00000000..1ad5ffca --- /dev/null +++ b/src/cli/creatures.cpp @@ -0,0 +1,83 @@ +#include "cli/common.hpp" +#include "creaturepublish.hpp" +#include "nlohmann/json.hpp" +#include +#include +#include +#include +#include + +namespace albion::cli { +namespace fs = std::filesystem; +std::optional runCreatures(const std::string& command,const Args& args) { + if(command!="creature-import") return {}; + if(args.size()==2 && args[1]=="--help") { usage(); return 0; } + const auto fail=[](const std::string& reason){throw std::runtime_error("creature-import: "+reason);}; + std::map options;std::set flags; + std::vector positional; + for(size_t i=1;i=args.size() || args[i].empty() || args[i].starts_with("--")) fail("missing value for "+arg); + options.emplace(arg,args[i]); + } else positional.push_back(arg); + } + if(positional.size()!=2) fail("expected model.glb|.gltf and NAME"); + for(const auto& option:{"--install","--out","--schema","--donor-index","--body","--alternative"}) + if(!options.contains(option)) fail(std::string("required option ")+option); + if(options.contains("--texture") && options.contains("--texture-id")) fail("choose --texture or --texture-id"); + if(options.contains("--texture-format") && !options.contains("--texture")) fail("--texture-format requires --texture"); + const auto& name=positional[1]; + if(name.empty() || name.size()>=80 || !std::all_of(name.begin(),name.end(),[](char c){return (c>='A' && c<='Z') || (c>='0' && c<='9') || c=='_';})) + fail("NAME must be 1-79 uppercase letters, numbers or underscores"); + const auto number=[&](const std::string& key,size_t maximum=std::numeric_limits::max()) { + const auto& value=options.at(key);size_t parsed=0; + const auto result=std::from_chars(value.data(),value.data()+value.size(),parsed); + if(result.ec!=std::errc{} || result.ptr!=value.data()+value.size() || parsed>maximum) fail("invalid unsigned decimal "+key); + return parsed; + }; + creatureimport::MeshRequest request; + request.donorIndex=number("--donor-index");request.bodyPart=number("--body");request.alternative=number("--alternative"); + request.rootName="CREATURE_"+name;request.bodyName="FORGE_"+name+"_BODY"; + request.appearanceName="FORGE_"+name+"_APPEARANCE";request.meshName="MESH_"+name;request.compress=!flags.contains("--stored"); + const uint32_t existingTexture=options.contains("--texture-id")?uint32_t(number("--texture-id",UINT32_MAX)):0; + const auto install=fs::absolute(fs::path(options.at("--install"))).lexically_normal(); + const auto output=fs::absolute(fs::path(options.at("--out"))).lexically_normal(); + const auto readPath=[&](const fs::path& relative){const auto staged=output/relative;return fs::exists(staged)?staged:install/relative;}; + fs::path graphicsRelative="data/graphics/graphics.big"; + if(!fs::exists(output/graphicsRelative) && fs::exists(output/"data/graphics/pc/graphics.big")) graphicsRelative="data/graphics/pc/graphics.big"; + else if(!fs::exists(readPath(graphicsRelative))) graphicsRelative="data/graphics/pc/graphics.big"; + const creatureimport::Sources source{readPath("data/CompiledDefs/names.bin"),readPath("data/CompiledDefs/game.bin"), + readPath(graphicsRelative),readPath("data/graphics/pc/textures.big")}; + const auto proof=creatureimport::captureSources(source); + const auto schema=forge::defschema::Schema::load(options.at("--schema")); + const auto model=meshimport::loadSkinnedModel(positional[0]); + std::vector materials; + for(size_t i=0;i texture; + if(options.contains("--texture")) texture=creatureimport::captureTexture(options.at("--texture"),name+"_DIFFUSE", + options.contains("--texture-format")?options.at("--texture-format"):"dxt1"); + const auto result=creatureimport::publishMesh(source,proof,output,graphicsRelative,schema,request,model,materials,true,{},texture?&*texture:nullptr); + const nlohmann::json report{{"creature",request.rootName},{"root_index",result.clones.rootIndex}, + {"body_index",result.clones.bodyIndex},{"appearance_index",result.clones.appearanceIndex}, + {"body_part",request.bodyPart},{"alternative",request.alternative},{"donor_model_id",result.binding.oldModelId}, + {"mesh_id",result.binding.newModelId},{"texture_id",texture?result.textureId:existingTexture}, + {"notes",model.notes}, + {"other_body_slots_shared",true},{"animations_shared",true},{"runtime_compatibility_verified",false}}; + if(flags.contains("--json")) std::cout< Date: Sat, 10 Oct 2026 14:40:39 -0600 Subject: [PATCH 47/75] Verify creature CLI publication and Restore against tiny native donor fixtures --- docs/CLI.md | 4 +- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 23 +++ src/cli/common.cpp | 2 +- tests/test_skinnedinput.cpp | 16 +- tools/gen_cli_reference.py | 2 +- tools/test_creature_import_cli.py | 194 ++++++++++++++++++++++ 6 files changed, 235 insertions(+), 6 deletions(-) create mode 100644 tools/test_creature_import_cli.py diff --git a/docs/CLI.md b/docs/CLI.md index bcafd1c4..fc85cd73 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -3,7 +3,7 @@ `FableForge 0.18.1-dev -- Fable: The Lost Chapters level editor (maps -> .glb / .obj, levels, terrain, textures)` Generated by `tools/gen_cli_reference.py` from `FableForge --help`; edit `src/cli/common.cpp`'s usage text, not this file. -Every command takes `--install ` (default: the Steam / GOG install found on this machine). +Every command takes `--install ` (usually defaults to the Steam / GOG install found on this machine; `creature-import` requires an explicit install and output root). Commands that write game files keep a one-time `.forge-orig` backup and are refused while Fable runs. ```text @@ -26,9 +26,9 @@ usage: (a ground theme from your own texture: appended to textures.big + a new ENGINE_THEME in game.bin; paint it from the editor) forge mesh-import [--texture | --texture-id ] [--donor ] [--stored] [--no-collision] [--install ] forge mesh-replace [--stored] [--install ] (static geometry; preserves materials and collision) + (a custom static object: MESH_ appended to graphics.big, its diffuse to textures.big, a collision hull from its own triangles, an OBJECT_ def copied from the donor; Y-up model space) forge creature-import --donor-index --body --alternative --schema --install --out [--texture | --texture-id ] [--texture-format dxt1|dxt3|argb8888] [--stored] [--json] (private root/body/appearance clone + one selected body mesh; other body slots and animations remain shared; runtime compatibility unverified) - (a custom static object: MESH_ appended to graphics.big, its diffuse to textures.big, a collision hull from its own triangles, an OBJECT_ def copied from the donor; Y-up model space) forge textures [filter] [--bank ] (list textures.big entries: id, size, format, bank) forge texture-export | texture-replace | texture-add [--bank B] [--format dxt1|dxt3|argb8888] forge minimap-bake [--region | --framing scale,offX,offY] (the minimap a new region gets, as a PNG; framed like the engine's hero marker; writes nothing) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 696bc0e8..d1f6700c 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -1164,3 +1164,26 @@ help now returns zero and the rerun passes. Generated docs/CLI.md refreshed. These are argument-only checks, not positive CLI import acceptance; next retain tiny donor source fixtures and independently test successful CLI publications and Restore before GUI wiring. Native graph, package and runtime gates remain open. + +Positive public CLI acceptance now passes in creature-cli-grenrlbw using retained +tiny source-install fixtures emitted by the rebuilt native-donor harness packet +skinned-input-donors-dkvikiu6. Five native skeleton donors each exercise untextured, +existing ID 7, DXT1/DXT3/ARGB image append and four-file in-place publication: +25 fresh + five in-place imports/Restore. Independent Python checks all six prior +definition records and raw identity rows/pool prefix, the exact three appended +payloads (private references plus body word at offset 27), original mesh/texture +payloads, donor raw LOD0/ghost skeleton spans, geometry/weights, texture references +and blue/original plus red-green/new pixels. All fixture source hashes stay exact. +Output paths contain spaces, percent and ampersand; commands run outside checkout. +Twenty-five CLI prewrite refusals cover absent part/alternative, non-Creature donor, +missing texture ID and mismatched source rig. No refusal creates an output tree. +Skin-input CTest passes after fixture-emission change; owned forge rebuilt. + +Initial CLI test assumed the descriptor contained a texture row for ID zero; +it actually lists nonzero referenced IDs. Pixel check initially used wrong DXT +format constants, corrected to established 0x1f/0x20. Mismatched-rig diagnostics +can identify parent mismatch rather than a missing joint; gate now requires the +specific donor-skin-mapping diagnostic. No production guards/tolerances relaxed. +Failed-run tiny outputs were removed through authoritative Restore; logs retained. +No full native definition corpus, GUI, package or runtime acceptance claimed. +Next: native definition-graph coverage and donor/mesh/texture GUI workflow. diff --git a/src/cli/common.cpp b/src/cli/common.cpp index a79c9c96..f88e4769 100644 --- a/src/cli/common.cpp +++ b/src/cli/common.cpp @@ -69,9 +69,9 @@ int usage() { " (a ground theme from your own texture: appended to textures.big + a new ENGINE_THEME in game.bin; paint it from the editor)\n" " forge mesh-import [--texture | --texture-id ] [--donor ] [--stored] [--no-collision] [--install ]\n" " forge mesh-replace [--stored] [--install ] (static geometry; preserves materials and collision)\n" + " (a custom static object: MESH_ appended to graphics.big, its diffuse to textures.big, a collision hull from its own triangles, an OBJECT_ def copied from the donor; Y-up model space)\n" " forge creature-import --donor-index --body --alternative --schema --install --out [--texture | --texture-id ] [--texture-format dxt1|dxt3|argb8888] [--stored] [--json]\n" " (private root/body/appearance clone + one selected body mesh; other body slots and animations remain shared; runtime compatibility unverified)\n" - " (a custom static object: MESH_ appended to graphics.big, its diffuse to textures.big, a collision hull from its own triangles, an OBJECT_ def copied from the donor; Y-up model space)\n" " forge textures [filter] [--bank ] (list textures.big entries: id, size, format, bank)\n" " forge texture-export | texture-replace | texture-add [--bank B] [--format dxt1|dxt3|argb8888]\n" " forge minimap-bake [--region | --framing scale,offX,offY] (the minimap a new region gets, as a PNG; framed like the engine's hero marker; writes nothing)\n" diff --git a/tests/test_skinnedinput.cpp b/tests/test_skinnedinput.cpp index 7a1261e7..842fc8ca 100644 --- a/tests/test_skinnedinput.cpp +++ b/tests/test_skinnedinput.cpp @@ -69,11 +69,12 @@ static int donorGate(char** argv) { const auto model=albion::meshimport::loadSkinnedModel(argv[4]); // Small independent definition graph; the donor's real skeleton/payload // remains lazy/read-only, with no full archive serialization or copies. - const auto schema=forge::defschema::Schema::loadText(R"({ + const std::string schemaText=R"({ "CREATURE":{"fields":[{"name":"Components","type":"uint32"},{"name":"Self","type":"CDefIndex"},{"name":"Other","type":"int32"}]}, "CCreatureDef":{"fields":[{"name":"Link","type":"CDefIndex"},{"name":"Other","type":"int32"}]}, "CAppearanceDef":{"fields":[{"name":"Link","type":"CDefIndex"},{"name":"Other","type":"int32"}]} - })"); + })"; + const auto schema=forge::defschema::Schema::loadText(schemaText); const auto field=[](Bytes& bytes,const char* name,uint32_t value){word(bytes,forge::defdecode::fieldTag(name));word(bytes,value);}; const auto root=[&](bool donorRoot) { Bytes bytes{1,0,1,3,0}; @@ -294,6 +295,17 @@ static int donorGate(char** argv) { stream.write(reinterpret_cast(bytes->data()),std::streamsize(bytes->size()));require(bool(stream),"donor output write failed"); } } + // Retain only these tiny fixture banks for the separate public CLI gate. + // Never copy the original full native graphics/texture archives. + const auto fixtureRoot=fs::path(argv[4]).parent_path()/"source-install"; + require(!fs::exists(fixtureRoot),"CLI fixture destination must be fresh"); + for(const auto& [from,relative]:std::array,4>{{ + {namesPath,"data/CompiledDefs/names.bin"},{bankPath,"data/CompiledDefs/game.bin"}, + {graphicsPath,"data/graphics/graphics.big"},{texturesPath,"data/graphics/pc/textures.big"}}}) { + fs::create_directories((fixtureRoot/relative).parent_path()); + require(fs::copy_file(from,fixtureRoot/relative),"CLI fixture copy failed"); + } + write(fs::path(argv[4]).parent_path()/"schema.json",Bytes(schemaText.begin(),schemaText.end())); std::cout<<"authored glTF -> donor remap -> stored/compressed readback PASS: "< int: out = ["# Command line reference", "", f"`{title}`", "", "Generated by `tools/gen_cli_reference.py` from `FableForge --help`; edit `src/cli/common.cpp`'s usage text, not this file.", - "Every command takes `--install ` (default: the Steam / GOG install found on this machine).", + "Every command takes `--install ` (usually defaults to the Steam / GOG install found on this machine; `creature-import` requires an explicit install and output root).", "Commands that write game files keep a one-time `.forge-orig` backup and are refused while Fable runs.", "", "```text", body, "```", ""] with open(os.path.join(ROOT, "docs", "CLI.md"), "w", encoding="utf-8", newline="\n") as f: diff --git a/tools/test_creature_import_cli.py b/tools/test_creature_import_cli.py new file mode 100644 index 00000000..c1b5ccc4 --- /dev/null +++ b/tools/test_creature_import_cli.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Public CLI acceptance using retained tiny native-donor fixtures, never game writes.""" +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import struct +import subprocess +import sys +import tempfile + +from test_fmp_links import package_records, read_bank + + +def inventory(root): + return {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in root.rglob('*') if p.is_file()} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--forge', type=Path, required=True) + parser.add_argument('--fixtures', type=Path, required=True) + parser.add_argument('--oracle-root', type=Path, required=True) + args = parser.parse_args() + forge = args.forge.resolve(strict=True) + fixtures = args.fixtures.resolve(strict=True) + assert json.loads((fixtures/'result.json').read_text())['passed'] + sys.path.insert(0, str(args.oracle_root.resolve()/'tools/blender_addon/io_scene_fable')) + import mesh_rw + fc = mesh_rw.fc + before = inventory(fixtures) + checks = [] + refusals = [] + # New outputs are tiny banks only; retain logs/manifest, Restore removes data. + work = Path(tempfile.mkdtemp(prefix='creature-cli-', dir=fixtures.parent)) + print('Evidence retained at', work, flush=True) + + def run(command, label): + result = subprocess.run([str(forge), *map(str, command)], cwd=work, + capture_output=True, text=True, timeout=120) + (work/(label+'.log')).write_text(result.stdout+result.stderr, encoding='utf-8') + return result + + for donor in (4521, 5121, 5150, 3985, 5110): + folder = fixtures/str(donor) + source = folder/'source-install' + originals = read_bank(source) + assert len(originals) == 6 + old_gfx = package_records(source/'data/graphics/graphics.big', by_id=True) + old_tex = package_records(source/'data/graphics/pc/textures.big', by_id=True) + source_buf, source_entries = fc.open_graphics_big(str(source/'data/graphics/graphics.big')) + source_entry = source_entries[0] + raw = source_buf[source_entry['offset']:source_entry['offset']+source_entry['size']] + skeleton = mesh_rw.clone_skeleton(raw, 5) + for mode in ('none', 'existing', 'dxt1', 'dxt3', 'argb8888', 'inplace'): + label = f'{donor}-{mode}' + output = work/(label+' output % &') + if mode == 'inplace': + shutil.copytree(source, output) + baseline = inventory(output) + name = 'CLI_'+str(donor)+'_'+mode.upper() + command = ['creature-import', folder/'source.gltf', name, '--donor-index', '1', + '--body', '0', '--alternative', '0', '--schema', folder/'schema.json', + '--install', output if mode == 'inplace' else source, '--out', output, '--json'] + texture_id = 0 + if mode == 'existing': + command += ['--texture-id', '7'] + texture_id = 7 + elif mode not in ('none', 'existing'): + command += ['--texture', folder/'diffuse.png', '--texture-format', + 'argb8888' if mode == 'inplace' else mode] + texture_id = 8 + if mode in ('none', 'dxt3'): + command += ['--stored'] + result = run(command, label) + assert result.returncode == 0, result.stdout+result.stderr + report = json.loads(result.stdout) + for key, value in {'root_index': 6, 'body_index': 7, 'appearance_index': 8, + 'donor_model_id': donor, 'mesh_id': donor+1, 'texture_id': texture_id, + 'other_body_slots_shared': True, 'animations_shared': True, + 'runtime_compatibility_verified': False}.items(): + assert report[key] == value, (label, key, report) + actual = read_bank(output) + assert len(actual) == 9 and actual[:6] == originals + expected_root = bytearray(originals[1]['data']) + for offset, value in ((9,7), (13,6), (21,8), (25,6), (53,6)): + struct.pack_into(' Date: Sat, 10 Oct 2026 14:44:42 -0600 Subject: [PATCH 48/75] Validate skinned import plans against native creature graphs --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 27 ++++ tests/test_skinnedinput.cpp | 42 ++++++- tools/test_creature_import_native_graph.py | 136 +++++++++++++++++++++ 3 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 tools/test_creature_import_native_graph.py diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index d1f6700c..319d441a 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -1187,3 +1187,30 @@ specific donor-skin-mapping diagnostic. No production guards/tolerances relaxed. Failed-run tiny outputs were removed through authoritative Restore; logs retained. No full native definition corpus, GUI, package or runtime acceptance claimed. Next: native definition-graph coverage and donor/mesh/texture GUI workflow. + +Native combined mesh-plan coverage now passes 140 actual compiled Creature +graph/slot cases in creature-native-mesh-plan-r2h8bzk9. The skin-input harness +adds read-only --native-plan: loads original names/game/graphics and the real +schema, derives the mesh donor from a selected native root/body/alternative, +prepares a private graph plus skinned mesh, and verifies all prior 14,764 records +remain exact. Writes only the three new component payloads and mesh payload/info, +never serializes or copies full native banks. The independent Python gate applies +the established relocation audit's explicit rewrites to original bytes, restores +every unselected gameplay triple, and verifies exact graph output plus the one +selected model word, next mesh ID 8113 and preserved raw LOD0/ghost skeleton spans. +Original compiled-bank and graphics streaming hashes remain unchanged. + +Counts by skeleton ID: 4521=32, 5121=48, 5150=59, 5110=1. ID3985 has no eligible +native root slot in this corpus and is explicitly NOT EXERCISED for this gate; +its synthetic graph/public CLI coverage remains separate. Two matching slots of +root1175 refuse with the established unclassified possible reference at offset347, +before output creation; no relocation guard weakened. Initial test asserted each +head fixture had a native root link, correctly exposing the unexercised 3985 case. +The refusal wording assertion was corrected to the actual specific diagnostic. +Final evidence contains 843 files/3,356,169 bytes (~3.2 MiB), not native bank copies. +Rebuilt skin-input harness and its CTest pass. This is native in-memory plan +acceptance, not native filesystem publication, GUI, package or runtime acceptance. +GUI investigation locates integration in gui/definitionbrowser.cpp's private +clone draft/confirmation and gui/app.hpp's shared definition-edit state. Next: +donor/model/texture card with captured source and destination identity, background +publication, draft/close protection, reload and real-control UI acceptance. diff --git a/tests/test_skinnedinput.cpp b/tests/test_skinnedinput.cpp index 842fc8ca..4a252ba9 100644 --- a/tests/test_skinnedinput.cpp +++ b/tests/test_skinnedinput.cpp @@ -308,9 +308,49 @@ static int donorGate(char** argv) { write(fs::path(argv[4]).parent_path()/"schema.json",Bytes(schemaText.begin(),schemaText.end())); std::cout<<"authored glTF -> donor remap -> stored/compressed readback PASS: "<(bytes.data()),std::streamsize(bytes.size())); + require(bool(stream),"native plan payload write failed"); + }; + for(const auto& [name,index]:std::array,3>{{ + {"root",plan.clones.rootIndex},{"body",plan.clones.bodyIndex},{"appearance",plan.clones.appearanceIndex}}}) + write(output/(std::string(name)+".bin"),plan.definitions.entries()[index].data); + const auto& mesh=plan.graphics.findBank("MBANK_ALLMESHES")->entries.back(); + write(output/"mesh.bin",mesh.data);write(output/"mesh.info",mesh.subHeader); + std::cout< Date: Sat, 10 Oct 2026 14:48:57 -0600 Subject: [PATCH 49/75] Record user-requested pause and custom creature GUI resume checklist --- docs/HANDOFF_CREATURE_CLONE_2026-10-10.md | 76 +++++++++++++++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 2 +- 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md index 319d441a..1f0ac435 100644 --- a/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md +++ b/docs/HANDOFF_CREATURE_CLONE_2026-10-10.md @@ -1,5 +1,81 @@ # Creature donor self-reference foundation, 2026-10-10 +## Paused at user request: next-session entry point + +User asked to stop and update documentation. Do not automatically resume until +asked. Last validated implementation commit is **5a91d6f**, pushed on +`feat/definition-clone-gui-20261010`. The preceding CLI commit is 2bd2b9c. +Owned checkout: `C:/Users/Cornelio/AppData/Local/FableForgeDevelopment/definition-clone-gui-20261010`. +RC2 publication is already complete; RC2/stable releases and development archives +remain unchanged. The full continuation goal is paused, not complete. + +Four source files contain **uncommitted, unbuilt, untested GUI WIP**: + +- `gui/app.hpp`: opaque custom-import draft, future/result state, model/texture + fields, body choices and review/publish/poll method declarations. definitionsBusy + now includes the custom-import future. +- `gui/definitionbrowser.cpp`: review/publish workers with source and destination + proofs, captured pack manifest, model/sidecar validation, publisher callback and + completion/reload logic. pollDefinitions polls this new worker; custom-mode clone + save dispatches to it. +- `src/meshimport.hpp` / `src/meshimport.cpp`: skinned glTF Model now records source + file paths, canonical identities and SHA256 of the exact consumed bytes, including + external sidecars. Static loading does not capture hashes. Model ABI changed; + existing binaries have NOT been rebuilt against this WIP. + +This is scaffolding, **not a usable or accepted custom-import GUI**. No widgets +have been attached; definitionCreatureMesh remains false, slot choices are not yet +populated, and no review can currently be started through the UI. Preserve these +edits and inspect/improve them; do not mistake dirty source for the last tested +commit. No owning FableForge build/test worker was live at pause. An unrelated +FableTLC native syntax-check process was observed and left untouched. + +Next actions after explicit resume: + +1. Populate body/alternative/model choices from the preflight private cloned body + in beginDefinitionClone; reset all custom state on new draft/cancel/discard. +2. Attach optional custom-mesh controls to the private Creature draft: model and + texture browse fields, explicit slot selection, mesh/texture names, existing + texture ID versus image, format and stored/compressed selection. Keep no-mesh + private cloning unchanged. Connect Review to prepareCreatureImport and Create + to the reviewed snapshot. Invalidate confirmation when any input changes. +3. Audit busy/close/discard/automation paths: widgets must not mutate or cancel a + publishing draft; existing definition_cancel automation bypasses UI guards. + Validate destination switches between review/Create, source bank/name changes, + glTF main/sidecar changes, PNG changes, pack-manifest edits and physical aliases. + Check modelRoot/context/texture refresh behavior for folder and pack output. +4. Build the owned GUI, forge and skin-input targets with `-j2` (ABI changed), then + test source-file proofs and the existing skin/static-loader neighbors. WIP has + no compile or test acceptance yet. Use tiny retained fixture banks for GUI tests + at both sizes, including cancellation, stale/refusal, exact deltas and Restore. + Native graph gates are read-only; do not publish full native banks merely to + test UI. Do not repeat unchanged accepted gates without a relevant change. +5. Only after real-control acceptance, update roadmap/current status, commit the + implementation and consider a separate immutable development checkpoint. + Placement, assembled creature presentation and runtime validation remain open; + changing one body alternative is not full-body replacement. + +Git commands in this worktree require these task-specific object directories: +`GIT_OBJECT_DIRECTORY=C:/Users/Cornelio/AppData/Local/FableForgeDevelopment/definition-clone-git-objects` +and `GIT_ALTERNATE_OBJECT_DIRECTORIES=D:/Code/FableForge/.git/objects`. +Do not remove sibling locks or change their object stores. Re-read root AGENTS and +the development/format-I/O skills before resuming; no sub-agent delegation enabled. + +Retained acceptance evidence under +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261010`: + +- `skinned-input-donors-dkvikiu6`: five native skeleton fixtures with tiny retained + source installs, native-donor/publisher/texture/rollback gates. +- `creature-cli-grenrlbw`: 30 public CLI import/Restore cases + 25 prewrite refusals; + independent graph/mesh/pixel preservation, unchanged fixture inputs. +- `creature-native-mesh-plan-r2h8bzk9`: 140 native graph plans + two verified strict + refusals; 3985 native-root linkage explicitly unexercised; native hashes exact. +- `build-scratch-cleanup-summary.txt`: both user-named build trees cleaned of + 216.96 GiB logical scratch data; binaries/releases/logs/results preserved. + +Continue avoiding full game/texture scratch copies. Original install remains +read-only: `C:/Programs/Steam/steamapps/common/Fable The Lost Chapters`. + Owned C development checkout/branch unchanged; starting clean at d2678ed. Previous turn is progress: recovered one Creature schema field, completed exact source/extracted acceptance and committed/pushed. See diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 0bdac8f4..49cd0971 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Active continuation: [Creature visual workflows](HANDOFF_CREATURE_CLONE_2026-10-10.md). Private cloning, body/appearance navigation, body-model binding and named model choice are implemented. Body checkpoint extracted acceptance passes nine selected workflows; 138 not run/two unrelated skips. Custom skinned import and skeleton/runtime compatibility remain next; no full-feature claim. RC2 unchanged. +> Paused at user request: [Creature visual workflows and next-session entry point](HANDOFF_CREATURE_CLONE_2026-10-10.md). Last tested/pushed implementation 5a91d6f: coordinated skinned CLI import/Restore and 140 native graph plans pass. Four source files contain uncommitted, unbuilt GUI scaffolding; widgets and acceptance remain unfinished. Resume only when asked. Runtime compatibility unverified; RC2 unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 79e4a99a..deae36d8 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Active continuation: [Creature visual workflows](HANDOFF_CREATURE_CLONE_2026-10-10.md). Private cloning, body/appearance navigation, body-model binding and named model choice are implemented. Body checkpoint extracted acceptance passes nine selected workflows; 138 not run/two unrelated skips. Custom skinned import and skeleton/runtime compatibility remain next; no full-feature claim. RC2 unchanged. +> Paused at user request: [Creature visual workflows and next-session entry point](HANDOFF_CREATURE_CLONE_2026-10-10.md). Last tested/pushed implementation 5a91d6f: coordinated skinned CLI import/Restore and 140 native graph plans pass. Four source files contain uncommitted, unbuilt GUI scaffolding; widgets and acceptance remain unfinished. Resume only when asked. Runtime compatibility unverified; RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there From 687fa300b1b65b68c124e6976b1c57d93e2c2b59 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:29:40 -0600 Subject: [PATCH 50/75] Add reviewed custom Creature import controls and guarded publication --- docs/AUTOMATION.md | 19 ++ docs/EDITOR.md | 4 +- docs/HANDOFF_CREATURE_GUI_2026-10-23.md | 97 ++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 2 +- gui/app.hpp | 17 +- gui/automation.cpp | 2 + gui/definitionbrowser.cpp | 222 ++++++++++++++++++++++- gui/editor.cpp | 2 +- src/meshimport.cpp | 19 +- src/meshimport.hpp | 2 + tests/test_skinnedinput.cpp | 9 + tools/check_all.py | 2 + tools/test_creature_import_ui.py | 230 ++++++++++++++++++++++++ tools/test_package.py | 2 + 15 files changed, 614 insertions(+), 17 deletions(-) create mode 100644 docs/HANDOFF_CREATURE_GUI_2026-10-23.md create mode 100644 tools/test_creature_import_ui.py diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md index a9d4a9ea..03a9e5b4 100644 --- a/docs/AUTOMATION.md +++ b/docs/AUTOMATION.md @@ -1116,6 +1116,25 @@ The distinct Creature mode uses `definition_clone_creature` or the real `definition_clone_creature_refused` checks preflight refusal; `definition_creature_cloning` distinguishes it from an exact clone draft. `creature_clone_ui` covers 36 synthetic cases across both window sizes. + +Private Creature custom import uses the real `check_definition_creature_mesh`, +`combo_definition_creature_slot` / `definition_creature_slot_`, +`input_definition_creature_model`, `input_definition_creature_mesh_name`, +`input_definition_creature_texture`, `input_definition_creature_texture_name`, +`input_definition_creature_texture_id`, `combo_definition_creature_format` / +`definition_creature_format_<0|1|2>` and `check_definition_creature_stored` +widgets. The existing Save button first reviews the import asynchronously; +`wait_definitions` waits for review/publication and the confirmation button +creates only the reviewed snapshot. `definition_cancel` refuses during workers. +`creature_import_ui` exercises retained tiny donor fixtures at both sizes, +including changed model/sidecar/image/bank/manifest and junction identities, +exact definition/asset preservation, output Restore and close protection. +Set `FABLEFORGE_CREATURE_FIXTURES` to the passed packet from +`tools/test_skinned_input_donors.py`, or pass `--fixtures` directly to the test. +Without that packet the workflow reports SKIP. Synthetic schema checks run +byte-identical executable/DLL copies in an owned runtime directory; their cwd +supplies the tiny schema, leaving packaged retail schemas untouched. + `creature_clone_gui_retail_1440x900` and `_800x600` independently compare all prior retail records, verified self-only deltas, paired recovery and explicit Restore on owned outputs. Appearance/body/components remain shared. diff --git a/docs/EDITOR.md b/docs/EDITOR.md index 10003539..e93b35b8 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1446,7 +1446,9 @@ The distinct **Clone creature** action appears for game-bank CREATURE records. I The same guarded shared-component mode is available through `forge-tools defs clone-creature --schema --out `. -The separate **Clone private creature visuals** action appends a coordinated root, CCreatureDef and CAppearanceDef, with explicit unique bounded ASCII names. Selected visual rows become owned by the new root; other gameplay components retain their original data and ownership. Models, textures and animation assets remain shared. The paired publisher independently recomputes the three-record plan before writing names.bin and game.bin together, retaining recovery records. Unsupported/malformed known layouts, missing visual components, unresolved references and new name-CRC ambiguities refuse before publication. Root, body and appearance names are separate fields in the scrolling card; the Create confirmation states that assets stay shared. Cancel and close-review retain the existing draft protection. Use owned outputs first; no gameplay validation is claimed. +The separate **Clone private creature visuals** action appends a coordinated root, CCreatureDef and CAppearanceDef, with explicit unique bounded ASCII names. Selected visual rows become owned by the new root; other gameplay components retain their original data and ownership. Without a custom model, models, textures and animation assets remain shared. The paired publisher independently recomputes the three-record plan before writing names.bin and game.bin together, retaining recovery records. Unsupported/malformed known layouts, missing visual components, unresolved references and new name-CRC ambiguities refuse before publication. Root, body and appearance names are separate fields in the scrolling card. Cancel and close-review retain the existing draft protection. Use owned outputs first; no gameplay validation is claimed. + +Enable **Import a custom body model** in that draft to choose one explicit body part/alternative and a skinned `.glb` or `.gltf` matching the donor skeleton. Supply a new `MESH_` name using uppercase letters, digits and underscores, then choose an existing texture ID or a new named image. New images support DXT1, DXT3 and ARGB8888; models can be compressed or stored. **Review custom import** checks the model before offering Create. Changing any input or destination requires a fresh review. Create rechecks the source banks, model and external buffer bytes, image, destination files and pack manifest before publishing the coordinated definitions, mesh and optional texture. Controls and cancellation wait for background review/publication. Restore removes created outputs or recovers in-place originals. Other body alternatives and animations remain shared; this does not replace a complete creature or establish gameplay compatibility. The same private mode is available through `forge-tools defs clone-creature-private --schema --out [--names ] [--json]`. diff --git a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md new file mode 100644 index 00000000..ddcfc84c --- /dev/null +++ b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md @@ -0,0 +1,97 @@ +# Creature custom-import GUI, 2026-10-23 + +User resumed FableForge until asked to stop. Active checkout remains +`C:/Users/Cornelio/AppData/Local/FableForgeDevelopment/definition-clone-gui-20261010`, +branch `feat/definition-clone-gui-20261010`, starting HEAD `6fc51e7`. +The [preceding handoff](HANDOFF_CREATURE_CLONE_2026-10-10.md) records the four +preserved source files and existing native/CLI acceptance. The separate Git object +directory documented there is still required; missing HEAD without those environment +variables does not establish repository corruption. An initial redundant fetch was +interrupted; no branch reset or object-store configuration change was made. + +## Current implementation + +Private Creature drafts enumerate confirmed body/alternative/model IDs during +clone preflight and offer optional custom skinned glTF import. Inputs include model +and image browsing, explicit body alternative, full MESH_ name, existing texture ID +or a new named image, texture format, and stored/compressed mesh selection. Review +prepares the existing mesh plan asynchronously; Create uses the reviewed snapshot +and the existing coordinated publisher. Input/destination changes invalidate review. +No-mesh private cloning retains the existing publisher. + +The captured glTF main file and external buffers retain canonical paths and hashes +of consumed bytes. Creation rechecks these, source banks, destination files/root, +the pack manifest and optional texture proof. Draft controls and cancellation are +disabled during review/publication; automation cancellation refuses while busy, +and close-prompt Discard waits for the import worker. Completion reloads the new +definition, model source, texture view and recovery inventory. + +Intended delta: append three private definitions and one mesh, retarget only the +selected body-model word and verified private references, optionally append one +texture. Prior definition payloads, identities, names and asset entries must survive. +Other body alternatives, gameplay components and animations stay shared. +Runtime compatibility and assembled-creature presentation remain unverified. + +## Source verification + +- Owned GUI/forge/skinned-input build passes. Updated skin-input regression checks + main/sidecar consumed-byte hashes and sidecar mutation. Skinned-input, static + mesh-input and skinned-composition CTests pass after rebuilding affected targets. +- Public CLI regression passes all five skeleton fixtures: 25 fresh and five + in-place publications/Restore plus 25 prewrite refusals, independent graph/mesh/ + pixel checks. Evidence: `creature-cli-9mqaudpa` under the retained 20261010 C root. +- Consolidated source GUI gate passes 40 real-control cases: stored/compressed, + all three image formats, existing/no texture, pack output, close/review, + cancel/absent-slot refusal, changed model/sidecar/image/bank/name/graphics/ + texture-bank/manifest refusal, and destination/input review invalidation. + Exact unrelated records, identity rows, prior name bytes and asset payloads + survive; Restore removes tracked outputs. Evidence: `creature-import-ui-ibb9tiok`. + Four additional same-bytes model-directory and destination-root junction + changes refuse at both sizes: `creature-import-ui-atqqqt8g`. + Byte-identical runtime-copy probes pass four image/pack cases: + `creature-import-ui-0mof94lx`. Existing clone save/cancel/pack/close neighbors + pass eight cases: `creature-clone-ui-1crcpc8_`. +- Inspected compact 800x600 review screenshot: body import controls scroll and the + Create/Cancel confirmation is visible. No game window or game install was changed. + +Initial UI setup lacked a level fixture, so wait_maps correctly timed out. The +test now creates an owned tiny install with an unused LEV marker. The first mesh +name lacked the established MESH_ prefix; the UI now explains the naming rule. +An early destination test tried to click confirmation after it had correctly been +invalidated; corrected test checks direct-save refusal and no output creation. +An early texture assertion expected a bank copy without an image append; the +publisher correctly retains the source texture bank, so the assertion now checks +that only image imports create a texture output. No production guard was weakened. + +## Immutable extracted checkpoint + +`dist/development/20261023-creature-import-gui/FableForge-0.18.1-dev-win64.zip` +SHA256 `ea4390e9c89cab0956ec0e6e7e8fd8caa0e9b4acf18dd97a28e4cb0071cff469`. +Packaged at starting HEAD `6fc51e7` with scoped source changes dirty after owned +GUI/forge/forge-tools builds; final evidence/docs updates follow packaging without +replacing it. Archive checks pass 118 identities and 142 local documentation links. + +The final `creature_import_ui` workflow passes all 44 cases against byte-identical +copies of the extracted executable/DLLs. The tiny synthetic schema is supplied +through an unrelated cwd; packaged retail schemas remain exact. New workflow is +registered in source/package gates. Supply `FABLEFORGE_CREATURE_FIXTURES` with the +passed retained donor packet; absent fixtures report SKIP, never exercised coverage. +Final receipt: `package-smoke-7x1gpz8v/result.json`; inner evidence: +`creature-import-ui-h3xpli7z`. The stripped GUI SHA256 is +`1dad94e0a3d9e9e411837f33f4f4a854cc2f97447b4c2ed723833e07d4835986`. + +Same-archive neighbors pass retail eight-map smoke, GUI smoke, all scalar draft +cases and all 36 existing Creature clone GUI cases. Receipts: +`package-smoke-x9qirc34/result.json` and `package-smoke-ps17m3zt/progress.json`. +The initial selector mistakenly included nonexistent `definition_drafts`; both +real selected workflows passed before the final selection assertion. Corrected +custom-import invocation passes, including the four additional junction cases. +No failed test was hidden or guard weakened. Combined receipt: +`build/creature-gui-checkpoint-20261023.json` records five distinct passed workflows, +143 not run and two unrelated missing controller/Aeon and F2 corpus skips. +No full-suite, clean-machine, native filesystem custom-import or gameplay claim. + +Next: commit/push the scoped milestone and update the existing draft PR. Keep +source work and RC2 separate. Runtime/placement and assembled-creature presentation +remain open; do not repeat the unchanged full native graph gate or write full +native banks merely to test the UI. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 49cd0971..006fc0c3 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Paused at user request: [Creature visual workflows and next-session entry point](HANDOFF_CREATURE_CLONE_2026-10-10.md). Last tested/pushed implementation 5a91d6f: coordinated skinned CLI import/Restore and 140 native graph plans pass. Four source files contain uncommitted, unbuilt GUI scaffolding; widgets and acceptance remain unfinished. Resume only when asked. Runtime compatibility unverified; RC2 unchanged. +> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Custom body-slot/model/texture controls are connected to reviewed background publication. All 44 custom-import GUI cases pass against the immutable extracted checkpoint, with exact deltas and Restore. Three affected CTests and five focused extracted workflows pass; gameplay remains unverified. RC2 unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index deae36d8..8220ae91 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Paused at user request: [Creature visual workflows and next-session entry point](HANDOFF_CREATURE_CLONE_2026-10-10.md). Last tested/pushed implementation 5a91d6f: coordinated skinned CLI import/Restore and 140 native graph plans pass. Four source files contain uncommitted, unbuilt GUI scaffolding; widgets and acceptance remain unfinished. Resume only when asked. Runtime compatibility unverified; RC2 unchanged. +> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Custom body-slot/model/texture controls are connected to reviewed background publication. All 44 custom-import GUI cases pass against the immutable extracted checkpoint, with exact deltas and Restore. Three affected CTests and five focused extracted workflows pass; gameplay remains unverified. RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there diff --git a/gui/app.hpp b/gui/app.hpp index 47144d91..2f9bfe90 100644 --- a/gui/app.hpp +++ b/gui/app.hpp @@ -594,8 +594,23 @@ class App { bool saveDefinitionClone(); bool setDefinitionEditValue(const std::string& value); bool saveDefinitionEdit(); - bool definitionsBusy() const {return definitionFuture_.valid();} + bool definitionsBusy() const {return definitionFuture_.valid() || creatureImportFuture_.valid();} private: + struct CreatureImportDraft; + struct CreatureImportJob {std::shared_ptr draft;std::string error;bool published=false;size_t rootIndex=0;}; + std::future creatureImportFuture_; + std::shared_ptr creatureImportDraft_; + bool definitionCreatureMesh_=false,definitionCreatureStored_=false; + char definitionCreatureModel_[2048]={},definitionCreatureTexture_[2048]={}; + char definitionCreatureMeshName_[80]={},definitionCreatureTextureName_[80]={},definitionCreatureTextureId_[16]="0"; + int definitionCreatureFormat_=0; + struct CreatureBodyChoice {size_t bodyPart,alternative;uint32_t modelId;}; + std::vector definitionCreatureSlots_; + std::optional definitionCreatureSlot_; + std::array creatureImportInputs() const; + bool prepareCreatureImport(); + bool saveCreatureImport(); + void pollCreatureImport(); struct DefinitionJob { std::filesystem::path names,bank; forge::bin::File file; diff --git a/gui/automation.cpp b/gui/automation.cpp index 24a9b0ba..8a58461f 100644 --- a/gui/automation.cpp +++ b/gui/automation.cpp @@ -713,7 +713,9 @@ bool Automation::tick(App& app) { if(app.saveDefinitionEdit()!=(cmd=="definition_save")) fail("definition save returned an unexpected result");else note("ok "+line);++pc_; } else if(cmd=="definition_cancel") { + if(app.definitionsBusy()) {fail("cannot cancel a definition worker in progress");++pc_;return true;} app.definitionEditing_=false;app.definitionCloning_=false;app.definitionEditConfirm_=false; + app.definitionCreatureMesh_=false;app.creatureImportDraft_.reset();app.definitionCreatureSlots_.clear();app.definitionCreatureSlot_.reset(); app.definitionCloneFile_={};app.definitionCloneNamesBytes_.clear();app.definitionCloneBankBytes_.clear();note("ok "+line);++pc_; } else if(cmd=="wait_definitions") { diff --git a/gui/definitionbrowser.cpp b/gui/definitionbrowser.cpp index bd9cf9c0..ace278b7 100644 --- a/gui/definitionbrowser.cpp +++ b/gui/definitionbrowser.cpp @@ -5,6 +5,7 @@ #include "definitionclone.hpp" #include "definitionbody.hpp" #include "definitionread.hpp" +#include "creaturepublish.hpp" #include "forge/fmplinks.hpp" #include "texturebrowse.hpp" #include "modpack.hpp" @@ -27,6 +28,145 @@ std::string definitionLower(std::string value) { for(auto& c:value) c=char(std::tolower(static_cast(c)));return value; } } +struct App::CreatureImportDraft { + creatureimport::Sources sources; + creatureimport::SourceProof proof; + creatureimport::MeshRequest request; + meshimport::Model model; + std::vector materials; + std::optional texture; + forge::defschema::Schema schema; + fs::path root,graphicsRelative,resolvedRoot; + std::array inputs; + struct Target {fs::path path,resolved;std::string hash;bool present;}; + std::vector targets; + void capture(const fs::path& path) { + const bool present=fs::exists(path); + if(present && !fs::is_regular_file(path)) throw std::runtime_error("Import review target is not a file."); + targets.push_back({path,paths::resolvedPath(path),present?forge::modorder::sha256Of(path):std::string(),present}); + } + void validate() const { + if(paths::resolvedPath(root)!=resolvedRoot) throw std::runtime_error("Import destination identity changed; review again."); + for(const auto& target:targets) + if(paths::resolvedPath(target.path)!=target.resolved || fs::exists(target.path)!=target.present || + (target.present && (!fs::is_regular_file(target.path) || forge::modorder::sha256Of(target.path)!=target.hash))) + throw std::runtime_error("Import destination or pack manifest changed; review again."); + for(const auto& source:model.sourceFiles) + if(!fs::is_regular_file(source.path) || paths::resolvedPath(source.path)!=source.resolved || + forge::modorder::sha256Of(source.path)!=source.sha256) + throw std::runtime_error("Authored model or sidecar changed; review again."); + creatureimport::requireSourceProof(sources,proof); + } +}; +std::array App::creatureImportInputs() const { + return {definitionEditValue_,definitionCloneBodyName_,definitionCloneAppearanceName_,definitionCreatureModel_, + definitionCreatureTexture_,definitionCreatureMeshName_,definitionCreatureTextureName_,definitionCreatureTextureId_, + std::to_string(definitionCreatureFormat_)+":"+std::to_string(definitionCreatureStored_)+":"+ + (definitionCreatureSlot_?std::to_string(*definitionCreatureSlot_):"none"), + definitionEditDestination_.string()+":"+packDest_+":"+saveRoot()}; +} +bool App::prepareCreatureImport() { + definitionEditError_.clear();creatureImportDraft_.reset();definitionEditConfirm_=false; + try { + if(!definitionEditing_ || definitionEditType_!="clone-creature-private" || !definitionCreatureMesh_) + throw std::runtime_error("Choose a private Creature draft with custom mesh enabled."); + if(definitionsBusy() || modReadersBusy() || gameWriteBlocked("review creature import")) + throw std::runtime_error("Wait for loads/writes and close the destination game."); + if(!definitionCreatureSlot_ || *definitionCreatureSlot_>=definitionCreatureSlots_.size()) + throw std::runtime_error("Choose the body part and alternative explicitly."); + const fs::path destination=fs::u8path(packDest_.empty()?saveRoot():packDest_)/"data/CompiledDefs/game.bin"; + if(definitionPaths()!=std::pair{definitionEditNames_,definitionEditBank_} || destination!=definitionEditDestination_) + throw std::runtime_error("Definition source or destination changed; start the draft again."); + if(!packDest_.empty()) {(void)modpack::load(fs::u8path(packDest_));} + const auto read=[&](const std::vector& relatives) { + for(const auto& root:{packDest_,saveRoot(),installPath_}) if(!root.empty()) + for(const auto& relative:relatives) {const auto path=fs::u8path(root)/relative;if(fs::exists(path)) return path;} + return fs::u8path(installPath_)/relatives.front(); + }; + auto draft=std::make_shared();draft->inputs=creatureImportInputs(); + draft->sources={definitionEditNames_,definitionEditBank_, + read({"data/graphics/graphics.big","data/graphics/pc/graphics.big"}),read({"data/graphics/pc/textures.big"})}; + draft->root=destination.parent_path().parent_path().parent_path();draft->resolvedRoot=paths::resolvedPath(draft->root); + draft->graphicsRelative=draft->sources.graphics.parent_path().filename()=="pc"?"data/graphics/pc/graphics.big":"data/graphics/graphics.big"; + draft->schema=definitionSchema_; + const auto slot=definitionCreatureSlots_[*definitionCreatureSlot_]; + draft->request={definitionEditIndex_,slot.bodyPart,slot.alternative,definitionEditValue_,definitionCloneBodyName_, + definitionCloneAppearanceName_,definitionCreatureMeshName_,!definitionCreatureStored_}; + const auto namesBytes=definitionCloneNamesBytes_,bankBytes=definitionCloneBankBytes_; + const auto pack=packDest_; + creatureImportFuture_=std::async(std::launch::async,[draft,namesBytes,bankBytes,pack] { + CreatureImportJob job;job.draft=draft; + try { + draft->proof=creatureimport::captureSources(draft->sources); + if(definitionwrite::cloneBytes(draft->sources.names)!=namesBytes || definitionwrite::cloneBytes(draft->sources.definitions)!=bankBytes) + throw std::runtime_error("The source bank or name pool changed; start the draft again."); + draft->capture(draft->root/"data/CompiledDefs/names.bin");draft->capture(draft->root/"data/CompiledDefs/game.bin"); + draft->capture(draft->root/draft->graphicsRelative);draft->capture(draft->root/"data/graphics/pc/textures.big"); + if(!pack.empty()) {const fs::path root=fs::u8path(pack);(void)modpack::load(root);draft->capture(root/modpack::kFileName);} + draft->model=meshimport::loadSkinnedModel(fs::u8path(draft->inputs[3])); + uint32_t textureId=0;const auto& text=draft->inputs[7]; + const auto parsed=std::from_chars(text.data(),text.data()+text.size(),textureId); + if(parsed.ec!=std::errc{} || parsed.ptr!=text.data()+text.size()) throw std::runtime_error("Choose a complete unsigned decimal texture ID."); + if(!draft->inputs[4].empty() && textureId) throw std::runtime_error("Choose a new texture or an existing texture ID, not both."); + for(size_t i=0;imodel.materialNames.size();++i) { + forge::meshcompose::Material material;material.name=draft->request.meshName+"_MAT"+std::to_string(i); + material.diffuseId=textureId;draft->materials.push_back(std::move(material)); + } + if(!draft->inputs[4].empty()) { + const int format=std::stoi(draft->inputs[8]); + if(format<0 || format>2) throw std::runtime_error("Invalid texture format."); + draft->texture=creatureimport::captureTexture(fs::u8path(draft->inputs[4]),draft->inputs[6], + std::array{"dxt1","dxt3","argb8888"}[size_t(format)]); + } + (void)creatureimport::prepareMesh(forge::bin::File::open(draft->sources.names,draft->sources.definitions), + forge::big::File::open(draft->sources.graphics),draft->schema,draft->request,draft->model,draft->materials); + draft->validate(); + } catch(const std::exception& error) {job.error=error.what();} + return job; + });return true; + } catch(const std::exception& error) {definitionEditError_=error.what();return false;} +} +bool App::saveCreatureImport() { + definitionEditError_.clear(); + try { + if(!creatureImportDraft_ || creatureImportDraft_->inputs!=creatureImportInputs()) throw std::runtime_error("Custom import fields changed; review again."); + if(definitionsBusy() || modReadersBusy() || gameWriteBlocked("create creature import")) throw std::runtime_error("Wait for loads/writes and close the destination game."); + if(definitionPaths()!=std::pair{definitionEditNames_,definitionEditBank_}) throw std::runtime_error("Definition source changed; start the draft again."); + const auto draft=creatureImportDraft_; + creatureImportFuture_=std::async(std::launch::async,[draft] { + CreatureImportJob job;job.draft=draft; + try { + draft->validate(); + const auto result=creatureimport::publishMesh(draft->sources,draft->proof,draft->root,draft->graphicsRelative, + draft->schema,draft->request,draft->model,draft->materials,true,[draft]{draft->validate();},draft->texture?&*draft->texture:nullptr); + job.published=true;job.rootIndex=result.clones.rootIndex; + } catch(const std::exception& error) {job.error=error.what();} + return job; + });definitionEditConfirm_=false;return true; + } catch(const std::exception& error) {definitionEditError_=error.what();return false;} +} +void App::pollCreatureImport() { + if(!creatureImportFuture_.valid() || creatureImportFuture_.wait_for(std::chrono::seconds(0))!=std::future_status::ready) return; + auto job=creatureImportFuture_.get(); + if(!job.error.empty()) {definitionEditError_=job.error;creatureImportDraft_.reset();definitionEditConfirm_=false;pushLog("creature import: "+job.error,1);return;} + if(!job.published) { + if(!definitionEditing_ || !definitionCreatureMesh_ || job.draft->inputs!=creatureImportInputs()) { + definitionEditError_="Draft changed during review; review again.";return; + } + creatureImportDraft_=std::move(job.draft);definitionEditConfirm_=true;return; + } + definitionEditSuccess_="Created "+job.draft->request.rootName+" with a custom body alternative. Other slots/animations remain shared; runtime compatibility unverified."; + for(const auto& note:job.draft->model.notes) pushLog("creature import: "+note,0); + definitionEditing_=false;definitionCloning_=false;definitionEditConfirm_=false;definitionCreatureMesh_=false;creatureImportDraft_.reset(); + definitionCloneFile_={};definitionCloneNamesBytes_.clear();definitionCloneBankBytes_.clear(); + definitionLoaded_=false;definitionBrowseRequest_.reset();definitionBrowseIndex_=job.rootIndex; + foliageexport::closeMeshBank();modelRoot_=job.draft->root.string(); + defThumbs_.clear();defList_.clear();themeGroupOf_.clear();envDefs_.clear();soundDefs_.clear();familyNames_.clear();issuesRev_=~0ull; + if(packDest_.empty() || (!ctx_.gameRoot().empty() && paths::resolvedPath(job.draft->root)==paths::resolvedPath(ctx_.gameRoot()))) { + ctxError_.clear();reloadWhenContextReady_=true;startContextLoad(job.draft->root.string()); + } + refreshTextures();rescanBackups();pushLog(definitionEditSuccess_,3); +} std::pair App::definitionPaths() const { for(const auto& root:{packDest_,saveRoot(),installPath_}) { if(root.empty()) continue; @@ -76,7 +216,8 @@ void App::refreshDefinitions() { }); } void App::pollDefinitions() { - if(!definitionsBusy() || definitionFuture_.wait_for(std::chrono::seconds(0))!=std::future_status::ready) return; + pollCreatureImport(); + if(!definitionFuture_.valid() || definitionFuture_.wait_for(std::chrono::seconds(0))!=std::future_status::ready) return; auto job=definitionFuture_.get(); const auto [names,bank]=definitionPaths(); if(job.names!=names || job.bank!=bank) {definitionLoaded_=false;return;} @@ -125,7 +266,7 @@ void App::browseDefinition(const std::string& name) { setDefinitionBank("game.bin");definitionSearch_[0]=0;definitionType_.clear();definitionFilterDirty_=true;definitionBrowseRequest_=name; } bool App::setDefinitionEditValue(const std::string& value) { - if(!definitionEditing_ || value.size()>=sizeof definitionEditValue_) return false; + if(!definitionEditing_ || definitionsBusy() || value.size()>=sizeof definitionEditValue_) return false; if(definitionEditType_=="CWideString") { try {std::vector validated;forge::unicode::appendUtf16z(validated,value);} catch(const std::exception&) {return false;} @@ -217,7 +358,15 @@ bool App::beginDefinitionClone(bool creature,bool privateVisual) { while(keys.contains(forge::bin::nameCrc(name))); keys.insert(forge::bin::nameCrc(name)); } - (void)forge::defedit::cloneCreatureVisuals(probe,definitionSchema_,*definitionSelected_,probeNames[0],probeNames[1],probeNames[2]); + const auto cloned=forge::defedit::cloneCreatureVisuals(probe,definitionSchema_,*definitionSelected_,probeNames[0],probeNames[1],probeNames[2]); + const auto body=forge::defdecode::decode(probe.entries()[cloned.bodyIndex],definitionSchema_); + const auto layout=forge::fmplinks::creaturePrefixLayout(body.prefix); + definitionCreatureSlots_.clear(); + for(const auto& slot:layout.bodyModels) { + uint32_t id=0; + for(size_t n=0;n<4;++n) id|=uint32_t(probe.entries()[cloned.bodyIndex].data.at(slot.offset+n))<<(8*n); + definitionCreatureSlots_.push_back({slot.bodyPart,slot.alternative,id}); + } } else (void)forge::defedit::cloneCreature(probe,definitionSchema_,*definitionSelected_,probeName); } definitionCloneNamesBytes_=definitionwrite::cloneBytes(definitionLoadedNames_); @@ -230,10 +379,16 @@ bool App::beginDefinitionClone(bool creature,bool privateVisual) { definitionEditField_=donor.name.empty()?"(unnamed record)":donor.name; definitionEditType_=privateVisual?"clone-creature-private":creature?"clone-creature":"clone";definitionEditing_=true;definitionCloning_=true; definitionEditConfirm_=false;definitionEditValue_[0]=0; - definitionCloneBodyName_[0]=0;definitionCloneAppearanceName_[0]=0;return true; + definitionCloneBodyName_[0]=0;definitionCloneAppearanceName_[0]=0; + definitionCreatureMesh_=false;definitionCreatureStored_=false;definitionCreatureSlot_.reset();creatureImportDraft_.reset(); + definitionCreatureModel_[0]=0;definitionCreatureTexture_[0]=0; + definitionCreatureMeshName_[0]=0;definitionCreatureTextureName_[0]=0; + std::snprintf(definitionCreatureTextureId_,sizeof definitionCreatureTextureId_,"0");definitionCreatureFormat_=0; + if(!privateVisual) definitionCreatureSlots_.clear();return true; } catch(const std::exception& error) {definitionEditError_=error.what();return false;} } bool App::saveDefinitionClone() { + if(definitionEditType_=="clone-creature-private" && definitionCreatureMesh_) return saveCreatureImport(); definitionEditError_.clear();definitionEditSuccess_.clear(); try { if(!definitionEditing_ || !definitionCloning_) throw std::runtime_error("Choose a record to clone first."); @@ -351,6 +506,7 @@ void App::drawDefinitionBrowser(float pad,float inner,float cardInner) { } } if(definitionEditing_) { + ImGui::BeginDisabled(definitionsBusy()); if(definitionCloning_) { ImGui::TextWrapped("Clone %s",definitionEditField_.c_str()); theme::hint(definitionEditType_=="clone-creature-private"? @@ -524,20 +680,72 @@ void App::drawDefinitionBrowser(float pad,float inner,float cardInner) { ImGui::TextDisabled("Appearance name (127 ASCII characters max)");ImGui::SetNextItemWidth(cardInner); ImGui::InputText("##privateappearance",definitionCloneAppearanceName_,sizeof definitionCloneAppearanceName_,ImGuiInputTextFlags_CallbackCharFilter,definitionASCIIChar); auto_.registerWidget("input_definition_clone_appearance"); + if(ImGui::Checkbox("Import a custom body model",&definitionCreatureMesh_)) {definitionEditConfirm_=false;creatureImportDraft_.reset();} + auto_.registerWidget("check_definition_creature_mesh"); + if(definitionCreatureMesh_) { + theme::hint("Replaces one body alternative. Other slots and animations remain shared. Use a glTF skin matching the donor skeleton."); + std::string selected="Choose body / alternative"; + if(definitionCreatureSlot_ && *definitionCreatureSlot_inputs!=creatureImportInputs()) {creatureImportDraft_.reset();definitionEditConfirm_=false;} + } } if(!definitionEditConfirm_) { - if(theme::ghostButton(definitionCloning_?"Create definition":"Save field",ImVec2(cardInner,0))) definitionEditConfirm_=true; + if(theme::ghostButton(definitionCreatureMesh_?"Review custom import":definitionCloning_?"Create definition":"Save field",ImVec2(cardInner,0))) { + if(definitionCreatureMesh_) prepareCreatureImport();else definitionEditConfirm_=true; + } auto_.registerWidget("btn_definition_save"); - if(theme::ghostButton("Cancel edit",ImVec2(cardInner,0))) {definitionEditing_=false;definitionCloning_=false;definitionEditConfirm_=false;definitionCloneFile_={};definitionCloneNamesBytes_.clear();definitionCloneBankBytes_.clear();} + if(theme::ghostButton("Cancel edit",ImVec2(cardInner,0))) {definitionEditing_=false;definitionCloning_=false;definitionEditConfirm_=false;definitionCreatureMesh_=false;creatureImportDraft_.reset();definitionCreatureSlots_.clear();definitionCreatureSlot_.reset();definitionCloneFile_={};definitionCloneNamesBytes_.clear();definitionCloneBankBytes_.clear();} auto_.registerWidget("btn_definition_edit_cancel"); } else { - const char* question=definitionCloning_?(definitionEditType_=="clone-creature-private"? + const char* question=definitionCreatureMesh_?"Create private definitions and import this body model?":definitionCloning_?(definitionEditType_=="clone-creature-private"? "Create three private definitions (assets shared)?":definitionEditType_=="clone-creature"? "Create creature with shared components?":"Create this definition in the destination bank?"): definitionEditType_=="body-model"?"Rebind this body model (shared users; compatibility unverified)?":"Write this field to the destination bank?"; const int answer=confirmRow(question,definitionCloning_?"Create":"Save",cardInner,S(28),"btn_definition_save_confirm"); if(answer>0) saveDefinitionEdit();if(answer) definitionEditConfirm_=false; } + ImGui::EndDisabled(); + if(creatureImportFuture_.valid()) ImGui::TextWrapped("Preparing or publishing custom creature import..."); } if(!definitionEditError_.empty()) {ImGui::TextWrapped("%s",definitionEditError_.c_str());auto_.registerWidget("definition_edit_error");} if(!definitionEditSuccess_.empty()) ImGui::TextWrapped("%s",definitionEditSuccess_.c_str()); diff --git a/gui/editor.cpp b/gui/editor.cpp index 2d855e50..558d360c 100644 --- a/gui/editor.cpp +++ b/gui/editor.cpp @@ -4247,7 +4247,7 @@ void App::drawUnsavedPrompt() { } auto_.registerWidget("btn_unsaved_save"); ImGui::SameLine(0, S(6)); - ImGui::BeginDisabled(worldFuture_.valid() || recoveryFuture_.valid()); + ImGui::BeginDisabled(worldFuture_.valid() || recoveryFuture_.valid() || creatureImportFuture_.valid()); if (theme::dangerButton("Discard", ImVec2(w, S(32)))) { retireRecovery(); if (closePending_) quit_ = true; diff --git a/src/meshimport.cpp b/src/meshimport.cpp index 615381a8..1923bfe6 100644 --- a/src/meshimport.cpp +++ b/src/meshimport.cpp @@ -25,6 +25,7 @@ #include "forge/defschema.hpp" #include "forge/meshpreview.hpp" #include "forge/terraintex.hpp" +#include "forge/modorder.hpp" namespace fs = std::filesystem; using json = nlohmann::json; @@ -55,7 +56,7 @@ Vec3 toFableDir(float x, float y, float z) { return {x, -z, y}; } // ---------------------------------------------------------------- glTF 2.0 -struct GltfBuffers { json doc; std::vector> buffers; }; +struct GltfBuffers { json doc; std::vector> buffers; std::vector sourceFiles; }; size_t gltfInteger(const json& value, const char* key) { // glTF integer properties may use decimal/exponent JSON notation. Check @@ -139,9 +140,16 @@ std::vector decodeBufferUri(const std::string& uri) { return out; } -GltfBuffers openGltf(const fs::path& path) { +GltfBuffers openGltf(const fs::path& path,bool captureSources=false) { GltfBuffers g; - const auto bytes = readFile(path); + const auto readSource=[&](const fs::path& source) { + if(!captureSources) return readFile(source); + const auto resolved=fs::canonical(source);auto bytes=readFile(resolved); + g.sourceFiles.push_back({fs::absolute(source),resolved,forge::modorder::sha256OfBytes( + std::string_view(reinterpret_cast(bytes.data()),bytes.size()))}); + return bytes; + }; + const auto bytes = readSource(path); std::vector glbBin; const bool glb = bytes.size() >= 4 && std::memcmp(bytes.data(), "glTF", 4) == 0; if (glb) { @@ -219,7 +227,7 @@ GltfBuffers openGltf(const fs::path& path) { } else { const auto decoded=decodeUriEscapes(uri); if(decoded.find('\0')!=std::string::npos) throw std::runtime_error("NUL in glTF buffer URI path"); - g.buffers.push_back(readFile(path.parent_path() / fs::u8path(decoded))); + g.buffers.push_back(readSource(path.parent_path() / fs::u8path(decoded))); } } else { if(!glb || !g.buffers.empty() || glbBin.empty()) @@ -504,7 +512,8 @@ void readSkinAttributes(const GltfBuffers& g,const json& attributes,size_t count Model loadGltf(const fs::path& path,bool skinned=false) { Model model; - const GltfBuffers g = openGltf(path); + const GltfBuffers g = openGltf(path,skinned); + model.sourceFiles=g.sourceFiles; if(skinned && g.doc.contains("animations") && !g.doc.at("animations").empty()) model.notes.push_back("glTF animation tracks are not imported; skin metadata retains the default joint transforms and inverse binds"); for (const auto& extension : g.doc.value("extensionsUsed", json::array())) diff --git a/src/meshimport.hpp b/src/meshimport.hpp index a53fc3ba..1627451a 100644 --- a/src/meshimport.hpp +++ b/src/meshimport.hpp @@ -43,6 +43,8 @@ struct Model { std::vector materialNames; // per slot std::vector notes; std::vector skins; // loadSkinnedModel only, parallel to prims + struct SourceFile {std::filesystem::path path,resolved;std::string sha256;}; + std::vector sourceFiles; // exact bytes consumed by skinned glTF loader, including sidecars }; // .glb / .gltf (embedded or sidecar buffers; POSITION, NORMAL, TEXCOORD_0, indices; diff --git a/tests/test_skinnedinput.cpp b/tests/test_skinnedinput.cpp index 4a252ba9..9a8e3fe3 100644 --- a/tests/test_skinnedinput.cpp +++ b/tests/test_skinnedinput.cpp @@ -5,6 +5,7 @@ #include "creaturepublish.hpp" #include "forge/temporarydirectory.hpp" #include "forge/texturewrite.hpp" +#include "forge/modorder.hpp" #include "nlohmann/json.hpp" #include #include @@ -493,6 +494,14 @@ int main(int argc,char** argv) { {std::ofstream out(dir/"skin.bin",std::ios::binary);out.write(reinterpret_cast(f.data.data()),std::streamsize(f.data.size()));} {std::ofstream out(external);out< Date: Fri, 23 Oct 2026 06:34:54 -0600 Subject: [PATCH 51/75] Refresh cached model directory after custom Creature pack imports --- docs/HANDOFF_CREATURE_GUI_2026-10-23.md | 26 +++++++++++++++++++++---- gui/definitionbrowser.cpp | 2 +- tools/test_creature_import_ui.py | 11 ++++++++--- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md index ddcfc84c..0ce15fd7 100644 --- a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md @@ -91,7 +91,25 @@ No failed test was hidden or guard weakened. Combined receipt: 143 not run and two unrelated missing controller/Aeon and F2 corpus skips. No full-suite, clean-machine, native filesystem custom-import or gameplay claim. -Next: commit/push the scoped milestone and update the existing draft PR. Keep -source work and RC2 separate. Runtime/placement and assembled-creature presentation -remain open; do not repeat the unchanged full native graph gate or write full -native banks merely to test the UI. +Implementation committed/pushed as `687fa30`; draft PR 2 has been rewritten around +the resulting authoring features and exact coverage. Windows CI run 39953743747 +was still running at the follow-up check. No merge or release publication. + +## Follow-up: pack model-list cache + +A new `pack-preview` case primes Models with the donor before importing into a +pack, then selects the new mesh. It fails against `687fa30` because the cached +model directory still contains the donor only: `creature-import-ui-pyeea_ti`. +Pack completion now calls the existing `refreshModels` path when it does not +reload the global context. Folder output retains its context reload. Six affected +pack-preview/pack/folder cases pass at both sizes: `creature-import-ui-2_1bu4d8`. +The appended mesh is independently selected with three vertices and one triangle; +prior definition/asset preservation and Restore still pass. Owned GUI rebuild +passes. This correction is newer than the immutable checkpoint above; that ZIP +retains the known cached-list behavior and has not been silently replaced. + +Next: check pack output with a separate save folder and imported-texture model +preview; the current pack fixture uses one folder for both. Preview texture +context and whole-creature assembly remain separate from filesystem acceptance. +Runtime/placement remain open; do not repeat the unchanged full native graph gate +or write full native banks merely to test the UI. diff --git a/gui/definitionbrowser.cpp b/gui/definitionbrowser.cpp index ace278b7..4fba3088 100644 --- a/gui/definitionbrowser.cpp +++ b/gui/definitionbrowser.cpp @@ -164,7 +164,7 @@ void App::pollCreatureImport() { defThumbs_.clear();defList_.clear();themeGroupOf_.clear();envDefs_.clear();soundDefs_.clear();familyNames_.clear();issuesRev_=~0ull; if(packDest_.empty() || (!ctx_.gameRoot().empty() && paths::resolvedPath(job.draft->root)==paths::resolvedPath(ctx_.gameRoot()))) { ctxError_.clear();reloadWhenContextReady_=true;startContextLoad(job.draft->root.string()); - } + } else refreshModels(); refreshTextures();rescanBackups();pushLog(definitionEditSuccess_,3); } std::pair App::definitionPaths() const { diff --git a/tools/test_creature_import_ui.py b/tools/test_creature_import_ui.py index ec3b0b90..5536fd2a 100644 --- a/tools/test_creature_import_ui.py +++ b/tools/test_creature_import_ui.py @@ -27,7 +27,7 @@ def main(): parser.add_argument('--fixtures', type=Path, default=os.environ.get('FABLEFORGE_CREATURE_FIXTURES'), help='Retained tiny donor packet from test_skinned_input_donors.py') parser.add_argument('--work-root', type=Path, default=repo/'build') - modes = ('save', 'stored', 'cancel', 'missing-slot', 'existing', 'image', 'image-dxt3', 'image-argb8888', 'pack', 'close-review', 'stale-model', 'stale-sidecar', 'stale-image', 'stale-bank', 'stale-names', 'stale-graphics', 'stale-textures', 'stale-manifest', 'stale-model-alias', 'stale-destination-alias', 'destination', 'changed-draft') + modes = ('save', 'stored', 'cancel', 'missing-slot', 'existing', 'image', 'image-dxt3', 'image-argb8888', 'pack', 'pack-preview', 'close-review', 'stale-model', 'stale-sidecar', 'stale-image', 'stale-bank', 'stale-names', 'stale-graphics', 'stale-textures', 'stale-manifest', 'stale-model-alias', 'stale-destination-alias', 'destination', 'changed-draft') parser.add_argument('--only', nargs='+', choices=modes) args = parser.parse_args() if args.fixtures is None: @@ -72,7 +72,7 @@ def main(): schema.parent.mkdir(parents=True) shutil.copyfile(fixtures/'4521/schema.json', schema) output = case/'output' - pack = mode in ('pack', 'stale-manifest') + pack = mode in ('pack', 'pack-preview', 'stale-manifest') manifest = None if pack: output.mkdir() @@ -83,6 +83,8 @@ def main(): f'set saveroot {output.as_posix()}'] if pack: lines += [f'pack_dest {output.as_posix()}'] + if mode == 'pack-preview': + lines += ['assets_tab 1', 'model_select 4521', 'assert_state model_ready 1'] lines += ['assets_tab 6', 'wait_definitions', 'definition_select 1', 'frames 3'] lines += click('btn_definition_clone_creature_private') @@ -130,6 +132,9 @@ def main(): else: lines += ['assert_state definition_edit_failed 0', 'assert_state definition_editing 0', 'assert_state definition_selected 6', f'screenshot {(case/"created.png").as_posix()}'] + if mode == 'pack-preview': + lines += ['assets_tab 1', 'model_select 4522', 'assert_state model_selected MESH_GUI_MODEL', + 'assert_state model_ready 1', 'assert_state model_vertices 3', 'assert_state model_triangles 1'] lines += ['quit'] script = case/'import.txt' script.write_text('\n'.join(lines)+'\n', encoding='utf-8') @@ -173,7 +178,7 @@ def main(): process.wait(timeout=10) trace = Path(str(script)+'.log').read_text(encoding='utf-8') assert returncode == 0 and 'RESULT PASS' in trace and 'FAIL ' not in trace, trace[-8000:] - if mode in ('save', 'stored', 'existing', 'image', 'image-dxt3', 'image-argb8888', 'pack', 'close-review'): + if mode in ('save', 'stored', 'existing', 'image', 'image-dxt3', 'image-argb8888', 'pack', 'pack-preview', 'close-review'): actual = read_bank(output) assert actual[:6] == originals and len(actual) == 9 old_bank = (source/'data/CompiledDefs/game.bin').read_bytes() From 65dba50d555f5d0204866a4ee80a526236642720 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:08:27 -0600 Subject: [PATCH 52/75] Preview imported Creature pack textures without replacing world assets --- docs/AUTOMATION.md | 7 ++ docs/EDITOR.md | 2 +- docs/HANDOFF_CREATURE_GUI_2026-10-23.md | 136 ++++++++++++++++++++++++ gui/app.cpp | 3 + gui/app.hpp | 5 + gui/definitionbrowser.cpp | 22 +++- gui/renderer.hpp | 3 + gui/textures.cpp | 8 +- tools/test_creature_import_ui.py | 35 ++++-- 9 files changed, 209 insertions(+), 12 deletions(-) diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md index 03a9e5b4..04c21c34 100644 --- a/docs/AUTOMATION.md +++ b/docs/AUTOMATION.md @@ -1135,6 +1135,13 @@ Without that packet the workflow reports SKIP. Synthetic schema checks run byte-identical executable/DLL copies in an owned runtime directory; their cwd supplies the tiny schema, leaving packaged retail schemas untouched. +`model_textured_parts` counts uploaded model parts with a bound texture SRV; +`asset_context_root` identifies the world asset context. Pack-preview cases use +separate pack/save folders, require actual diffuse binding, retain the original +world context, refresh Models, and switch back to the install. An existing-texture +case also checks that no texture bank is created. The GUI driver's result pins its +startup SHA256 and refuses a driver change during execution. + `creature_clone_gui_retail_1440x900` and `_800x600` independently compare all prior retail records, verified self-only deltas, paired recovery and explicit Restore on owned outputs. Appearance/body/components remain shared. diff --git a/docs/EDITOR.md b/docs/EDITOR.md index e93b35b8..39a8d755 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1448,7 +1448,7 @@ The same guarded shared-component mode is available through `forge-tools defs cl The separate **Clone private creature visuals** action appends a coordinated root, CCreatureDef and CAppearanceDef, with explicit unique bounded ASCII names. Selected visual rows become owned by the new root; other gameplay components retain their original data and ownership. Without a custom model, models, textures and animation assets remain shared. The paired publisher independently recomputes the three-record plan before writing names.bin and game.bin together, retaining recovery records. Unsupported/malformed known layouts, missing visual components, unresolved references and new name-CRC ambiguities refuse before publication. Root, body and appearance names are separate fields in the scrolling card. Cancel and close-review retain the existing draft protection. Use owned outputs first; no gameplay validation is claimed. -Enable **Import a custom body model** in that draft to choose one explicit body part/alternative and a skinned `.glb` or `.gltf` matching the donor skeleton. Supply a new `MESH_` name using uppercase letters, digits and underscores, then choose an existing texture ID or a new named image. New images support DXT1, DXT3 and ARGB8888; models can be compressed or stored. **Review custom import** checks the model before offering Create. Changing any input or destination requires a fresh review. Create rechecks the source banks, model and external buffer bytes, image, destination files and pack manifest before publishing the coordinated definitions, mesh and optional texture. Controls and cancellation wait for background review/publication. Restore removes created outputs or recovers in-place originals. Other body alternatives and animations remain shared; this does not replace a complete creature or establish gameplay compatibility. +Enable **Import a custom body model** in that draft to choose one explicit body part/alternative and a skinned `.glb` or `.gltf` matching the donor skeleton. Supply a new `MESH_` name using uppercase letters, digits and underscores, then choose an existing texture ID or a new named image. New images support DXT1, DXT3 and ARGB8888; models can be compressed or stored. **Review custom import** checks the model before offering Create. Changing any input or destination requires a fresh review. Create rechecks the source banks, model and external buffer bytes, image, destination files and pack manifest before publishing the coordinated definitions, mesh and optional texture. Controls and cancellation wait for background review/publication. Pack imports refresh Models and preview their diffuse textures using a separate pack context; the loaded world's assets stay unchanged. Restore removes created outputs or recovers in-place originals. Other body alternatives and animations remain shared; this does not replace a complete creature or establish gameplay compatibility. The same private mode is available through `forge-tools defs clone-creature-private --schema --out [--names ] [--json]`. diff --git a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md index 0ce15fd7..dda5021a 100644 --- a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md @@ -113,3 +113,139 @@ preview; the current pack fixture uses one folder for both. Preview texture context and whole-creature assembly remain separate from filesystem acceptance. Runtime/placement remain open; do not repeat the unchanged full native graph gate or write full native banks merely to test the UI. + +## F2Native coordination and offline preview preparation + +Cache fix committed/pushed as `1da1ab3`; draft PR 2 records the regression and +newer-than-checkpoint limitation. Windows CI run 39954426240 now completed with +success, independently read back at 06:52 Edmonton. The original implementation +run was 39953743747; its outcome was not inferred from the successor. This CI +does not cover the newer uncommitted texture-context candidate. + +F2Native thread `01a168a5-fbee-72f0-87fe-39450caddb64` requested a bounded hidden +capture window. Responded with `codex queue` using message +`01a168c3-eece-78f0-870f-b55c6088a0b1`, identifying the existing shared +`D:/Documents/FableTLC/.agents/runtime-lease.json` and its exclusive-create/token +helper in the texture checkout's `tools/ingame/runtime_lease.py`. It was absent at +the readback, and no owned Forge GUI/test process was active. An unrelated native +particle test was left alone. No games were stopped. + +The peer then explicitly required no GPU/game tests until a queued release. +Acknowledged through queue message `01a168c5-7ab3-7550-86ee-c98d5055c42c`. +**This supersedes the earlier timed window. Do not infer release from the clock, +90-second estimate, 420-second watchdog, process disappearance or absent lease.** +Wait for the explicit message, then recheck the shared lease and active games. +Current local receipt: `build/runtime-coordination-20261023.json`. Overall Forge +development remains active; source/docs work continues offline. + +Uncommitted preview work; GPU execution still deferred: + +- Pack fixtures now use a separate save folder and require it to remain absent. + `pack-image-preview` primes Models, imports an image into the pack, selects the + appended mesh, and captures its viewport at both sizes. +- Renderer exposes a read-only count of model parts holding actual texture SRVs; + `model_textured_parts` reports it to automation. The new case requires one + textured part; untextured pack preview requires zero. This distinguishes + drawable geometry from successfully bound diffuse imagery. +- Static tracing shows model preview uses `ctx_` for diffuse decoding while pack + completion deliberately retains the install context. New pack texture ID 8 + cannot resolve in the tiny source bank containing only ID 7. This motivates + the regression, but it is not a runtime failure claim yet. Keep global world + context independent of undeployed pack contents when choosing the fix. + +CPU-only diagnostic GUI build passes at below-normal priority/j2. Frozen before +the texture-context candidate at +`build/frozen/pack-preview-diagnostics-20261023/FableForge.exe`, SHA256 +`40163fe36224f3f72f0ada0488135de95200b2bc9d63d538c568d1bbce2668cb`. +Its `provenance.json` pins the executable and exact diagnostic source files +beyond starting `1da1ab3`. It includes the SRV count but predates the subsequent +`asset_context_root` state key; account for this when interpreting negative-control +logs. No baseline GPU run has occurred during the hold. + +Read-only Python audit of retained native-donor output independently confirms +mesh 4522 references texture 8; source bank has only 7 while pack bank has 7 and 8. +All three input hashes remain unchanged. Receipt: +`build/pack-preview-texture-source-audit-20261023.json`. No executable/GPU launch. + +Candidate source now prepares a separate Context after successful pack publication +inside the existing import worker, choosing the appended bank or its exact source +bank when no image was added. Completion retains that context only for the resolved +physical graphics bank; Models uses it for diffuse lookup. The world `ctx_` stays +unchanged. Normal context reload clears the model-only context. A preview-load +failure remains separate from already-successful publication and blocks that +pack's texture preview with its own diagnostic. `asset_context_root` additionally +lets positive fixtures assert the original world root survives pack creation. +Candidate compiled successfully below-normal/j2 with output in +`build/pack-model-texture-context-build-20261023.log`; no runtime acceptance yet. +Frozen candidate: +`build/frozen/pack-preview-candidate-20261023/FableForge.exe`, SHA256 +`3ac5c5f9cbcc4a4b06b472eea1638c48f17181e721264c8a7f854d5a25782d49`. +Its provenance records exact source hashes. Comparison receipt: +`build/pack-preview-frozen-comparison-20261023.json`. The negative-control driver +is a scratch copy of the GUI gate omitting only the newer world-context state key +unavailable in the frozen diagnostic; it retains the renderer texture assertion +and all source/output checks. The production positive gate keeps both assertions. +Both Python drivers compile; whitespace check passes. + +After explicit release, recheck shared lease and game processes. Use the frozen +diagnostic GUI for the pack-image negative control, then the candidate GUI for +pack-preview/pack-image-preview and existing pack neighbors. Inspect the actual +viewport, require the SRV count and unchanged world context, and retain exact +delta/Restore/source preservation. Do not weaken the new SRV assertion or +silently switch the global world context to the undeployed pack. + +Further offline driver preparation adds `pack-existing-preview`: a pack using +source texture ID 7 must bind one texture without creating a new textures.big. +The image-preview case also refreshes Models and reselects the imported mesh, +then switches back to the original install and requires the new pack mesh to +disappear from the model list while the original remains drawable. The world +context root must stay at the original install throughout. Driver receipts now +pin the driver's startup SHA256 and refuse if it changes during execution; +supplied donor packets must have a passed result receipt. These additions are +syntax/whitespace checked only; runtime acceptance remains pending. The earlier +frozen negative-control driver remains intentionally unchanged. + +## Source acceptance after explicit F2Native release + +The peer explicitly released its hold. Immediate process/lease checks found no +Fable/xenia/V83 process and an absent lease. A bounded supervisor acquired the +existing RuntimeLease protocol, and each hidden GUI child repeated lease-token, +live-owner and Fable/xenia/V83 process checks before launch. No user-owned process +was closed. Both binaries were the pinned frozen snapshots above. + +Canonical successful window: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/pv-x1mr2v9q/result.json`. +The baseline creates/selects the imported mesh successfully and fails only +`model_textured_parts=0 != 1`; its four created outputs were removed with explicit +Restore. Inner receipt: `creature-import-ui-fi5ozchh`. +Candidate passes ten focused cases (`creature-import-ui-c1k034hm`) and the full +50-case gate (`creature-import-ui-bdonedle`). The focused cases are a subset, +not 60 distinct cases. Pack/source texture selection, actual SRV binding, +refresh, source switching and unchanged world context all pass at both sizes. +Exact definition/pool/identity/asset preservation and Restore remain intact; +native donor fixture inventory stays unchanged. Inspected compact 800x600 +pack-image model viewport: the triangle is drawn with the red/green image's +filtered colour, and the appended model is selected. + +All owned GUIs closed, and the supervisor released the lease with +`pv-x1mr2v9q/released-runtime-lease.json`. Immediate process readback was empty; +lease absence was verified. Peer availability queued as +`01a168de-3d41-7591-90f2-0b03237ec025`. New GUI windows must claim the lease again; +do not treat the peer's earlier release as permanent exclusive ownership. + +Two earlier runner attempts are retained. A deeply nested build scratch root +refused recovery-marker preparation before publication; a shorter C evidence +root succeeds without any production guard changes. That failed attempt is not +the texture regression. A second attempt reached the intended texture failure, +but the runner accidentally included a traceback's quoted `FAIL ` literal in +its failure filter; fixed to match timestamped automation failure lines only. +Its created outputs were independently removed with Restore. Successful canonical +negative control now requires every automation failure to be the missing SRV. +Logs: `build/pack-preview-window-svdvrnhu` and +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/pv-5p1xve_x`. + +Source texture-context correction is accepted. Next: commit the scoped change, +prepare a distinct immutable development checkpoint, then exercise its extracted +GUI under a new shared-lease window. The earlier ZIP remains untouched and still +has its documented cache/texture-preview limitations. No gameplay or assembled +creature claim follows from these per-mesh fixtures. diff --git a/gui/app.cpp b/gui/app.cpp index 403c98d6..2dacbec2 100644 --- a/gui/app.cpp +++ b/gui/app.cpp @@ -919,6 +919,7 @@ void App::startContextLoad(const std::string& fromRoot) { modelsLoaded_ = modelUsersLoaded_ = modelReady_ = false; modelRows_.clear(); modelUsers_.clear(); modelName_.clear(); modelError_.clear(); modelGeometry_ = {}; modelId_ = 0; + modelPreviewContext_.reset();modelPreviewGraphics_.clear();modelPreviewContextError_.clear(); renderer_.clearModelPreview(); renderer_.clearHeadPreview(); dialogueHeadReady_=false; @@ -1978,6 +1979,8 @@ std::vector App::stateDump() const { v.push_back("models_filtered=" + std::to_string(modelFiltered_)); v.push_back("model_selected=" + modelName_); v.push_back("model_ready=" + std::to_string(modelReady_)); + v.push_back("model_textured_parts=" + std::to_string(renderer_.modelTexturedParts())); + v.push_back("asset_context_root=" + ctx_.gameRoot().generic_string()); v.push_back("model_import_open=" + std::to_string(modelImportOpen_)); v.push_back("mesh_name=" + std::string(meshName_[0] ? meshName_ : "-")); v.push_back("map_list_visible=" + std::to_string(settings_.showExplorer && !standaloneAssetPreview())); diff --git a/gui/app.hpp b/gui/app.hpp index 2f9bfe90..d5a196ca 100644 --- a/gui/app.hpp +++ b/gui/app.hpp @@ -1064,6 +1064,11 @@ class App { std::map> modelUsers_; bool modelsLoaded_ = false, modelUsersLoaded_ = false, modelReady_ = false, modelImportOpen_ = false; std::string modelRoot_, modelBankPath_, modelName_, modelError_; + // An undeployed pack may contain textures absent from the loaded world. + // Built by the import worker and used only for its physical graphics bank. + std::shared_ptr modelPreviewContext_; + std::filesystem::path modelPreviewGraphics_; + std::string modelPreviewContextError_; uint32_t modelId_ = 0; forge::meshpreview::Geometry modelGeometry_; char modelSearch_[96] = {}; diff --git a/gui/definitionbrowser.cpp b/gui/definitionbrowser.cpp index 4fba3088..8f780f3c 100644 --- a/gui/definitionbrowser.cpp +++ b/gui/definitionbrowser.cpp @@ -37,6 +37,10 @@ struct App::CreatureImportDraft { std::optional texture; forge::defschema::Schema schema; fs::path root,graphicsRelative,resolvedRoot; + bool packPreview=false; + std::shared_ptr previewContext; + fs::path previewGraphics; + std::string previewError; std::array inputs; struct Target {fs::path path,resolved;std::string hash;bool present;}; std::vector targets; @@ -87,6 +91,7 @@ bool App::prepareCreatureImport() { draft->sources={definitionEditNames_,definitionEditBank_, read({"data/graphics/graphics.big","data/graphics/pc/graphics.big"}),read({"data/graphics/pc/textures.big"})}; draft->root=destination.parent_path().parent_path().parent_path();draft->resolvedRoot=paths::resolvedPath(draft->root); + draft->packPreview=!packDest_.empty(); draft->graphicsRelative=draft->sources.graphics.parent_path().filename()=="pc"?"data/graphics/pc/graphics.big":"data/graphics/graphics.big"; draft->schema=definitionSchema_; const auto slot=definitionCreatureSlots_[*definitionCreatureSlot_]; @@ -140,6 +145,16 @@ bool App::saveCreatureImport() { const auto result=creatureimport::publishMesh(draft->sources,draft->proof,draft->root,draft->graphicsRelative, draft->schema,draft->request,draft->model,draft->materials,true,[draft]{draft->validate();},draft->texture?&*draft->texture:nullptr); job.published=true;job.rootIndex=result.clones.rootIndex; + if(draft->packPreview) { + // Publication is already complete. A preview load failure + // must not be reported as a failed/rolled-back import. + try { + draft->previewGraphics=paths::resolvedPath(draft->root/draft->graphicsRelative); + auto context=std::make_shared(); + const auto textures=draft->texture?draft->root/"data/graphics/pc/textures.big":draft->sources.textures; + if(context->load(draft->root,textures,draft->previewError)) draft->previewContext=std::move(context); + } catch(const std::exception& error) {draft->previewError=error.what();} + } } catch(const std::exception& error) {job.error=error.what();} return job; });definitionEditConfirm_=false;return true; @@ -164,7 +179,12 @@ void App::pollCreatureImport() { defThumbs_.clear();defList_.clear();themeGroupOf_.clear();envDefs_.clear();soundDefs_.clear();familyNames_.clear();issuesRev_=~0ull; if(packDest_.empty() || (!ctx_.gameRoot().empty() && paths::resolvedPath(job.draft->root)==paths::resolvedPath(ctx_.gameRoot()))) { ctxError_.clear();reloadWhenContextReady_=true;startContextLoad(job.draft->root.string()); - } else refreshModels(); + } else { + modelPreviewContext_=job.draft->previewContext;modelPreviewGraphics_=job.draft->previewGraphics; + modelPreviewContextError_=job.draft->previewError; + if(!modelPreviewContextError_.empty()) pushLog("creature import completed; model texture preview: "+modelPreviewContextError_,1); + refreshModels(); + } refreshTextures();rescanBackups();pushLog(definitionEditSuccess_,3); } std::pair App::definitionPaths() const { diff --git a/gui/renderer.hpp b/gui/renderer.hpp index 9d9be07c..1a46bfa2 100644 --- a/gui/renderer.hpp +++ b/gui/renderer.hpp @@ -304,6 +304,9 @@ class Renderer { const std::vector& images, uint32_t size); void clearModelPreview(); bool setModelPreview(const foliageexport::Mesh& mesh, const std::vector& images); + size_t modelTexturedParts() const { + size_t count=0;for(const auto& part:modelMesh_.parts) if(part.srv) ++count;return count; + } ID3D11ShaderResourceView* modelPreview(uint32_t size, float yaw, float pitch, float zoom, bool wire); void clearHeadPreview(); bool setHeadPreview(const foliageexport::Mesh& mesh, const std::vector& images); diff --git a/gui/textures.cpp b/gui/textures.cpp index a402a8dc..f4c428bc 100644 --- a/gui/textures.cpp +++ b/gui/textures.cpp @@ -10,6 +10,7 @@ #include "texturebrowse.hpp" #include "meshimport.hpp" #include "forge/big.hpp" +#include "resolvedpath.hpp" namespace albion::gui { using theme::S; @@ -465,7 +466,12 @@ bool App::selectModelById(uint32_t id) { std::vector images; std::map textureIds; std::vector warnings; - const auto mesh = foliageexport::makeMesh(modelId_, modelName_, modelName_, modelGeometry_, true, ctx_, images, textureIds, warnings); + const terrainexport::Context* textureContext=&ctx_; + if(!modelPreviewGraphics_.empty() && paths::resolvedPath(modelBankPath_)==modelPreviewGraphics_) { + if(!modelPreviewContext_) throw std::runtime_error("Pack model texture preview unavailable: "+modelPreviewContextError_); + textureContext=modelPreviewContext_.get(); + } + const auto mesh = foliageexport::makeMesh(modelId_, modelName_, modelName_, modelGeometry_, true, *textureContext, images, textureIds, warnings); modelReady_ = renderer_.setModelPreview(mesh, images); if (!modelReady_) modelError_ = "This model has no drawable geometry."; for (const auto& warning : warnings) pushLog("models: " + warning, 1); diff --git a/tools/test_creature_import_ui.py b/tools/test_creature_import_ui.py index 5536fd2a..a4aa823a 100644 --- a/tools/test_creature_import_ui.py +++ b/tools/test_creature_import_ui.py @@ -21,19 +21,21 @@ def inventory(root): def main(): repo = Path(__file__).resolve().parents[1] + driver_hash = hashlib.sha256(Path(__file__).read_bytes()).hexdigest() parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--gui', type=Path, default=repo/'build/FableForge.exe') parser.add_argument('--forge', type=Path, default=repo/'build/forge.exe') parser.add_argument('--fixtures', type=Path, default=os.environ.get('FABLEFORGE_CREATURE_FIXTURES'), help='Retained tiny donor packet from test_skinned_input_donors.py') parser.add_argument('--work-root', type=Path, default=repo/'build') - modes = ('save', 'stored', 'cancel', 'missing-slot', 'existing', 'image', 'image-dxt3', 'image-argb8888', 'pack', 'pack-preview', 'close-review', 'stale-model', 'stale-sidecar', 'stale-image', 'stale-bank', 'stale-names', 'stale-graphics', 'stale-textures', 'stale-manifest', 'stale-model-alias', 'stale-destination-alias', 'destination', 'changed-draft') + modes = ('save', 'stored', 'cancel', 'missing-slot', 'existing', 'image', 'image-dxt3', 'image-argb8888', 'pack', 'pack-preview', 'pack-image-preview', 'pack-existing-preview', 'close-review', 'stale-model', 'stale-sidecar', 'stale-image', 'stale-bank', 'stale-names', 'stale-graphics', 'stale-textures', 'stale-manifest', 'stale-model-alias', 'stale-destination-alias', 'destination', 'changed-draft') parser.add_argument('--only', nargs='+', choices=modes) args = parser.parse_args() if args.fixtures is None: print('SKIP custom Creature GUI: supply --fixtures or FABLEFORGE_CREATURE_FIXTURES with a retained tiny donor packet') return fixtures = args.fixtures.resolve(strict=True) + assert json.loads((fixtures/'result.json').read_text())['passed'], 'donor fixture packet has no passed receipt' source = fixtures/'4521/source-install' before = inventory(fixtures) originals = read_bank(source) @@ -55,7 +57,9 @@ def main(): try: for size, scale in [('1440x900', 1), ('800x600', 1.5)]: for mode in args.only or modes: - has_image = mode.startswith('image') or mode == 'stale-image' + has_image = mode.startswith('image') or mode in ('stale-image', 'pack-image-preview') + preview = mode in ('pack-preview', 'pack-image-preview', 'pack-existing-preview') + existing_texture = mode in ('existing', 'pack-existing-preview') case = work/(size+'-'+mode) case.mkdir() install = case/'install' @@ -72,7 +76,8 @@ def main(): schema.parent.mkdir(parents=True) shutil.copyfile(fixtures/'4521/schema.json', schema) output = case/'output' - pack = mode in ('pack', 'pack-preview', 'stale-manifest') + pack = preview or mode in ('pack', 'stale-manifest') + save_folder = case/'separate-save-folder' if pack else output manifest = None if pack: output.mkdir() @@ -80,10 +85,10 @@ def main(): manifest.write_text(json.dumps({'version': 1, 'name': 'Creature UI'})) original_manifest = manifest.read_bytes() if manifest else None lines = ['wait_maps', 'wait_ready', f'set uiscale {scale}', - f'set saveroot {output.as_posix()}'] + f'set saveroot {save_folder.as_posix()}'] if pack: lines += [f'pack_dest {output.as_posix()}'] - if mode == 'pack-preview': + if preview: lines += ['assets_tab 1', 'model_select 4521', 'assert_state model_ready 1'] lines += ['assets_tab 6', 'wait_definitions', 'definition_select 1', 'frames 3'] @@ -98,7 +103,7 @@ def main(): for widget, text in [('input_definition_creature_model', str(inputs/'source.gltf')), ('input_definition_creature_mesh_name', 'MESH_GUI_MODEL')]: lines += click(widget)+['input_text '+text, 'frames 3'] - if mode == 'existing': + if existing_texture: lines += click('input_definition_creature_texture_id')+['input_text 7', 'frames 3'] if has_image: lines += click('input_definition_creature_texture')+['input_text '+str(inputs/'diffuse.png'), 'frames 3'] @@ -132,9 +137,18 @@ def main(): else: lines += ['assert_state definition_edit_failed 0', 'assert_state definition_editing 0', 'assert_state definition_selected 6', f'screenshot {(case/"created.png").as_posix()}'] - if mode == 'pack-preview': + if preview: lines += ['assets_tab 1', 'model_select 4522', 'assert_state model_selected MESH_GUI_MODEL', - 'assert_state model_ready 1', 'assert_state model_vertices 3', 'assert_state model_triangles 1'] + 'assert_state model_ready 1', 'assert_state model_vertices 3', 'assert_state model_triangles 1', + f'screenshot {(case/"model-preview.png").as_posix()}'] + lines += [f'assert_state model_textured_parts {1 if has_image or existing_texture else 0}'] + lines += [f'assert_state asset_context_root {install.as_posix()}'] + if mode == 'pack-image-preview': + lines += click('btn_model_refresh')+['model_select 4522', 'assert_state model_textured_parts 1', + f'assert_state asset_context_root {install.as_posix()}', + f'set saveroot {install.as_posix()}', 'frames 3', 'model_search MESH_GUI_MODEL', 'frames 3', + 'assert_state models_filtered 0', 'model_select 4521', 'assert_state model_ready 1', + f'assert_state asset_context_root {install.as_posix()}'] lines += ['quit'] script = case/'import.txt' script.write_text('\n'.join(lines)+'\n', encoding='utf-8') @@ -178,7 +192,7 @@ def main(): process.wait(timeout=10) trace = Path(str(script)+'.log').read_text(encoding='utf-8') assert returncode == 0 and 'RESULT PASS' in trace and 'FAIL ' not in trace, trace[-8000:] - if mode in ('save', 'stored', 'existing', 'image', 'image-dxt3', 'image-argb8888', 'pack', 'pack-preview', 'close-review'): + if preview or mode in ('save', 'stored', 'existing', 'image', 'image-dxt3', 'image-argb8888', 'pack', 'close-review'): actual = read_bank(output) assert actual[:6] == originals and len(actual) == 9 old_bank = (source/'data/CompiledDefs/game.bin').read_bytes() @@ -222,12 +236,15 @@ def main(): assert not (case/'other-output').exists() if manifest: assert manifest.read_bytes() == original_manifest+(b' ' if mode == 'stale-manifest' else b'') + assert not save_folder.exists(), 'pack import wrote to the independent save folder' cases.append({'size': size, 'mode': mode}) print(size, mode, 'PASS', flush=True) finally: assert inventory(fixtures) == before, 'fixture sources changed' assert hashlib.sha256(gui_source.read_bytes()).hexdigest() == gui_hash + assert hashlib.sha256(Path(__file__).read_bytes()).hexdigest() == driver_hash, 'test driver changed during the run' (work/'result.json').write_text(json.dumps({'passed': True, 'cases': cases, 'gui_sha256': gui_hash, + 'driver_sha256': driver_hash, 'schema_source': 'tiny donor fixture', 'runtime': 'byte-identical executable and adjacent DLL copies'}, indent=2)+'\n') From e8280108b42916b4eebcb3bcb59e0e1eeb078012 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:18:47 -0600 Subject: [PATCH 53/75] Record extracted pack texture checkpoint and shared launch evidence --- docs/HANDOFF_CREATURE_GUI_2026-10-23.md | 56 +++++++++++++++++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 2 +- 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md index dda5021a..83784c3d 100644 --- a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md @@ -249,3 +249,59 @@ prepare a distinct immutable development checkpoint, then exercise its extracted GUI under a new shared-lease window. The earlier ZIP remains untouched and still has its documented cache/texture-preview limitations. No gameplay or assembled creature claim follows from these per-mesh fixtures. + + +## Texture-context checkpoint prepared + +Source correction committed and pushed as `65dba50d555f5d0204866a4ee80a526236642720`. +Windows CI run 39959326852 is in progress; draft PR #2 records the source gate. +New immutable package: `dist/development/20261023-pack-model-textures/FableForge-0.18.1-dev-win64.zip`, +SHA256 `2e5addece0de168c4f4a7dec4c079e1115548b1bae59d6adcf4d577d36e31413`. +It was prepared from the clean source commit with `package.py --no-check`; +this is packaging, not an extracted-runtime acceptance claim. +The old custom-import ZIP remains unchanged. + +A fresh extracted-window preflight found unrelated native particle tests, +then definition-catalog tests. No lease was claimed or GUI launched, and those +processes were left alone. Peer notified through queue message +`01a168e4-5e19-7752-a3c3-549001b6c296`. The prepared bounded runner +`build/run_pack_package_window_20261023.py` will claim the shared lease only +when the immediate process guard is clear. Its scoped sitecustomize guard is +inherited by child Python drivers and verifies token/live owner/processes before +every FableForge launch. Planned extracted workflows: creature_import_ui, gui, +and retail. No unchanged core gate is repeated. + + +Offline extraction acceptance passes for the new ZIP: 118 file identities, +142 local documentation links, and the eight-map retail export smoke. +Receipt: `C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/package-smoke-9mmhqrzx/result.json`. +This invocation ran one workflow, left 147 not run, and explicitly skipped two +unrelated missing-corpus workflows. The remaining GUI window will select only +`gui` and `creature_import_ui`; retail will not be repeated. + + +## Extracted texture-preview checkpoint acceptance + +The new archive passes GUI smoke and all 50 custom-import cases at both sizes, +including actual texture SRVs, Refresh/source switching, unchanged global world +context, physical-alias refusals, exact preservation and Restore. Receipt: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/pk-1q8hw2fe/package-smoke-8r8legjw/result.json`; +inner custom gate `creature-import-ui-4pcwvqs3/result.json`. Extracted GUI SHA256 +`697ad0d49496249d186d7c8ffe766f7a51764c11bb9bfeed73e32aa80b922ed0`. +Combined with offline retail: three distinct workflows passed, 145 not run, +two unrelated missing-corpus skips. Combined receipt: +`build/pack-texture-checkpoint-20261023.json`. + +All owned GUIs closed; shared lease released and absence read back. Release +receipt `pk-1q8hw2fe/released-runtime-lease.json`; peer availability queued as +`01a168e9-6932-70e1-a36b-a986378c6a54`. An earlier attempt claimed/released +without launching because Python could not resolve PowerShell's codex wrapper; +the runner now uses the explicit codex.cmd path. No peer process was closed. + +Screenshot inspection found a separate presentation issue: the pack's diffuse +texture visibly renders, but Materials still queries the base Textures list and +reports texture ID 8 missing. This does not invalidate the actual SRV gate, but +is a known UI limitation of this immutable archive. Next source correction should +show material identity from the same context used by the model preview and avoid +navigating into an unrelated base texture with a coincident ID. Keep this ZIP +unchanged; no whole-creature assembly, placement or gameplay claim. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 006fc0c3..a522bdcf 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Custom body-slot/model/texture controls are connected to reviewed background publication. All 44 custom-import GUI cases pass against the immutable extracted checkpoint, with exact deltas and Restore. Three affected CTests and five focused extracted workflows pass; gameplay remains unverified. RC2 unchanged. +> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Reviewed private model/texture imports pass all 50 extracted GUI cases. The new immutable pack-texture checkpoint passes 118 identities, 142 documentation links and three focused workflows. Its Materials panel still uses the base texture list; source correction is next. Gameplay remains unverified; RC2 unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 8220ae91..235dfda3 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Custom body-slot/model/texture controls are connected to reviewed background publication. All 44 custom-import GUI cases pass against the immutable extracted checkpoint, with exact deltas and Restore. Three affected CTests and five focused extracted workflows pass; gameplay remains unverified. RC2 unchanged. +> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Reviewed private model/texture imports pass all 50 extracted GUI cases. The new immutable pack-texture checkpoint passes 118 identities, 142 documentation links and three focused workflows. Its Materials panel still uses the base texture list; source correction is next. Gameplay remains unverified; RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there From 730ea7055964ed365ca6549e70672c52170b50a8 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:27:10 -0600 Subject: [PATCH 54/75] Show pack model material identity from its rendered texture source --- docs/AUTOMATION.md | 5 +- docs/EDITOR.md | 2 +- docs/HANDOFF_CREATURE_GUI_2026-10-23.md | 66 +++++++++++++++++++++++++ gui/app.cpp | 2 +- gui/app.hpp | 2 + gui/textures.cpp | 19 +++++-- tools/test_creature_import_ui.py | 6 ++- 7 files changed, 94 insertions(+), 8 deletions(-) diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md index 04c21c34..e039af82 100644 --- a/docs/AUTOMATION.md +++ b/docs/AUTOMATION.md @@ -1140,7 +1140,10 @@ supplies the tiny schema, leaving packaged retail schemas untouched. separate pack/save folders, require actual diffuse binding, retain the original world context, refresh Models, and switch back to the install. An existing-texture case also checks that no texture bank is created. The GUI driver's result pins its -startup SHA256 and refuses a driver change during execution. +startup SHA256 and refuses a driver change during execution. Pack material metadata registers +`model_preview_texture__` only +when the selected preview context resolves that texture. Clear the retained +widget registry before asserting absence after a source switch. `creature_clone_gui_retail_1440x900` and `_800x600` independently compare all prior retail records, verified self-only deltas, paired recovery and explicit diff --git a/docs/EDITOR.md b/docs/EDITOR.md index 39a8d755..405d3c49 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1448,7 +1448,7 @@ The same guarded shared-component mode is available through `forge-tools defs cl The separate **Clone private creature visuals** action appends a coordinated root, CCreatureDef and CAppearanceDef, with explicit unique bounded ASCII names. Selected visual rows become owned by the new root; other gameplay components retain their original data and ownership. Without a custom model, models, textures and animation assets remain shared. The paired publisher independently recomputes the three-record plan before writing names.bin and game.bin together, retaining recovery records. Unsupported/malformed known layouts, missing visual components, unresolved references and new name-CRC ambiguities refuse before publication. Root, body and appearance names are separate fields in the scrolling card. Cancel and close-review retain the existing draft protection. Use owned outputs first; no gameplay validation is claimed. -Enable **Import a custom body model** in that draft to choose one explicit body part/alternative and a skinned `.glb` or `.gltf` matching the donor skeleton. Supply a new `MESH_` name using uppercase letters, digits and underscores, then choose an existing texture ID or a new named image. New images support DXT1, DXT3 and ARGB8888; models can be compressed or stored. **Review custom import** checks the model before offering Create. Changing any input or destination requires a fresh review. Create rechecks the source banks, model and external buffer bytes, image, destination files and pack manifest before publishing the coordinated definitions, mesh and optional texture. Controls and cancellation wait for background review/publication. Pack imports refresh Models and preview their diffuse textures using a separate pack context; the loaded world's assets stay unchanged. Restore removes created outputs or recovers in-place originals. Other body alternatives and animations remain shared; this does not replace a complete creature or establish gameplay compatibility. +Enable **Import a custom body model** in that draft to choose one explicit body part/alternative and a skinned `.glb` or `.gltf` matching the donor skeleton. Supply a new `MESH_` name using uppercase letters, digits and underscores, then choose an existing texture ID or a new named image. New images support DXT1, DXT3 and ARGB8888; models can be compressed or stored. **Review custom import** checks the model before offering Create. Changing any input or destination requires a fresh review. Create rechecks the source banks, model and external buffer bytes, image, destination files and pack manifest before publishing the coordinated definitions, mesh and optional texture. Controls and cancellation wait for background review/publication. Pack imports refresh Models and preview their diffuse textures using a separate pack context; the loaded world's assets stay unchanged. Materials shows verified names and dimensions from that preview source as read-only entries; normal install textures retain their navigation links. Restore removes created outputs or recovers in-place originals. Other body alternatives and animations remain shared; this does not replace a complete creature or establish gameplay compatibility. The same private mode is available through `forge-tools defs clone-creature-private --schema --out [--names ] [--json]`. diff --git a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md index 83784c3d..2a0c6a3a 100644 --- a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md @@ -305,3 +305,69 @@ is a known UI limitation of this immutable archive. Next source correction shoul show material identity from the same context used by the model preview and avoid navigating into an unrelated base texture with a coincident ID. Keep this ZIP unchanged; no whole-creature assembly, placement or gameplay claim. + + +## Materials identity follow-up prepared offline + +The selected pack model now retains the exact preview texture context and uses +its textureInfo metadata for Materials. Pack/source-snapshot entries show their +verified name and dimensions read-only, avoiding navigation to an unrelated base +texture with the same numeric ID. Normal install material links keep existing +behavior. Selection and context reload clear the retained selection snapshot. +Focused driver assertions require the rendered metadata widget for image/existing +pack textures, retain it after Refresh, and require it absent after switching to +the original donor. GUI rebuild passes; source snapshot is pinned under +`build/frozen/model-material-source-20261023`, SHA256 +`b340f5111a124839461ecb8aaf9ea139257a1df6f3a952e7b00ebb9ed1e545c8`. +This correction is not runtime accepted yet. F2Native owns opening-vulkan1 lease; +no GUI is launched while it exists. Peer readiness queued as +`01a168ec-585d-7173-9376-27b1895a339f`. + + +Materials follow-up diagnostics: the immutable checkpoint fails only the new +metadata-widget assertion after successful actual texture binding; four outputs +Restored. Negative receipt `mi-p_e_epfh/creature-import-ui-ubovo1ds` under the +20261023 C evidence root. Initial candidate shows correct `Diffuse 8: GUI_TEXTURE +(4 x 4)` in the screenshot, but a source-switch assertion mistakenly used the +cumulative widget registry. Explicit clear fixes that assertion. Next compact +attempts reveal a clipped Materials header: its children are not submitted by +ImGui while offscreen, so revealing a child cannot scroll its parent into view. +The final driver now reveals the registered Materials header first, then the +metadata row, clears the registry, and requires the row on the current screen. +The source-switch absence assertion also clears the registry. No metadata +assertion or production guard was removed. All failed created outputs explicitly +Restored; no owned GUI remains between windows. Retained attempts: +`mi-p_e_epfh`, `mi-tgljwu1q`, `mi-7tg_rdw9`. + +Header-instrumented GUI rebuild passes. Frozen snapshot: +`build/frozen/model-material-source-header-20261023/FableForge.exe`, SHA256 +`334ba3b7343e174f336e7ae872134d7d3e70fd5084afd2d1a45ac0434a39fed8`. +A new claim correctly refused an existing peer lease before launching any GUI; +waiting for a fresh clear window. Windows CI run 39959326852 for 65dba50 completed +success; this does not cover the newer local Materials correction. + + +## Materials identity source acceptance + +Final focused candidate gate passes all ten cases (five modes at both sizes): +pack image/existing/no-texture preview plus folder image/save neighbors. Receipt: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/mi-pvwas_r7/creature-import-ui-5b27g5sg/result.json`. +Candidate executable/source hashes match the pinned header-instrumented snapshot. +Initial metadata, parent-first reveal after Refresh, current-screen metadata and +source-switch absence pass. Texture SRVs, original world context, exact +preservation and Restore still pass. Inspected full-size screenshot shows +`Diffuse 8: GUI_TEXTURE (4 x 4)` with the textured triangle. Compact initial +screenshot does not show the below-fold Materials panel; the refreshed compact +metadata is verified by the current-screen registry assertion after parent reveal. + +All owned GUIs closed, lease released at 13:26:29 UTC, peer availability queued as +`01a168f1-0bc6-7733-8e94-f458a8c4e405`; receipt +`mi-pvwas_r7/released-runtime-lease.json`. Peer acquired a new token at 13:26:51; +a later blanket lease-absence assertion correctly failed. Verified different +owner/token and released own receipt, leaving peer lease untouched. +This is a newer source-only correction; retained pack-texture ZIP still has its +documented Materials limitation. No new package is required for this small +presentation follow-up. Next work is scratch placement/whole-creature presentation +planning using existing editor paths; gameplay and assembled-creature behavior +remain unverified. Do not repeat accepted full50/core/native gates without a +relevant change. diff --git a/gui/app.cpp b/gui/app.cpp index 2dacbec2..c63c94bb 100644 --- a/gui/app.cpp +++ b/gui/app.cpp @@ -919,7 +919,7 @@ void App::startContextLoad(const std::string& fromRoot) { modelsLoaded_ = modelUsersLoaded_ = modelReady_ = false; modelRows_.clear(); modelUsers_.clear(); modelName_.clear(); modelError_.clear(); modelGeometry_ = {}; modelId_ = 0; - modelPreviewContext_.reset();modelPreviewGraphics_.clear();modelPreviewContextError_.clear(); + modelPreviewContext_.reset();modelSelectedTextureContext_.reset();modelPreviewGraphics_.clear();modelPreviewContextError_.clear(); renderer_.clearModelPreview(); renderer_.clearHeadPreview(); dialogueHeadReady_=false; diff --git a/gui/app.hpp b/gui/app.hpp index d5a196ca..b2d97c6d 100644 --- a/gui/app.hpp +++ b/gui/app.hpp @@ -1067,6 +1067,8 @@ class App { // An undeployed pack may contain textures absent from the loaded world. // Built by the import worker and used only for its physical graphics bank. std::shared_ptr modelPreviewContext_; + // Retain the selected mesh's actual texture snapshot for material metadata. + std::shared_ptr modelSelectedTextureContext_; std::filesystem::path modelPreviewGraphics_; std::string modelPreviewContextError_; uint32_t modelId_ = 0; diff --git a/gui/textures.cpp b/gui/textures.cpp index f4c428bc..9fed17c5 100644 --- a/gui/textures.cpp +++ b/gui/textures.cpp @@ -10,6 +10,7 @@ #include "texturebrowse.hpp" #include "meshimport.hpp" #include "forge/big.hpp" +#include "forge/terraintex.hpp" #include "resolvedpath.hpp" namespace albion::gui { @@ -458,7 +459,7 @@ bool App::selectModelById(uint32_t id) { pushLog("models: ambiguous mesh ID " + std::to_string(it->id),1); return false; } if (!ctx_.ready() || ctxFuture_.valid()) { pushLog("models: wait for assets to finish loading", 1); return false; } - modelReady_ = false; modelGeometry_ = {}; renderer_.clearModelPreview(); + modelReady_ = false; modelGeometry_ = {}; modelSelectedTextureContext_.reset(); renderer_.clearModelPreview(); modelId_ = it->id; modelName_ = it->name; modelError_.clear(); modelYaw_ = 0.8f; modelPitch_ = 0.55f; modelZoom_ = 1; try { @@ -469,7 +470,8 @@ bool App::selectModelById(uint32_t id) { const terrainexport::Context* textureContext=&ctx_; if(!modelPreviewGraphics_.empty() && paths::resolvedPath(modelBankPath_)==modelPreviewGraphics_) { if(!modelPreviewContext_) throw std::runtime_error("Pack model texture preview unavailable: "+modelPreviewContextError_); - textureContext=modelPreviewContext_.get(); + modelSelectedTextureContext_=modelPreviewContext_; + textureContext=modelSelectedTextureContext_.get(); } const auto mesh = foliageexport::makeMesh(modelId_, modelName_, modelName_, modelGeometry_, true, *textureContext, images, textureIds, warnings); modelReady_ = renderer_.setModelPreview(mesh, images); @@ -614,13 +616,24 @@ void App::drawModelBrowser(float pad, float inner, float cardInner) { if (theme::ghostButton("Reset view", ImVec2(stackReset ? cardInner : resetWidth, S(28)))) { modelYaw_ = 0.8f; modelPitch_ = 0.55f; modelZoom_ = 1; } auto_.registerWidget("btn_model_reset"); theme::hint("Drag the preview to orbit. Scroll to zoom."); - if (ImGui::CollapsingHeader("Materials", ImGuiTreeNodeFlags_DefaultOpen)) { + const bool materialsOpen=ImGui::CollapsingHeader("Materials", ImGuiTreeNodeFlags_DefaultOpen); + auto_.registerWidget("header_model_materials"); + if (materialsOpen) { for (size_t i = 0; i < modelGeometry_.materials.size(); ++i) { const auto& mat = modelGeometry_.materials[i]; ImGui::PushID(int(i)); ImGui::Text("Material %d", mat.id); auto link = [&](const char* label, uint32_t id) { if (id == 0) return; + if(modelSelectedTextureContext_) { + std::string name; + forge::terraintex::TextureInfo info; + if(modelSelectedTextureContext_->textureInfo(id,name,info)) { + ImGui::TextWrapped("%s %u: %s (%u x %u)",label,id,name.c_str(),unsigned(info.frameWidth),unsigned(info.frameHeight)); + auto_.registerWidget(("model_preview_texture_"+std::to_string(i)+"_"+label).c_str()); + } else ImGui::TextDisabled("%s %u: absent from the model preview source",label,id); + return; + } std::string error; const auto* row = texbrowse::findTextureRowById(texRows_, "GBANK_MAIN_PC", uint32_t(id), error); const std::string caption = std::string(label) + " " + std::to_string(id); diff --git a/tools/test_creature_import_ui.py b/tools/test_creature_import_ui.py index a4aa823a..8a33aee6 100644 --- a/tools/test_creature_import_ui.py +++ b/tools/test_creature_import_ui.py @@ -143,11 +143,13 @@ def main(): f'screenshot {(case/"model-preview.png").as_posix()}'] lines += [f'assert_state model_textured_parts {1 if has_image or existing_texture else 0}'] lines += [f'assert_state asset_context_root {install.as_posix()}'] + if has_image or existing_texture: + lines += ['assert_widget model_preview_texture_0_Diffuse'] if mode == 'pack-image-preview': - lines += click('btn_model_refresh')+['model_select 4522', 'assert_state model_textured_parts 1', + lines += click('btn_model_refresh')+['model_select 4522', 'reveal header_model_materials', 'frames 3', 'reveal header_model_materials', 'frames 3', 'reveal model_preview_texture_0_Diffuse', 'frames 3', 'reveal model_preview_texture_0_Diffuse', 'frames 3', 'clear_widget_registry', 'frames 2', 'assert_state model_textured_parts 1', 'assert_widget model_preview_texture_0_Diffuse', f'assert_state asset_context_root {install.as_posix()}', f'set saveroot {install.as_posix()}', 'frames 3', 'model_search MESH_GUI_MODEL', 'frames 3', - 'assert_state models_filtered 0', 'model_select 4521', 'assert_state model_ready 1', + 'assert_state models_filtered 0', 'model_select 4521', 'clear_widget_registry', 'frames 2', 'assert_state model_ready 1', 'assert_widget_absent model_preview_texture_0_Diffuse', f'assert_state asset_context_root {install.as_posix()}'] lines += ['quit'] script = case/'import.txt' From bbfe9f49f9a26d05a0743eb115946edd070c1df3 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:28:24 -0600 Subject: [PATCH 55/75] Link accepted Materials identity follow-up from lane status --- docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index a522bdcf..97da992b 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Reviewed private model/texture imports pass all 50 extracted GUI cases. The new immutable pack-texture checkpoint passes 118 identities, 142 documentation links and three focused workflows. Its Materials panel still uses the base texture list; source correction is next. Gameplay remains unverified; RC2 unchanged. +> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Reviewed private model/texture imports pass all 50 extracted GUI cases. The new immutable pack-texture checkpoint passes 118 identities, 142 documentation links and three focused workflows. A newer Materials identity correction passes ten focused source cases; the immutable ZIP retains its documented panel limitation. Gameplay remains unverified; RC2 unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 235dfda3..daa6113d 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Reviewed private model/texture imports pass all 50 extracted GUI cases. The new immutable pack-texture checkpoint passes 118 identities, 142 documentation links and three focused workflows. Its Materials panel still uses the base texture list; source correction is next. Gameplay remains unverified; RC2 unchanged. +> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Reviewed private model/texture imports pass all 50 extracted GUI cases. The new immutable pack-texture checkpoint passes 118 identities, 142 documentation links and three focused workflows. A newer Materials identity correction passes ten focused source cases; the immutable ZIP retains its documented panel limitation. Gameplay remains unverified; RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there From 91d284e5c62a9234423374f3eb7965a4a29d9f39 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:47:22 -0600 Subject: [PATCH 56/75] Route definition placement by compiled class instead of name prefixes --- docs/AUTOMATION.md | 9 ++ docs/EDITOR.md | 2 +- docs/HANDOFF_CREATURE_GUI_2026-10-23.md | 106 ++++++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 2 +- gui/editor.cpp | 13 +- src/terrainexport.cpp | 6 + src/terrainexport.hpp | 2 + tools/check_all.py | 2 + tools/test_definition_placement_types_ui.py | 149 ++++++++++++++++++++ tools/test_package.py | 2 + 11 files changed, 286 insertions(+), 9 deletions(-) create mode 100644 tools/test_definition_placement_types_ui.py diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md index e039af82..6b421ffc 100644 --- a/docs/AUTOMATION.md +++ b/docs/AUTOMATION.md @@ -1145,6 +1145,15 @@ startup SHA256 and refuses a driver change during execution. Pack material metad when the selected preview context resolves that texture. Clear the retained widget registry before asserting absence after a source switch. +`definition_placement_types` runs two tiny loose-install GUI cases with eight +placements each: custom and misleading names, Creature/Object/Building classes, +Auto/explicit owners, undo/redo and the Actors Place button. An independent TNG +reader checks class, owner and creature InitialPos. Definitions, texture and LEV +stay exact; Restore retains only the byte-exact first-original TNG backup. +The standalone driver's `--prepare-only` writes and verifies fixtures/scripts +without launching an executable; that receipt explicitly does not claim a pass. +These marker fixtures do not test whole-creature geometry or gameplay. + `creature_clone_gui_retail_1440x900` and `_800x600` independently compare all prior retail records, verified self-only deltas, paired recovery and explicit Restore on owned outputs. Appearance/body/components remain shared. diff --git a/docs/EDITOR.md b/docs/EDITOR.md index 405d3c49..127f204f 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1448,7 +1448,7 @@ The same guarded shared-component mode is available through `forge-tools defs cl The separate **Clone private creature visuals** action appends a coordinated root, CCreatureDef and CAppearanceDef, with explicit unique bounded ASCII names. Selected visual rows become owned by the new root; other gameplay components retain their original data and ownership. Without a custom model, models, textures and animation assets remain shared. The paired publisher independently recomputes the three-record plan before writing names.bin and game.bin together, retaining recovery records. Unsupported/malformed known layouts, missing visual components, unresolved references and new name-CRC ambiguities refuse before publication. Root, body and appearance names are separate fields in the scrolling card. Cancel and close-review retain the existing draft protection. Use owned outputs first; no gameplay validation is claimed. -Enable **Import a custom body model** in that draft to choose one explicit body part/alternative and a skinned `.glb` or `.gltf` matching the donor skeleton. Supply a new `MESH_` name using uppercase letters, digits and underscores, then choose an existing texture ID or a new named image. New images support DXT1, DXT3 and ARGB8888; models can be compressed or stored. **Review custom import** checks the model before offering Create. Changing any input or destination requires a fresh review. Create rechecks the source banks, model and external buffer bytes, image, destination files and pack manifest before publishing the coordinated definitions, mesh and optional texture. Controls and cancellation wait for background review/publication. Pack imports refresh Models and preview their diffuse textures using a separate pack context; the loaded world's assets stay unchanged. Materials shows verified names and dimensions from that preview source as read-only entries; normal install textures retain their navigation links. Restore removes created outputs or recovers in-place originals. Other body alternatives and animations remain shared; this does not replace a complete creature or establish gameplay compatibility. +Enable **Import a custom body model** in that draft to choose one explicit body part/alternative and a skinned `.glb` or `.gltf` matching the donor skeleton. Supply a new `MESH_` name using uppercase letters, digits and underscores, then choose an existing texture ID or a new named image. New images support DXT1, DXT3 and ARGB8888; models can be compressed or stored. **Review custom import** checks the model before offering Create. Changing any input or destination requires a fresh review. Create rechecks the source banks, model and external buffer bytes, image, destination files and pack manifest before publishing the coordinated definitions, mesh and optional texture. Controls and cancellation wait for background review/publication. Pack imports refresh Models and preview their diffuse textures using a separate pack context; the loaded world's assets stay unchanged. Materials shows verified names and dimensions from that preview source as read-only entries; normal install textures retain their navigation links. Restore removes created outputs or recovers in-place originals. Other body alternatives and animations remain shared; this does not replace a complete creature or establish gameplay compatibility. The Actors palette and placement use the stored definition class, so a creature name does not need a `CREATURE_` prefix. Automatic ownership follows the creature definition; an object with a similar name stays an object. The same private mode is available through `forge-tools defs clone-creature-private --schema --out [--names ] [--json]`. diff --git a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md index 2a0c6a3a..81113623 100644 --- a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md @@ -371,3 +371,109 @@ presentation follow-up. Next work is scratch placement/whole-creature presentati planning using existing editor paths; gameplay and assembled-creature behavior remain unverified. Do not repeat accepted full50/core/native gates without a relevant change. + + +## Native window offered; offline placement investigation + +Peer requested a short Native window after our extracted-package release. Offer +queued as `01a168f3-19a6-70f2-877d-b31ec0c5f828`: no owned Forge frontend remains, +no Forge GPU work pending for this offered window, continue shared-lease/immediate +process guards. No peer lease/process reclaimed or closed. Current Materials +source CI run 39961421194 is in progress (730ea70); source gate remains accepted. + +Offline routing review found a concrete next candidate: placeDefinitionAt and +ownerFor use a CREATURE_ name prefix, while private Creature cloning explicitly +accepts generic ASCII names (the accepted GUI fixtures create GUI_ROOT). Context +already retains the compiled record's definition type and enumerates by that +identity. Before changing routing, prepare a scratch placement regression for a +non-prefixed CREATURE plus a misleading prefix on another type; verify TNG thing +class, owner and InitialPos/undo independently. This is source inspection only; +no placement/runtime acceptance claim and no additional GUI launch while peer +uses the offered window. + + +Native requested the next gap before any further Forge comparison for one D3D +real opening (~90 seconds; 420-second cold watchdog). Acknowledged by queue +`01a168f5-e210-7af3-99e5-6573601a0d07`: no Forge GUI/GPU launch until peer's +explicit release/notification, even if the shared lease temporarily disappears. +Current peer gpu-vulkan-off1 lease was observed and left untouched. This explicit +hold supersedes the earlier general available-window messages. Offline work may +continue; no owned Forge GUI was running at acknowledgment. + + +## Placement class routing prepared offline + +New regression `tools/test_definition_placement_types_ui.py` writes only tiny +owned synthetic definitions, one texture and a valid loose LEV/TNG. It covers a +non-prefixed CREATURE (`GUI_ROOT`), misleading CREATURE_ object name, unprefixed +building, ordinary names, Auto and explicit owner, placement undo/redo, and the +real Actors Place button. Independent TNG checks require AICreature/Object/ +Building classes, owner values and creature world-space InitialPos; Restore must +recover exact original inventory. `--prepare-only` launches no executable and +marks its receipt prepared_only=true, passed=false. Current prepared fixture: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/placement-types-_c_a4xmm`. +Six identity rows independently parse; this is preparation, not GUI acceptance. + +Proposed source change exposes the immutable compiled class through the existing +Context and uses it in placement, Auto ownership and the Actors Place button. +The UI hint no longer implies that CREATURE_ is mandatory. It reuses existing +Document serializers and changes no binary/TNG writer. Below-normal -j2 GUI build +is in progress; no test frontend launched. Runtime acceptance must first compare +old pinned Materials GUI against the candidate after explicit Native release. +Do not claim whole-creature rendering/gameplay: this fixture deliberately uses +editor markers, and Document's separate VILLAGER name heuristic remains outside +this class-routing correction. Live-link spawning also retains its old prefix +check and has not been exercised. + +Placement class-routing GUI build passed (including the final hint change). +Frozen candidate `build\frozen\definition-placement-types-20261023\FableForge.exe`, SHA256 `7d9915a9938a4e3a57c7635c8181c5542e45314c9ca4caab46eb2470611a9384`. +Provenance pins the three C++ sources and new driver. Prepared fixtures also +independently verify complete ordered schema tag anchors and owner tags at both +sizes. No GUI/runtime acceptance yet: Native explicit-release hold is active, +and the coordination receipt now clears the stale prior-release flag. Materials +source CI run 39961421194 completed success; placement candidate is newer. + + +Own queue was inspected read-only (`queue_1.sqlite`, mode=ro, this thread only). +Incoming `01a168f7-35cd-74b3-89ae-bd29b5ded43b` is a NEW explicit Native release: +D3D real opening passed 60.3 seconds/nine dialogue tags; frontend closed and lease +released; no need to hold indefinitely for the older request. Acknowledged by +`01a168ff-8574-71d3-80d6-6d0c7ac13314`. Coordination receipt clears the explicit +hold, but still requires a fresh exclusive claim and immediate process guards. + +Prepared placement-window supervisor refused an existing peer gpu-vulkan-off4 +lease before any GUI launch. Owner PID15076 is verified live as python.exe with +process start1792762928.3455365. Lease/token left untouched. Supervisor and pinned +baseline/candidate remain ready; wait for this actual process/window rather than +repeating build/tests or treating the prior release as exclusive ownership. + + +## Definition-class placement source acceptance + +Baseline independently saves GUI_ROOT as Object/owner4, CREATURE_DECOY as +AICreature/owner0, and PLAIN_HOUSE as Object; Actors Place ignores GUI_ROOT. +Receipt `C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/pt-evcljbni/baseline-independent-tng.json`; +baseline original TNG explicitly Restored. Candidate passes all eight placements +at both sizes (16 total), including Auto owners2/3, explicit owner1, ordinary names, +real Actors button, undo/redo counts and independent TNG class/InitialPos checks. +Canonical candidate receipt: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/pt-qkwy_fgp/placement-types-0dsrocc6/result.json`. +GUI hash matches frozen 7d9915a9938a4e3a57c7635c8181c5542e45314c9ca4caab46eb2470611a9384. +Driver pins startup GUI/driver hashes and refuses mutation during execution. + +An initial candidate passed its complete full-size GUI/TNG gate but the oracle +wrongly expected Restore to remove an existing-file first-original backup. Current +CLI defaults to keepBackup; reviewed backups::restore confirms this contract. +Original files were all exact and only PlacementProbe.tng.forge-orig was added. +Final gate strictly requires original inventory plus exactly that byte-identical +first-original backup, and passes at both sizes. No recovery implementation or +production guard changed. Initial receipt `pt-evcljbni/placement-types-_ga9vvaa`. +All owned GUIs closed; released lease receipt `pt-qkwy_fgp/released-runtime-lease.json`, +peer availability queued `01a16901-caad-7e33-babc-5a23f50ec008`. No immediate further +GUI launch planned while source review/docs proceed. + +Regression is registered as definition_placement_types in source and package +gates. No unchanged broad/core/import/native gate repeated. This corrects class +routing for custom-named definitions; marker fixtures do not prove custom body +assembly or live gameplay. Existing VILLAGER-name heuristic and live-link prefix +check remain separate, unexercised follow-ups. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 97da992b..5b6acaf2 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Reviewed private model/texture imports pass all 50 extracted GUI cases. The new immutable pack-texture checkpoint passes 118 identities, 142 documentation links and three focused workflows. A newer Materials identity correction passes ten focused source cases; the immutable ZIP retains its documented panel limitation. Gameplay remains unverified; RC2 unchanged. +> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Reviewed private model/texture imports pass all 50 extracted GUI cases. The new immutable pack-texture checkpoint passes 118 identities, 142 documentation links and three focused workflows. A newer Materials identity correction passes ten focused source cases; the immutable ZIP retains its documented panel limitation. Gameplay remains unverified; RC2 unchanged. Custom-name placement now passes sixteen marker placements at both sizes using compiled classes and correct owners. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index daa6113d..f46f2001 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Reviewed private model/texture imports pass all 50 extracted GUI cases. The new immutable pack-texture checkpoint passes 118 identities, 142 documentation links and three focused workflows. A newer Materials identity correction passes ten focused source cases; the immutable ZIP retains its documented panel limitation. Gameplay remains unverified; RC2 unchanged. +> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Reviewed private model/texture imports pass all 50 extracted GUI cases. The new immutable pack-texture checkpoint passes 118 identities, 142 documentation links and three focused workflows. A newer Materials identity correction passes ten focused source cases; the immutable ZIP retains its documented panel limitation. Gameplay remains unverified; RC2 unchanged. Custom-name placement now passes sixteen marker placements at both sizes using compiled classes and correct owners. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there diff --git a/gui/editor.cpp b/gui/editor.cpp index 558d360c..ec635b5d 100644 --- a/gui/editor.cpp +++ b/gui/editor.cpp @@ -2480,7 +2480,7 @@ void App::drawPlacementOptions(float width) { int App::ownerFor(const std::string& def) const { if (placeOwner_ >= 0) return std::min(placeOwner_, 4); - if (def.rfind("CREATURE_", 0) == 0) { + if (ctx_.definitionType(def) == "CREATURE") { if (const auto v = ctx_.defIntField(def, "DefaultOwner")) return int(*v); return 0; // defs not readable: FableForge's old creature default } @@ -2550,7 +2550,7 @@ bool App::placeDefinitionAt(const std::string& def, const float position[3], con if (code < 0 || modelId == 0) pushLog("editor: " + def + " has no mesh; it will appear as an editor marker", 1); forge::thingplacer::Placement p; p.definitionType = def; - p.thingType = def.rfind("BUILDING_", 0) == 0 ? "Building" : "Object"; + p.thingType = ctx_.definitionType(def) == "BUILDING" ? "Building" : "Object"; p.scriptName = scriptName; float at[3] = {position[0], position[1], position[2]}; snapPlacementPoint(at); @@ -2570,7 +2570,7 @@ bool App::placeDefinitionAt(const std::string& def, const float position[3], con if (fl > 1e-6f) p.forward = {fx / fl, fy / fl, 0.0f}; } try { - const bool creature = def.rfind("CREATURE_", 0) == 0; + const bool creature = ctx_.definitionType(def) == "CREATURE"; const float pos[3] = {p.position.x, p.position.y, p.position.z}; const float fwd[2] = {p.forward.x, p.forward.y}; const size_t n = creature ? doc_.placeCreature(pos, fwd, def, scriptName, p.player) : doc_.place(p); @@ -4715,12 +4715,13 @@ void App::drawEditPanel(float pad, float inner, float cardInner) { theme::label("Creatures by group"); ImGui::SetNextItemWidth(cardInner); ImGui::PushStyleVar(ImGuiStyleVar_FramePadding, ImVec2(S(10), S(6))); - ImGui::InputTextWithHint("##actorsearch", "Search creatures (CREATURE_...)", defSearch_, sizeof defSearch_); + ImGui::InputTextWithHint("##actorsearch", "Search creatures", defSearch_, sizeof defSearch_); ImGui::PopStyleVar(); auto_.registerWidget("input_actorsearch"); drawDefPalette("##actorlist", {"CREATURE"}, cardInner, S(260)); - const std::string actorLabel = placeDef_.rfind("CREATURE_", 0) == 0 ? "Place " + placeDef_ : std::string("Pick a creature above"); - if (theme::ghostButton(actorLabel.c_str(), ImVec2(cardInner, S(30))) && placeDef_.rfind("CREATURE_", 0) == 0) placeDefinition(placeDef_); + const bool actorSelected = ctx_.definitionType(placeDef_) == "CREATURE"; + const std::string actorLabel = actorSelected ? "Place " + placeDef_ : std::string("Pick a creature above"); + if (theme::ghostButton(actorLabel.c_str(), ImVec2(cardInner, S(30))) && actorSelected) placeDefinition(placeDef_); auto_.registerWidget("btn_place_actor"); drawRuleNotice("creature", cardInner); theme::endCard(); diff --git a/src/terrainexport.cpp b/src/terrainexport.cpp index 81bb5809..f2db953a 100644 --- a/src/terrainexport.cpp +++ b/src/terrainexport.cpp @@ -593,6 +593,12 @@ std::optional Context::definitionIndex(const std::string& name) const return std::nullopt; } +std::optional Context::definitionType(const std::string& name) const { + if (!impl_ || !impl_->defs) return std::nullopt; + const auto* entry = impl_->defs->find(name); + return entry ? std::optional(entry->definition) : std::nullopt; +} + std::vector Context::groupedDefinitions(const std::vector& types) const { std::vector out; if (!impl_ || !impl_->defs || !impl_->schema) return out; diff --git a/src/terrainexport.hpp b/src/terrainexport.hpp index efc02006..18306342 100644 --- a/src/terrainexport.hpp +++ b/src/terrainexport.hpp @@ -214,6 +214,8 @@ class Context { struct GroupedDefinition { std::string name, type, group; }; // A def's game.bin entry index (the value LEV palettes store); nullopt when absent. std::optional definitionIndex(const std::string& name) const; + // The compiled game.bin class; custom definition names need no type prefix. + std::optional definitionType(const std::string& name) const; std::vector groupedDefinitions(const std::vector& types) const; // Defs only (names.bin + game.bin, no textures.big): enough for themeLibrary(). bool loadDefs(const std::filesystem::path& gameRoot, std::string& error); diff --git a/tools/check_all.py b/tools/check_all.py index fcaa750a..348a66dc 100644 --- a/tools/check_all.py +++ b/tools/check_all.py @@ -373,6 +373,8 @@ def main(): [sys.executable, "tools/test_creature_clone_ui.py"], capture_output=True) ok &= run("Custom Creature GUI, reviewed source proofs, exact deltas and Restore", [sys.executable, "tools/test_creature_import_ui.py"], capture_output=True) + ok &= run("Definition class placement, owners, Actors button and exact recovery", + [sys.executable, "tools/test_definition_placement_types_ui.py"], capture_output=True) ok &= run("Creature visual graph synthetic ownership observations", [sys.executable, "tools/audit_creature_visual_graph.py"], capture_output=True) ok &= run("Creature visual navigation numeric names and malformed-reference UI refusal", diff --git a/tools/test_definition_placement_types_ui.py b/tools/test_definition_placement_types_ui.py new file mode 100644 index 00000000..0b58d656 --- /dev/null +++ b/tools/test_definition_placement_types_ui.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Place definitions by their compiled class, independently of their names.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import struct +import subprocess +import tempfile +import zlib +from test_fmp_links import crc0, read_bank +from test_setup_restore_ui import level +from test_texture_lookup import texture_bank +from test_definition_edit_ui import click + +ROWS = [('CREATURE', 'GUI_ROOT', 2), ('OBJECT', 'CREATURE_DECOY', 4), + ('BUILDING', 'PLAIN_HOUSE', 4), ('CREATURE', 'CREATURE_NORMAL', 3), + ('OBJECT', 'OBJECT_NORMAL', 4), ('BUILDING', 'BUILDING_NORMAL', 4)] + + +def fixture(root, schema): + folder = root/'data/CompiledDefs' + folder.mkdir(parents=True) + pool, refs, table, offsets, payloads = bytearray(), {}, bytearray(), [], bytearray() + counts = {} + for kind, name, owner in ROWS: + for value in (kind, name): + if value not in refs: + refs[value] = len(pool)+4 + pool += struct.pack(' Date: Fri, 23 Oct 2026 07:50:11 -0600 Subject: [PATCH 57/75] Run packaged placement checks with shipped schema from unrelated cwd --- docs/HANDOFF_CREATURE_GUI_2026-10-23.md | 11 +++++++++++ tools/test_definition_placement_types_ui.py | 6 ++++-- tools/test_package.py | 3 ++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md index 81113623..af535a4d 100644 --- a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md @@ -477,3 +477,14 @@ gates. No unchanged broad/core/import/native gate repeated. This corrects class routing for custom-named definitions; marker fixtures do not prove custom body assembly or live gameplay. Existing VILLAGER-name heuristic and live-link prefix check remain separate, unexercised follow-ups. + + +## Combined checkpoint preparation + +Source correction committed/pushed as 91d284e; CI run39964147797 is in progress. +Next distinct checkpoint destination is dist/development/20261023-creature-placement +(previous immutable archives stay untouched). Owned GUI/forge/forge-tools are +being rebuilt at below-normal priority/-j2. Placement package workflow now passes +an unrelated cwd and the archive's schema explicitly, avoiding an implicit repo +working-directory dependency. Source driver defaults are unchanged. Extracted +runtime acceptance for this combined checkpoint is still pending. diff --git a/tools/test_definition_placement_types_ui.py b/tools/test_definition_placement_types_ui.py index 0b58d656..6b8659ef 100644 --- a/tools/test_definition_placement_types_ui.py +++ b/tools/test_definition_placement_types_ui.py @@ -88,6 +88,8 @@ def main(): parser.add_argument('--gui', type=Path, default=repo/'build/FableForge.exe') parser.add_argument('--forge', type=Path, default=repo/'build/forge.exe') parser.add_argument('--work-root', type=Path, default=repo/'build') + parser.add_argument('--cwd', type=Path, default=repo) + parser.add_argument('--schema', type=Path, default=repo/'docs/re_reference/def_schema.json') parser.add_argument('--prepare-only', action='store_true', help='Write and independently inspect fixtures/scripts without launching any executable') args = parser.parse_args() driver_hash = hashlib.sha256(Path(__file__).read_bytes()).hexdigest() @@ -95,7 +97,7 @@ def main(): args.work_root.mkdir(parents=True, exist_ok=True) work = Path(tempfile.mkdtemp(prefix='placement-types-', dir=args.work_root.resolve())) print('Evidence retained at', work, flush=True) - schema = json.loads((repo/'docs/re_reference/def_schema.json').read_text(encoding='utf-8')) + schema = json.loads(args.schema.resolve(strict=True).read_text(encoding='utf-8')) cases = [] for size, scale in [('1440x900', 1), ('800x600', 1.5)]: case = work/size @@ -121,7 +123,7 @@ def main(): continue gui = args.gui.resolve(strict=True) result = subprocess.run([str(gui), '--install', str(root), '--size', size, '--auto', str(script)], - cwd=repo, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=120, + cwd=args.cwd.resolve(strict=True), capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=120, env=dict(os.environ, FABLEFORGE_AUTOMATION_HIDDEN='1', APPDATA=str(case/'profile'), LOCALAPPDATA=str(case/'local'))) (case/'output.log').write_text(result.stdout+result.stderr, encoding='utf-8') trace = Path(str(script)+'.log').read_text(encoding='utf-8') diff --git a/tools/test_package.py b/tools/test_package.py index c73476fc..faf4581a 100644 --- a/tools/test_package.py +++ b/tools/test_package.py @@ -267,7 +267,8 @@ def captured(value): run('creature_import_ui', [sys.executable, repo / 'tools/test_creature_import_ui.py', '--gui', package / 'FableForge.exe', '--forge', package / 'forge.exe', '--work-root', work]) run('definition_placement_types', [sys.executable, repo/'tools/test_definition_placement_types_ui.py', - '--gui', package/'FableForge.exe', '--forge', package/'forge.exe', '--work-root', work]) + '--gui', package/'FableForge.exe', '--forge', package/'forge.exe', '--work-root', work, + '--cwd', cwd, '--schema', package/'docs/re_reference/def_schema.json']) for window_size in ('1440x900', '800x600'): run('creature_clone_gui_retail_' + window_size, [sys.executable, repo / 'tools/test_creature_clone_retail.py', '--install', args.root.resolve(), '--gui', package / 'FableForge.exe', '--forge', package / 'forge.exe', From 91985fbea65092b4c22f1823154bd0db523c5235 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:57:29 -0600 Subject: [PATCH 58/75] Record combined Creature checkpoint extracted acceptance --- docs/HANDOFF_CREATURE_GUI_2026-10-23.md | 51 +++++++++++++++++++++++++ docs/HANDOFF_NAV.md | 2 +- docs/ROADMAP_1.0.md | 2 +- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md index af535a4d..71c37b0b 100644 --- a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md @@ -488,3 +488,54 @@ being rebuilt at below-normal priority/-j2. Placement package workflow now passe an unrelated cwd and the archive's schema explicitly, avoiding an implicit repo working-directory dependency. Source driver defaults are unchanged. Extracted runtime acceptance for this combined checkpoint is still pending. + + +Combined checkpoint prepared from clean77eddec after GUI/forge/forge-tools build: +`dist/development/20261023-creature-placement/FableForge-0.18.1-dev-win64.zip`, +SHA256 `c49bf3555d9f423bcf21714110e61ae3d889b6da3d5696ca333ed546068735e8`. +Offline extraction gate passes118 identities,142 documentation links and the +retail eight-map smoke. Receipt +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/package-smoke-o1x_sutd/result.json`. +One workflow run,148 not run,two unrelated missing-corpus skips. New archive is +immutable; prior checkpoints remain unchanged. Extracted GUI/placement/import +workflows remain pending: immediate guard found an unrelated native definition +block-cache test, no lease claimed or GUI launched. Readiness queued as +`01a16907-ef2f-7f31-8013-866ebf1671b5`. Ready supervisor: +`build/run_creature_placement_package_window_20261023.py`. + + +## Combined immutable checkpoint acceptance + +The new creature-placement archive passes all four selected extracted workflows: +retail eight-map smoke, GUI smoke,50 custom-import cases and16 marker placements +at both sizes. Pack diffuse SRVs/metadata, Refresh/source switching, stale-source +refusals, exact preservation/Restore and shipped-schema/unrelated-cwd placement +all pass.118 file identities and142 local documentation links pass.145 workflows +were not run; two unrelated missing controller/Aeon and F2 corpora remain explicit +skips. No full-suite, clean-machine, whole-creature geometry or gameplay claim. + +Offline receipt: `package-smoke-o1x_sutd/result.json` under the 20261023 C evidence +root. GUI receipt: `cp-iynr178j/package-smoke-wai5ad7z/result.json`; inner import +`creature-import-ui-tdbm9g1m/result.json` and placement +`placement-types-b0seid45/result.json`. Both pin the same stripped GUI SHA256 +`13e2f3949b331837e23d72c93e8b3b92c82b7f08793d5df065142f869454b542`. +Combined receipt: `build/creature-placement-checkpoint-20261023.json`. +Archive SHA256 remains +`c49bf3555d9f423bcf21714110e61ae3d889b6da3d5696ca333ed546068735e8`, +packaging checkout clean77eddec912a631be491f8c14b0f46ec94e06058d. Prior two +checkpoint ZIP hashes independently rechecked unchanged. No archive overwritten. + +Inspected full-size pack model screenshot: textured triangle with Materials +`Diffuse 8: GUI_TEXTURE (4 x 4)`. Compact placement screenshot shows the enabled +`Place GUI_ROOT` button and the saved marker fixture; stacked fixture warnings +obscure much of the tiny viewport, so this is not a geometry presentation claim. +All owned GUIs closed; lease released with +`cp-iynr178j/released-runtime-lease.json`. Native availability queued as +`01a1690b-4ee9-72b0-9273-b90dbae6d55f`. Later peer claims remain independent and +must not be removed. No immediate further GUI launch planned. + +Windows CI39964147797 for91d284e remains in progress, verified at the Windows +Build step (Unit tests/WARP/package still pending). Do not infer success from +local package acceptance or earlier green Materials CI. Next: finish draft-PR +checkpoint documentation, then investigate the remaining creature presentation +and placement-profile gaps with existing native evidence before new algorithms. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 5b6acaf2..1af8b256 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,4 +1,4 @@ -> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Reviewed private model/texture imports pass all 50 extracted GUI cases. The new immutable pack-texture checkpoint passes 118 identities, 142 documentation links and three focused workflows. A newer Materials identity correction passes ten focused source cases; the immutable ZIP retains its documented panel limitation. Gameplay remains unverified; RC2 unchanged. Custom-name placement now passes sixteen marker placements at both sizes using compiled classes and correct owners. +> Current development checkpoint, 2026-10-23: [Creature authoring continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). The combined creature-placement archive passes 118 identities, 142 documentation links and four extracted workflows, including 50 import cases and 16 marker placements at both sizes. Pack material identity and custom-name class/owner routing are included. 145 workflows not run; two unrelated corpus skips. Whole-creature presentation and gameplay remain unverified; RC2 unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index f46f2001..17ca05fb 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Resumed at user request, 2026-10-23: [Creature custom-import GUI continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). Reviewed private model/texture imports pass all 50 extracted GUI cases. The new immutable pack-texture checkpoint passes 118 identities, 142 documentation links and three focused workflows. A newer Materials identity correction passes ten focused source cases; the immutable ZIP retains its documented panel limitation. Gameplay remains unverified; RC2 unchanged. Custom-name placement now passes sixteen marker placements at both sizes using compiled classes and correct owners. +> Current development checkpoint, 2026-10-23: [Creature authoring continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). The combined creature-placement archive passes 118 identities, 142 documentation links and four extracted workflows, including 50 import cases and 16 marker placements at both sizes. Pack material identity and custom-name class/owner routing are included. 145 workflows not run; two unrelated corpus skips. Whole-creature presentation and gameplay remain unverified; RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there From bd25f071bc34fc2e6a21ed77ffafd8b72d273564 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 08:18:36 -0600 Subject: [PATCH 59/75] Recover runtime component rule for Creature village placement profiles --- docs/HANDOFF_CREATURE_GUI_2026-10-23.md | 15 ++- docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md | 95 ++++++++++++++ docs/HANDOFF_NAV.md | 2 + tools/audit_creature_placement_profiles.py | 130 +++++++++++++++++++ 4 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md create mode 100644 tools/audit_creature_placement_profiles.py diff --git a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md index 71c37b0b..8f32fade 100644 --- a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md @@ -534,8 +534,21 @@ All owned GUIs closed; lease released with `01a1690b-4ee9-72b0-9273-b90dbae6d55f`. Later peer claims remain independent and must not be removed. No immediate further GUI launch planned. -Windows CI39964147797 for91d284e remains in progress, verified at the Windows +Windows CI 39964147797 for 91d284e remains in progress, verified at the Windows Build step (Unit tests/WARP/package still pending). Do not infer success from local package acceptance or earlier green Materials CI. Next: finish draft-PR checkpoint documentation, then investigate the remaining creature presentation and placement-profile gaps with existing native evidence before new algorithms. + + +## Next profile correction recovered + +Placement CI 39964147797 completed success. Read-only native investigation now +recovers a concrete VillageMember profile rule from runtime Components rather +than definition spelling; [profile evidence and next steps](HANDOFF_CREATURE_PROFILES_2026-10-23.md). +All 517 retail CREATURE records decode; 344 list CTCVillageMember, 173 do not. +The name heuristic differs on 215 definitions. 32 sampled retail placements match +runtime components exactly. Eight targeted debug function byte ranges match the +original PE; direct GetByteSize disassembly confirms nine-byte records and native +TransferBinaryIn verifies0x01ab6cce. No production writer changed, no GPU launched, +no native install bytes changed. The current immutable checkpoint remains exact. diff --git a/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md new file mode 100644 index 00000000..2c4fbf9b --- /dev/null +++ b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md @@ -0,0 +1,95 @@ +# Creature placement profiles, 2026-10-23 + +The current automatic VillageMember scaffold in `Document::placeCreature` uses +`definition.find("VILLAGER")`. That does not describe the runtime component list. +The next correction should use the selected compiled definition's `Components` +value, preserving that list when a donor is cloned under a custom name. +No production placement change is included in this investigation. + +## Native construction evidence + +Debug executable: `D:/Documents/FableTLC/debug_build/FableWin.exe`,55,571,968 bytes, +SHA256 `a9d6d0977d9d7fc8242da4292ae7ed92e926ca80364a95c003d9b9f689e41845`. +PDB-name export: `D:/Documents/FableTLC/ghidra_out/fablewin_pdb_names.tsv`, SHA256 +`d87957089ac064c8d43632114660795223965fd01675606a12163b9d8f8860ef`. +These are debug addresses, not retail addresses. + +Targeted `analyzeHeadless ... -process FableWin.exe -readOnly -noanalysis` +queries used existing `DumpFuncBytesAndC.java`. They did not import, analyze, +edit types, save a program or reclaim a project lock. Eight complete function +byte ranges independently match the original PE. Seven decompile; creature +ConstructFromParams does not, so its owner branch is checked by direct disassembly. +Evidence root: `build/creature-placement-native-20261023`. + +| Debug address | Function | Observation | +|---|---|---| +| 02994490 | CEditControlCentre::PaintInputPlaceThingAt | Dispatches a type-value switch to distinct creature/building/object constructors; supports the already accepted class-routing correction. | +| 01d6db30 | CThingAICreature::ConstructFromParams | At01d6db89, owner5 selects the definition's DefaultOwner byte at+0x12c. No pseudocode acceptance claimed. | +| 01b9d1b9 | CTCVillageMember constructor | Clears village pointer/UID state. | +| 01cf0320 | CThing::CreateSerialiseCreatableTCs | Parses explicit CreateTC entries and calls AddTC; distinct from default definition components. | +| 025758e0 | CTCVillageMember::OnCreate | Looks up the parent definition, then attempts GetSubDef; not a name-substring test. | +| 02576e70 | CTCVillageMember::OnSerialise | Transfers VillageUID, defaults/clears to zero, and remaps nonzero UIDs. | +| 01cf3810 | CThing::CreateFromDef | Iterates GetNoComponents/GetComponent, resolves each name and calls AddTC with the entry's byte+8 flag. It does not skip component creation based on the definition's spelling. | +| 028b15c0 | CThingComponentSet::TransferBinaryIn | Reads count, raw entry-array bytes, then verifies trailing 28011726 (0x01ab6cce). | + +Direct original-byte disassembly of GetByteSize at 028b21a0 ends with +`imul eax,eax,9` at 028b21af, confirming the record stride independently of the +retail reader. Each record is `{name offset:u32, parameter:u32, flag:u8}`. +This layout already exists in `src/cli/nav.cpp` and the core FMP link resolver; +reuse the core abstractions when implementing profile lookup. + +## Independent retail observations + +`tools/audit_creature_placement_profiles.py` reads the original names/game banks +and selected loose TNGs without launching an executable or writing the install. +Its malformed fixtures refuse truncated/miscounted lists, unknown name boundaries, +and a wrong verification word. The CDefs prefix/subdefinition table is validated +separately from the tagged runtime Components value. + +Final receipt: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/creature-placement-profile-vbu6_x0p/result.json`. +All 517 CREATURE records decode; no refused records.344 list CTCVillageMember, +173 do not. Every observed Components value has the four-byte 0x01ab6cce trailer. +The name heuristic disagrees on 215 roots:214 have the runtime CTC without VILLAGER +in the name; CREATURE_RIVAL_HERO_BRIAR_ROSE_VILLAGER has the name but no such CTC. +Traders, hero family and guards are concrete non-prefixed examples; the chicken +has no VillageMember runtime CTC. + +Thirty-two actual AICreature placements in StartOakValeWest/GuildExterior agree +with the runtime-component rule, including their VillageMember TNG block +presence. No sampled mismatch and no missing map. Original bank/map hashes +remain exact. This is a bounded sample, not proof about every saved TNG: explicit +CreateTC additions are supported by the engine and existing TNG edits must survive. + +An initial hypothesis inspected CVillageMemberDef CDefs rows. That finds no direct +root rows, despite 12 CVillageMemberDef records existing. It is the wrong structure +for runtime-component presence. The corrected audit checks pool-referenced CTC +names in Components; do not implement the initial hypothesis. A broad scratch +file search was stopped after encountering protected unrelated fixture folders; +retail maps were read at their known install paths. No peer process was stopped. + +## Implementation and acceptance next + +1. Expose a read-only optional component-presence lookup through Context, backed + by a validated Components value and original names-pool boundaries. Absent + component, absent definition and malformed/unsupported metadata must remain + distinguishable; no fallback to definition-name guesses. +2. Pass an explicit village-membership profile to the existing creature writer; + keep automatic ownership and the world-space InitialPos behavior unchanged. + Reuse the existing TNG serializer and recovery paths. Preserve existing TNG + blocks, order, unrelated properties and undo behavior. +3. Test a custom-named creature with CTCVillageMember and a VILLAGER-named creature + without it, plus malformed count/name/sentinel prewrite refusal. Independently + inspect TNG blocks, undo/redo, exact bank/asset preservation and Restore. +4. Exercise both window sizes against a frozen old baseline and candidate under + a fresh shared runtime lease. Do not repeat accepted import/native graph gates + without a relevant change. Check every placeCreature caller, including CLI/ + direct-document tests, before changing its API. + +The current immutable creature-placement checkpoint is unchanged. It includes +accepted material identity and compiled-class routing but retains this name-based +membership heuristic. Whole-creature assembly, animation and gameplay remain +separate unverified work. No new GPU/game/GUI window was used for this investigation. + +Placement Windows CI 39964147797 for 91d284e completed success, verified directly; +that CI predates the new read-only audit and any future profile correction. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 1af8b256..318da075 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,5 +1,7 @@ > Current development checkpoint, 2026-10-23: [Creature authoring continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). The combined creature-placement archive passes 118 identities, 142 documentation links and four extracted workflows, including 50 import cases and 16 marker placements at both sizes. Pack material identity and custom-name class/owner routing are included. 145 workflows not run; two unrelated corpus skips. Whole-creature presentation and gameplay remain unverified; RC2 unchanged. +Read-only follow-up: [Creature placement-profile evidence](HANDOFF_CREATURE_PROFILES_2026-10-23.md) recovers runtime component membership for the next correction. + # Navigation lane, 2026-09-27 Checkout: `D:/Code/FableForge-nav`, branch `feat/nav-thing-lines`, starting commit diff --git a/tools/audit_creature_placement_profiles.py b/tools/audit_creature_placement_profiles.py new file mode 100644 index 00000000..c1c1aed7 --- /dev/null +++ b/tools/audit_creature_placement_profiles.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Read-only audit of creature runtime Components and retail placement profiles.""" +import argparse +from collections import Counter +import hashlib +import json +from pathlib import Path +import re +import struct +import tempfile +from audit_creature_visual_graph import listing +from test_fmp_links import crc0, read_bank + + +def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def runtime_components(record, names): + # CDefs prefix and runtime Components are distinct serialized structures. + rows = listing(record) + at = 5+12*len(rows)+4 + data = record['data'] + next_tag = struct.pack('end: + raise ValueError('missing Components value/next field boundary') + count, = struct.unpack_from('4096 or count>(end-at-4)//9: + raise ValueError('invalid runtime component count') + tail = end-at-4-count*9 + if tail != 4: + raise ValueError(f'unsupported opaque runtime component tail: {tail}') + if struct.unpack_from(' Date: Fri, 23 Oct 2026 08:38:13 -0600 Subject: [PATCH 60/75] Derive Creature village membership from verified runtime components --- docs/AUTOMATION.md | 11 ++- docs/EDITOR.md | 2 +- docs/HANDOFF_CREATURE_GUI_2026-10-23.md | 9 +++ docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md | 83 ++++++++++++++++++++ docs/HANDOFF_NAV.md | 2 +- gui/automation.cpp | 5 +- gui/editor.cpp | 11 ++- libs/forgecore/include/forge/defdecode.hpp | 5 ++ libs/forgecore/src/defdecode.cpp | 11 +++ src/leveledit.cpp | 6 +- src/leveledit.hpp | 4 +- src/terrainexport.cpp | 29 +++++++ src/terrainexport.hpp | 4 + tests/test_defdecode.cpp | 24 ++++++ tests/test_export.cpp | 28 ++++++- tools/check_all.py | 2 + tools/test_creature_profile_refusal_ui.py | 58 ++++++++++++++ tools/test_definition_placement_types_ui.py | 20 ++++- tools/test_package.py | 3 + 19 files changed, 299 insertions(+), 18 deletions(-) create mode 100644 tools/test_creature_profile_refusal_ui.py diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md index 6b421ffc..4328feef 100644 --- a/docs/AUTOMATION.md +++ b/docs/AUTOMATION.md @@ -501,7 +501,8 @@ Settings persistence is disabled under `--auto` so runs are deterministic. | `set_entrance` | the map's region entrance (FinalAlbion.gtg, this map's WLD slot) at the view centre, facing the camera (state `entrance` = `x,y,h` or `-`; widget `btn_set_entrance`) | | `preset_place ` / `preset_save ` | place a preset (shipped `presets/` or the user folder) at the view centre as a selected group; save the selection as a user preset (state `presets` = how many are listed) | | `select_toggle ` / `select_added` / `copy` / `paste` | Ctrl+click a thing into/out of the selection; select the first thing the Changes list reports as added; copy the selection as a fragment; paste it at the view centre (state `selection_count`) | -| `place [scriptname]` | place a new thing at the camera focus, on the ground (`CREATURE_*` as the retail AICreature block) | +| `place [scriptname]` | place a new thing at the camera focus, on the ground, using its compiled class and validated runtime component profile | +| `place_refused [scriptname]` | require placement to refuse; malformed Creature profiles must leave the document and recovery files unchanged | | `set place_def ` / `place_at ` | choose the palette definition and place it at viewport-relative pointed terrain coordinates; Shift+click tests the actual input path | | `snapshot_frame` / `assert_selected_moved_xy` / `assert_selected_frame_same` / `assert_ground_offset_same` | capture a selected frame, then check XY movement, exact frame restoration, or preserved height above terrain | | `ground_place_carry.txt` | synthetic Shift+click placement, plain and Ctrl+Shift ground drag, Ctrl+D cursor carry, grouped undo, fixed height, lock refusal and Escape cancellation | @@ -1152,7 +1153,13 @@ reader checks class, owner and creature InitialPos. Definitions, texture and LEV stay exact; Restore retains only the byte-exact first-original TNG backup. The standalone driver's `--prepare-only` writes and verifies fixtures/scripts without launching an executable; that receipt explicitly does not claim a pass. -These marker fixtures do not test whole-creature geometry or gameplay. +These marker fixtures do not test whole-creature geometry or gameplay. They also +require village membership from runtime Components, independent of the name. +`creature_placement_profile_refusals` adds eight malformed-count/name/sentinel/ +missing-field cases at both sizes, requiring an unchanged document and complete +fixture inventory. The read-only C++ test mode `--creature-component-profiles +` reports Context membership for comparison with the independent audit. + `creature_clone_gui_retail_1440x900` and `_800x600` independently compare all prior retail records, verified self-only deltas, paired recovery and explicit diff --git a/docs/EDITOR.md b/docs/EDITOR.md index 127f204f..730e4b05 100644 --- a/docs/EDITOR.md +++ b/docs/EDITOR.md @@ -1448,7 +1448,7 @@ The same guarded shared-component mode is available through `forge-tools defs cl The separate **Clone private creature visuals** action appends a coordinated root, CCreatureDef and CAppearanceDef, with explicit unique bounded ASCII names. Selected visual rows become owned by the new root; other gameplay components retain their original data and ownership. Without a custom model, models, textures and animation assets remain shared. The paired publisher independently recomputes the three-record plan before writing names.bin and game.bin together, retaining recovery records. Unsupported/malformed known layouts, missing visual components, unresolved references and new name-CRC ambiguities refuse before publication. Root, body and appearance names are separate fields in the scrolling card. Cancel and close-review retain the existing draft protection. Use owned outputs first; no gameplay validation is claimed. -Enable **Import a custom body model** in that draft to choose one explicit body part/alternative and a skinned `.glb` or `.gltf` matching the donor skeleton. Supply a new `MESH_` name using uppercase letters, digits and underscores, then choose an existing texture ID or a new named image. New images support DXT1, DXT3 and ARGB8888; models can be compressed or stored. **Review custom import** checks the model before offering Create. Changing any input or destination requires a fresh review. Create rechecks the source banks, model and external buffer bytes, image, destination files and pack manifest before publishing the coordinated definitions, mesh and optional texture. Controls and cancellation wait for background review/publication. Pack imports refresh Models and preview their diffuse textures using a separate pack context; the loaded world's assets stay unchanged. Materials shows verified names and dimensions from that preview source as read-only entries; normal install textures retain their navigation links. Restore removes created outputs or recovers in-place originals. Other body alternatives and animations remain shared; this does not replace a complete creature or establish gameplay compatibility. The Actors palette and placement use the stored definition class, so a creature name does not need a `CREATURE_` prefix. Automatic ownership follows the creature definition; an object with a similar name stays an object. +Enable **Import a custom body model** in that draft to choose one explicit body part/alternative and a skinned `.glb` or `.gltf` matching the donor skeleton. Supply a new `MESH_` name using uppercase letters, digits and underscores, then choose an existing texture ID or a new named image. New images support DXT1, DXT3 and ARGB8888; models can be compressed or stored. **Review custom import** checks the model before offering Create. Changing any input or destination requires a fresh review. Create rechecks the source banks, model and external buffer bytes, image, destination files and pack manifest before publishing the coordinated definitions, mesh and optional texture. Controls and cancellation wait for background review/publication. Pack imports refresh Models and preview their diffuse textures using a separate pack context; the loaded world's assets stay unchanged. Materials shows verified names and dimensions from that preview source as read-only entries; normal install textures retain their navigation links. Restore removes created outputs or recovers in-place originals. Other body alternatives and animations remain shared; this does not replace a complete creature or establish gameplay compatibility. The Actors palette and placement use the stored definition class, so a creature name does not need a `CREATURE_` prefix. Automatic ownership follows the creature definition; an object with a similar name stays an object. Village membership follows its runtime `CTCVillageMember` component, so renaming a donor does not change that placement profile. Unsupported or malformed component metadata refuses placement before changing the level. The same private mode is available through `forge-tools defs clone-creature-private --schema --out [--names ] [--json]`. diff --git a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md index 8f32fade..f0c1d69b 100644 --- a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md @@ -552,3 +552,12 @@ runtime components exactly. Eight targeted debug function byte ranges match the original PE; direct GetByteSize disassembly confirms nine-byte records and native TransferBinaryIn verifies0x01ab6cce. No production writer changed, no GPU launched, no native install bytes changed. The current immutable checkpoint remains exact. + + +Runtime village-membership source correction is now accepted; see +[profile handoff](HANDOFF_CREATURE_PROFILES_2026-10-23.md). Two affected unit +executables,517 native profile comparisons,16 placements/eight malformed-profile +refusals and the additional unknown-block preservation/undo assertion pass. +The immutable checkpoint is not replaced; it still uses the old membership +name heuristic. Latest own runtime lease released with pu-2qfi4vr0 receipt and +Native availability queued01a16931-32c3-7081-9df8-5c8563fb6cfa. diff --git a/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md index 2c4fbf9b..36a0b80e 100644 --- a/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md @@ -93,3 +93,86 @@ separate unverified work. No new GPU/game/GUI window was used for this investiga Placement Windows CI 39964147797 for 91d284e completed success, verified directly; that CI predates the new read-only audit and any future profile correction. + + +## Source correction prepared + +Core defdecode now validates the canonical counted nine-byte Components list and +verification word before returning original name offsets. Context resolves every +reference against reconstructed original-pool boundaries (originalNames stores +CRCs plus strings, not offsets), returns optional presence with a diagnostic, +and does not guess from names. App preflights this before any document mutation. +Document::placeCreature now requires an explicit membership flag; the sole GUI +caller and sole direct unit-test caller were updated. InitialPos/owners and the +existing serializers/recovery paths are unchanged. + +The existing placement fixture now includes a valid runtime list: GUI_ROOT has +CTCVillageMember; CREATURE_VILLAGER_DECOY does not. The independent TNG oracle +checks membership as well as class, owner, InitialPos and Restore. A new refusal +driver covers bad count, non-boundary name reference, wrong verification word +and missing Components at both sizes, requiring no document/file/recovery change. +Preparation-only receipts: placement-types-m9bie7kw and profile-refusal-ivlbjghy +under the 20261023 C evidence root. Core decoder malformed-list tests and explicit +Document profile/undo tests were added. Build is in progress; runtime acceptance +is not claimed yet. A targeted C++ Context audit mode will compare all517 native +roots with the independent Python observation, without writing native files. + + +## First source validation window + +Canonical receipt: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/rp-0izfcq1z/result.json`. +Both affected executables pass: fableforge_defdecode_tests and fableforge_tests. +The C++ Context audit agrees with the independent Python audit on all517 native +roots, including344 VillageMember profiles. Original bank/map hashes remain exact. +Frozen candidate GUI SHA256: +`89a5aa62a1e3be33bc8ead71ce44dc610d403c1fed5c1e2bea7390bddad7a23f`. + +Old baseline's GUI script passes its class/owner/undo actions but its independently +read TNG lacks membership on GUI_ROOT and adds it to CREATURE_VILLAGER_DECOY. +Baseline failure is exactly the new membership oracle; original TNG explicitly +Restored. Receipt `placement-types-eic8nmrm` within that window. +Candidate passes16 placements at both sizes, with exact membership, class, owner, +InitialPos, undo/redo and recovery: `placement-types-04pkjjce/result.json`. +All eight malformed-profile cases pass with unchanged document, fixture and +recovery inventory: `profile-refusal-5eridtq2/result.json`. +Own lease released; Native availability queued as +`01a1692a-b927-7740-9d98-34925f8df5ae`; receipt `rp-0izfcq1z/released-runtime-lease.json`. + +An additional focused unit assertion now seeds an unrelated unknown TNG block, +requires it byte-exact after both profile choices, and requires undo to recover +the whole original source. Its rebuilt test executable has not run yet: shared +lease is owned by Texture Archive4GB, verified live PID13444/start1792765885.4409208, +and an actual Fable process is active. Both are left untouched. No new GUI gate +is needed for this test-only addition; run only the updated unit once a fresh +lease/process guard permits. CLI neighbors are rebuilding offline at -j2. +No production GUI/helper code changed after the successful comparison. + + +## Source correction accepted + +Texture owner exited and its lease/game closed normally. Fresh process guard +and exclusive RuntimeLease allowed only the updated preservation unit to run; +it passed. Receipt: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/pu-2qfi4vr0/result.json`. +It proves the unknown existing TNG block is byte-exact for both profile choices +and undo restores the entire original text. No production code or GUI changed +since the prior successful comparison; no GUI/native audit was repeated. +All517 C++ audit rows have unique names and match all517 independent rows. +Core/GUI source hashes still match the frozen candidate; only the added unit +assertion differs from that snapshot's test source. Updated unit binary SHA is +pinned in its receipt. CLI neighbor builds pass. + +Unit lease released, Native availability queued as +`01a16931-32c3-7081-9df8-5c8563fb6cfa`; receipt `pu-2qfi4vr0/released-runtime-lease.json`. +No owned runtime remains. New refusal workflow is registered in source/package +gates; source docs describe metadata-based membership and place_refused. + +Acceptance now covers two affected unit executables, all517 read-only native +profiles, the old membership negative control,16 positive placements,8 malformed +profile refusals, exact recovery, and existing unknown-block preservation/undo. +No source bank/asset writer changed. The retained immutable ZIP is unchanged and +still has the old name heuristic. Next: record/push source correction, follow CI, +and validate the new profile/refusal workflows in a distinct extracted checkpoint +when batching the next authoring improvements. Whole-creature rendering remains +unverified; do not repeat unrelated accepted import/native graph gates. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index 318da075..aab679b8 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,6 +1,6 @@ > Current development checkpoint, 2026-10-23: [Creature authoring continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). The combined creature-placement archive passes 118 identities, 142 documentation links and four extracted workflows, including 50 import cases and 16 marker placements at both sizes. Pack material identity and custom-name class/owner routing are included. 145 workflows not run; two unrelated corpus skips. Whole-creature presentation and gameplay remain unverified; RC2 unchanged. -Read-only follow-up: [Creature placement-profile evidence](HANDOFF_CREATURE_PROFILES_2026-10-23.md) recovers runtime component membership for the next correction. +Source follow-up: [Creature placement profiles](HANDOFF_CREATURE_PROFILES_2026-10-23.md) now use runtime components; units, 517 native profiles, 16 placements and eight malformed-profile refusals pass. The retained ZIP predates this correction. # Navigation lane, 2026-09-27 diff --git a/gui/automation.cpp b/gui/automation.cpp index 8a58461f..12fd3236 100644 --- a/gui/automation.cpp +++ b/gui/automation.cpp @@ -1401,9 +1401,10 @@ bool Automation::tick(App& app) { else if (cmd == "edit_tab") { app.setEditTab(std::atoi(rest.c_str())); note("ok " + line); ++pc_; } // 0 Objects 1 Terrain 2 Actors 3 Level else if (cmd == "dismiss_rule") { app.dismissRule(rest); note("ok " + line); ++pc_; } // what the notice's "Got it" does (the notice may sit below the panel fold) else if (cmd == "redo") { app.editRedo(); note("ok " + line); ++pc_; } - else if (cmd == "place") { // place [scriptname] + else if (cmd == "place" || cmd == "place_refused") { // [scriptname] std::istringstream rs(rest); std::string def, sn; rs >> def >> sn; - if (!app.placeDefinition(def, sn)) fail("place failed: " + rest); else note("ok " + line); ++pc_; + const bool okay=app.placeDefinition(def,sn); + if(okay!=(cmd=="place")) fail("unexpected placement result: "+rest); else note("ok "+line);++pc_; } else if (cmd == "compact_stb") { if (!app.compactBank()) fail("compact_stb failed"); else note("ok " + line); ++pc_; } else if (cmd == "compact_stb_refused") { if (app.compactBank()) fail("compact_stb unexpectedly started"); else note("ok " + line); ++pc_; } diff --git a/gui/editor.cpp b/gui/editor.cpp index ec635b5d..18deedf9 100644 --- a/gui/editor.cpp +++ b/gui/editor.cpp @@ -2544,6 +2544,14 @@ bool App::placeDefinition(const std::string& def, const std::string& scriptName) bool App::placeDefinitionAt(const std::string& def, const float position[3], const std::string& scriptName) { if (!documentLoaded()) { pushLog("editor: no level document", 1); return false; } if (!std::isfinite(position[0]) || !std::isfinite(position[1]) || !std::isfinite(position[2])) return false; + const bool creature=ctx_.definitionType(def)=="CREATURE"; + bool villageMember=false; + if(creature) { + std::string error; + const auto profile=ctx_.definitionHasRuntimeComponent(def,"CTCVillageMember",error); + if(!profile) {pushLog("editor: cannot read creature placement profile: "+error,2);return false;} + villageMember=*profile; + } uint32_t modelId = 0; const int code = ctx_.graphicModelId(def, modelId); if (code == 0) { pushLog("editor: " + def + " is not in game.bin", 2); return false; } @@ -2570,10 +2578,9 @@ bool App::placeDefinitionAt(const std::string& def, const float position[3], con if (fl > 1e-6f) p.forward = {fx / fl, fy / fl, 0.0f}; } try { - const bool creature = ctx_.definitionType(def) == "CREATURE"; const float pos[3] = {p.position.x, p.position.y, p.position.z}; const float fwd[2] = {p.forward.x, p.forward.y}; - const size_t n = creature ? doc_.placeCreature(pos, fwd, def, scriptName, p.player) : doc_.place(p); + const size_t n = creature ? doc_.placeCreature(pos, fwd, def, villageMember, scriptName, p.player) : doc_.place(p); selectedUid_ = doc_.uidOf(n); selectedThing_ = int(n); renderer_.selectedThing = selectedThing_; diff --git a/libs/forgecore/include/forge/defdecode.hpp b/libs/forgecore/include/forge/defdecode.hpp index 0b7ac792..7f42390d 100644 --- a/libs/forgecore/include/forge/defdecode.hpp +++ b/libs/forgecore/include/forge/defdecode.hpp @@ -62,6 +62,11 @@ struct Decoded { // Decode one entry against its def-type schema. `def` must match entry.definition. Decoded decode(const std::vector& payload, const defschema::DefType& def); +// Validated retail CThingComponentSet: count, nine-byte entries, verification +// word 0x01ab6cce. Returns the original names-pool offsets; throws on unsupported +// or malformed fields. Resolving each offset remains the owning bank's job. +std::vector runtimeComponentNameOffsets(const DecodedField& field); + // Rebuild a payload from a decode: prefix bytes, then [4-byte tag][value] per // field. encode(decode(p)) == p for any clean() decode. std::vector encode(const Decoded& d); diff --git a/libs/forgecore/src/defdecode.cpp b/libs/forgecore/src/defdecode.cpp index fd045738..5e5e781e 100644 --- a/libs/forgecore/src/defdecode.cpp +++ b/libs/forgecore/src/defdecode.cpp @@ -162,6 +162,17 @@ Decoded decode(const bin::Entry& entry, const defschema::Schema& schema) { return decode(entry.data, *def); } +std::vector runtimeComponentNameOffsets(const DecodedField& field) { + auto fail=[] { throw std::invalid_argument("invalid or unsupported runtime Components list"); }; + if(field.name!="Components" || !field.tagOk || field.tag!=fieldTag("Components") || field.value.size()<8) fail(); + const auto count=rd32(field.value,0); + if(count>4096 || count>(field.value.size()-8)/9 || field.value.size()!=8+size_t(count)*9) fail(); + if(rd32(field.value,4+size_t(count)*9)!=0x01ab6cce) fail(); + std::vector result;result.reserve(count); + for(size_t i=0;i>& versions, diff --git a/src/leveledit.cpp b/src/leveledit.cpp index 28a0d78b..4bfdcf8a 100644 --- a/src/leveledit.cpp +++ b/src/leveledit.cpp @@ -3366,7 +3366,7 @@ size_t Document::placeCreatureGenerator(const float pos[3], const std::vector Context::definitionType(const std::string& name) cons return entry ? std::optional(entry->definition) : std::nullopt; } +std::optional Context::definitionHasRuntimeComponent(const std::string& name, + const std::string& component, std::string& error) const { + error.clear(); + if(!impl_ || !impl_->defs || !impl_->schema) {error="definition metadata is not loaded";return std::nullopt;} + std::lock_guard lock(impl_->defMutex); + const auto* entry=impl_->defs->find(name); + if(!entry) {error="definition not found: "+name;return std::nullopt;} + try { + const auto decoded=forge::defdecode::decode(*entry,*impl_->schema); + const auto field=std::find_if(decoded.fields.begin(),decoded.fields.end(),[](const auto& f){return f.name=="Components";}); + if(!decoded.clean() || field==decoded.fields.end()) throw std::runtime_error("runtime Components field is unavailable"); + const auto offsets=forge::defdecode::runtimeComponentNameOffsets(*field); + std::map names; + uint64_t nameOffset=4; + for(const auto& original:impl_->defs->originalNames()) { + if(nameOffset>0xffffffffULL) throw std::runtime_error("original names pool exceeds offset range"); + names.emplace(uint32_t(nameOffset),original.second); + nameOffset+=original.second.size()+5; + } + bool found=false; + for(const auto offset:offsets) { + const auto it=names.find(offset); + if(it==names.end()) throw std::runtime_error("runtime component name is not an original pool boundary"); + found|=it->second==component; + } + return found; + } catch(const std::exception& e) {error=e.what();return std::nullopt;} +} + std::vector Context::groupedDefinitions(const std::vector& types) const { std::vector out; if (!impl_ || !impl_->defs || !impl_->schema) return out; diff --git a/src/terrainexport.hpp b/src/terrainexport.hpp index 18306342..092a08f3 100644 --- a/src/terrainexport.hpp +++ b/src/terrainexport.hpp @@ -216,6 +216,10 @@ class Context { std::optional definitionIndex(const std::string& name) const; // The compiled game.bin class; custom definition names need no type prefix. std::optional definitionType(const std::string& name) const; + // nullopt with a diagnostic for unavailable/malformed metadata; false only + // for a fully validated list that does not contain the named runtime CTC. + std::optional definitionHasRuntimeComponent(const std::string& name, + const std::string& component, std::string& error) const; std::vector groupedDefinitions(const std::vector& types) const; // Defs only (names.bin + game.bin, no textures.big): enough for themeLibrary(). bool loadDefs(const std::filesystem::path& gameRoot, std::string& error); diff --git a/tests/test_defdecode.cpp b/tests/test_defdecode.cpp index 81760a42..25ddaa89 100644 --- a/tests/test_defdecode.cpp +++ b/tests/test_defdecode.cpp @@ -169,5 +169,29 @@ int main() { try { (void)forge::defdecode::mergeFields(base, {{"first",first}}, {}, {{3,"first"}}); } catch (const std::invalid_argument&) { refused = true; } check(refused, "invalid part index refuses rather than selecting another field"); + { + forge::defdecode::DecodedField field; + field.name="Components";field.tag=forge::defdecode::fieldTag(field.name);field.tagOk=true; + auto word=[](std::vector& bytes,uint32_t value) { + for(unsigned i=0;i<4;++i) bytes.push_back(uint8_t(value>>(8*i))); + }; + word(field.value,2);word(field.value,17);word(field.value,99);field.value.push_back(0); + word(field.value,42);word(field.value,0);field.value.push_back(1);word(field.value,0x01ab6cce); + check(forge::defdecode::runtimeComponentNameOffsets(field)==std::vector({17,42}), + "runtime component names use the recovered nine-byte stride"); + for(int kind=0;kind<5;++kind) { + auto bad=field; + if(kind==0) bad.value[0]=3; + if(kind==1) bad.value.pop_back(); + if(kind==2) bad.value.back()=0; + if(kind==3) bad.tagOk=false; + if(kind==4) for(int i=0;i<4;++i) bad.value[i]=0xff; + bool rejected=false; + try{(void)forge::defdecode::runtimeComponentNameOffsets(bad);}catch(const std::invalid_argument&){rejected=true;} + check(rejected,"malformed runtime component list refuses before use"); + } + field.value.clear();word(field.value,0);word(field.value,0x01ab6cce); + check(forge::defdecode::runtimeComponentNameOffsets(field).empty(),"valid empty runtime component list"); + } return failures ? 1 : 0; } diff --git a/tests/test_export.cpp b/tests/test_export.cpp index 56b0dc63..56755f9f 100644 --- a/tests/test_export.cpp +++ b/tests/test_export.cpp @@ -2838,13 +2838,28 @@ void testDayNightSections() { CHECK(!twin.setDayNight(0, 1) && twin.sectionOf(0) == "NULL"); // placed creatures and villages carry the owner the combo resolved const float pos[3] = {1.0f, 2.0f, 3.0f}, fwd[2] = {0.0f, 1.0f}; - const size_t c = doc.placeCreature(pos, fwd, "CREATURE_PRISON_GUARD", "", 2); + const size_t c = doc.placeCreature(pos, fwd, "CREATURE_PRISON_GUARD", false, "", 2); const size_t v = doc.placeVillage(pos, "VILLAGE_TEST"); auto player = [&](size_t i) { for (const auto& r : doc.propertiesOf(i)) if (r.ctc.empty() && r.key == "Player") return r.value; return std::string("none"); }; CHECK(player(c) == "2" && player(v) == "4"); + ed::Document profiled; + const std::string profileSource="Version 2;\r\nXXXSectionStart NULL;\r\n" + "NewThing Object;\r\nUID 88;\r\nDefinitionType \"OBJECT_OPAQUE\";\r\n" + "UnclassifiedProperty 73;\r\nStartCTCUnknown;\r\nValue \"keep\";\r\nEndCTCUnknown;\r\n" + "EndThing;\r\nXXXSectionEnd;\r\n"; + CHECK(profiled.openText("Profiled", profileSource, err)); + const auto untouchedProfileBlock=profiled.file().thingBlockText(0); + profiled.placeCreature(pos,fwd,"CUSTOM_NPC",true,"",2); + CHECK(profiled.text().find("StartCTCVillageMember;")!=std::string::npos); + CHECK(profiled.file().thingBlockText(0)==untouchedProfileBlock); + profiled.undo();CHECK(profiled.text()==profileSource); + profiled.placeCreature(pos,fwd,"CREATURE_VILLAGER_DECOY",false,"",0); + CHECK(profiled.text().find("StartCTCVillageMember;")==std::string::npos); + CHECK(profiled.file().thingBlockText(0)==untouchedProfileBlock); + // Section names use a delimited native string, so numeric names and spaces // stay supported. A new header cannot contain statement/line delimiters. for (const std::string name : {std::string(""), std::string(" "), std::string("\t"), @@ -4189,6 +4204,17 @@ void testGroundNavigation(const fs::path& dir) { } int main(int argc, char** argv) { + if(argc==3 && std::string_view(argv[1])=="--creature-component-profiles") { + albion::terrainexport::Context context;std::string error; + if(!context.loadDefs(argv[2],error)) {std::cerr< Date: Fri, 23 Oct 2026 08:48:22 -0600 Subject: [PATCH 61/75] Record extracted Creature runtime-profile checkpoint acceptance --- docs/HANDOFF_CREATURE_GUI_2026-10-23.md | 11 +++- docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md | 63 ++++++++++++++++++-- docs/HANDOFF_NAV.md | 4 +- docs/ROADMAP_1.0.md | 2 +- 4 files changed, 69 insertions(+), 11 deletions(-) diff --git a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md index f0c1d69b..a5b64796 100644 --- a/docs/HANDOFF_CREATURE_GUI_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_GUI_2026-10-23.md @@ -496,7 +496,7 @@ SHA256 `c49bf3555d9f423bcf21714110e61ae3d889b6da3d5696ca333ed546068735e8`. Offline extraction gate passes118 identities,142 documentation links and the retail eight-map smoke. Receipt `C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/package-smoke-o1x_sutd/result.json`. -One workflow run,148 not run,two unrelated missing-corpus skips. New archive is +One workflow run,148 not run, two unrelated missing-corpus skips. New archive is immutable; prior checkpoints remain unchanged. Extracted GUI/placement/import workflows remain pending: immediate guard found an unrelated native definition block-cache test, no lease claimed or GUI launched. Readiness queued as @@ -560,4 +560,11 @@ executables,517 native profile comparisons,16 placements/eight malformed-profile refusals and the additional unknown-block preservation/undo assertion pass. The immutable checkpoint is not replaced; it still uses the old membership name heuristic. Latest own runtime lease released with pu-2qfi4vr0 receipt and -Native availability queued01a16931-32c3-7081-9df8-5c8563fb6cfa. +Native availability queued 01a16931-32c3-7081-9df8-5c8563fb6cfa. + + +A newer distinct runtime-profile checkpoint now passes its four extracted gates +(retail/GUI, 16 profile placements, eight malformed-profile refusals), 119 identities +and 145 docs links. See [profile acceptance](HANDOFF_CREATURE_PROFILES_2026-10-23.md). +This supersedes the previous source-only membership limitation; older archives +remain unchanged. No unrelated 50-case import rerun or live-game claim. diff --git a/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md index 36a0b80e..664036e4 100644 --- a/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md @@ -114,7 +114,7 @@ and missing Components at both sizes, requiring no document/file/recovery change Preparation-only receipts: placement-types-m9bie7kw and profile-refusal-ivlbjghy under the 20261023 C evidence root. Core decoder malformed-list tests and explicit Document profile/undo tests were added. Build is in progress; runtime acceptance -is not claimed yet. A targeted C++ Context audit mode will compare all517 native +is not claimed yet. A targeted C++ Context audit mode will compare all 517 native roots with the independent Python observation, without writing native files. @@ -123,8 +123,8 @@ roots with the independent Python observation, without writing native files. Canonical receipt: `C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/rp-0izfcq1z/result.json`. Both affected executables pass: fableforge_defdecode_tests and fableforge_tests. -The C++ Context audit agrees with the independent Python audit on all517 native -roots, including344 VillageMember profiles. Original bank/map hashes remain exact. +The C++ Context audit agrees with the independent Python audit on all 517 native +roots, including 344 VillageMember profiles. Original bank/map hashes remain exact. Frozen candidate GUI SHA256: `89a5aa62a1e3be33bc8ead71ce44dc610d403c1fed5c1e2bea7390bddad7a23f`. @@ -158,7 +158,7 @@ it passed. Receipt: It proves the unknown existing TNG block is byte-exact for both profile choices and undo restores the entire original text. No production code or GUI changed since the prior successful comparison; no GUI/native audit was repeated. -All517 C++ audit rows have unique names and match all517 independent rows. +All 517 C++ audit rows have unique names and match all 517 independent rows. Core/GUI source hashes still match the frozen candidate; only the added unit assertion differs from that snapshot's test source. Updated unit binary SHA is pinned in its receipt. CLI neighbor builds pass. @@ -168,11 +168,62 @@ Unit lease released, Native availability queued as No owned runtime remains. New refusal workflow is registered in source/package gates; source docs describe metadata-based membership and place_refused. -Acceptance now covers two affected unit executables, all517 read-only native -profiles, the old membership negative control,16 positive placements,8 malformed +Acceptance now covers two affected unit executables, all 517 read-only native +profiles, the old membership negative control, 16 positive placements, 8 malformed profile refusals, exact recovery, and existing unknown-block preservation/undo. No source bank/asset writer changed. The retained immutable ZIP is unchanged and still has the old name heuristic. Next: record/push source correction, follow CI, and validate the new profile/refusal workflows in a distinct extracted checkpoint when batching the next authoring improvements. Whole-creature rendering remains unverified; do not repeat unrelated accepted import/native graph gates. + + +## Runtime-profile checkpoint prepared + +Distinct archive prepared from clean 37c1ed1 after a no-work three-target shipping +build check: `dist/development/20261023-creature-profiles/FableForge-0.18.1-dev-win64.zip`. +SHA256 `485b985ee33202b0e9df792800817e06952a27023c82231b1579efa8905092d0`. +Offline CRC/duplicate-member/inventory/hash verification passes 119 file identities; +receipt `build/creature-profiles-archive-identity-20261023.json`. No executable ran +for this archive gate. Previous archives remain unchanged. + +Planned extracted checks: gui, retail, definition_placement_types and +creature_placement_profile_refusals. Do not rerun the unrelated unchanged full50 +import gate. Supervisor `build/run_creature_profiles_package_window_20261023.py` +is prepared (240-second bound, inherited per-GUI lease/process guard). +Texture Archive4GB owns the current shared lease, verified live python PID15444, +start1792766194.58565. No new frontend launched; await fresh lease/process guard. +Membership CI 39971393188 is in progress; no success claim yet. + + +## Runtime-profile checkpoint extracted acceptance + +The new immutable archive passes four selected workflows: retail eight-map smoke, +GUI smoke, 16 metadata-profile placements and eight malformed-profile refusals at +both sizes. Placement/refusal drivers use the shipped schema and an unrelated +cwd. 119 exact file identities and 145 local documentation links pass; 146 workflows +not run, two unrelated missing-corpus skips. The unchanged 50-case import gate was +not repeated. No full-suite, clean-machine, whole-creature assembly or gameplay +claim follows from these focused checks. + +Receipt: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/mp-sd7yt8w5/package-smoke-vzpipwdd/result.json`. +Inner placement `placement-types-j39j4ix6/result.json`; refusal +`profile-refusal-pv8ln9bk/result.json`. Both pin stripped GUI SHA256 +`67ee4c1914c8308fbc0c1fd3ec7eb10d3ac2abd0559e2b62f8f7a1c47c270b0e`. +Combined local receipt: `build/creature-profiles-checkpoint-20261023.json`. +ZIP SHA256 remains 485b985ee33202b0e9df792800817e06952a27023c82231b1579efa8905092d0; +archive prepared from clean 37c1ed1. Older archives remain unchanged. + +Texture run finished; Native then held a verified live hidden-vulkan capture. +One claim correctly refused; peer readiness queued 01a16938-3a04-7a53-b2b7-c8c018ab58e7. +After that owner exited, a fresh exclusive claim/process guard ran the extracted +checks. No peer process or lease was reclaimed. All owned GUIs closed and lease +released; receipt `mp-sd7yt8w5/released-runtime-lease.json`. Native availability +queued 01a16939-3a0d-7f03-8b6a-8da15723c99a. No immediate further GUI launch planned. + +Next authoring gap remains whole-creature presentation/skin assembly and live +compatibility. Reuse the existing native appearance/body evidence and establish +which meshes/transforms the engine selects before changing scene algorithms. +Do not repeat the accepted profile/import/native graph gates without a relevant +change. Keep gameplay checks separate from marker/individual-model acceptance. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index aab679b8..a4b9c1a7 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -1,6 +1,6 @@ -> Current development checkpoint, 2026-10-23: [Creature authoring continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). The combined creature-placement archive passes 118 identities, 142 documentation links and four extracted workflows, including 50 import cases and 16 marker placements at both sizes. Pack material identity and custom-name class/owner routing are included. 145 workflows not run; two unrelated corpus skips. Whole-creature presentation and gameplay remain unverified; RC2 unchanged. +> Current development checkpoint, 2026-10-23: [Creature placement profiles](HANDOFF_CREATURE_PROFILES_2026-10-23.md). The runtime-profile archive passes 119 identities, 145 documentation links and four extracted workflows: retail/GUI smoke, 16 metadata-profile placements and eight malformed-profile refusals. 146 workflows not run; two unrelated corpus skips. The unchanged import gate was not repeated. Whole-creature presentation and gameplay remain unverified; RC2 unchanged. -Source follow-up: [Creature placement profiles](HANDOFF_CREATURE_PROFILES_2026-10-23.md) now use runtime components; units, 517 native profiles, 16 placements and eight malformed-profile refusals pass. The retained ZIP predates this correction. +[Creature authoring history](HANDOFF_CREATURE_GUI_2026-10-23.md) retains the earlier import and material evidence; older checkpoint ZIPs remain unchanged. # Navigation lane, 2026-09-27 diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 17ca05fb..da3680e1 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -1,4 +1,4 @@ -> Current development checkpoint, 2026-10-23: [Creature authoring continuation](HANDOFF_CREATURE_GUI_2026-10-23.md). The combined creature-placement archive passes 118 identities, 142 documentation links and four extracted workflows, including 50 import cases and 16 marker placements at both sizes. Pack material identity and custom-name class/owner routing are included. 145 workflows not run; two unrelated corpus skips. Whole-creature presentation and gameplay remain unverified; RC2 unchanged. +> Current development checkpoint, 2026-10-23: [Creature placement profiles](HANDOFF_CREATURE_PROFILES_2026-10-23.md). The runtime-profile archive passes 119 identities, 145 documentation links and four extracted workflows: retail/GUI smoke, 16 metadata-profile placements and eight malformed-profile refusals. 146 workflows not run; two unrelated corpus skips. The unchanged import gate was not repeated. Whole-creature presentation and gameplay remain unverified; RC2 unchanged. # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there From 24ade7972b0d433c8cd9a042a2e49dff25d94f7c Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:02:11 -0600 Subject: [PATCH 62/75] Recover native Creature appearance selection and verify RNG bytes --- docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md | 77 ++++++++++++++++ tools/audit_creature_appearance_rng.py | 94 ++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 tools/audit_creature_appearance_rng.py diff --git a/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md index 664036e4..44dd8846 100644 --- a/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md @@ -227,3 +227,80 @@ compatibility. Reuse the existing native appearance/body evidence and establish which meshes/transforms the engine selects before changing scene algorithms. Do not repeat the accepted profile/import/native graph gates without a relevant change. Keep gameplay checks separate from marker/individual-model acceptance. + +## Offline appearance investigation + +Membership correction CI [39971393188](https://github.com/BuffJesus/FableForge/actions/runs/39971393188) +completed successfully on 37c1ed1: build, unit tests, WARP GUI without an install +and CLI usage passed. Package/upload steps were skipped. This does not add +coverage to the separately retained extracted-checkpoint receipts above. + +No new Forge runtime claim was needed during Native's V84 night/hidden captures. +The previously requested membership gap was explicitly cleared through the +authorized Native queue; no Forge reservation remains. Continue using fresh +guarded per-window claims for future runtime work. + +The next falsifiable question is which body meshes the engine selects for a +random-appearance creature, rather than rendering its root Graphic alone or +every alternative in its CCreatureDef. Evidence is retained in +`build/creature-assembly-native-20261023/`. All exports used the existing FableWin +Ghidra program with `-readOnly -noanalysis`; no import, analysis or project save. +Executable SHA256 remains +`a9d6d0977d9d7fc8242da4292ae7ed92e926ca80364a95c003d9b9f689e41845`. +`byte-verification.json` independently matches all 14 complete function byte +ranges to that PE. Clean C exports retain their original decompiler warnings; +ported labels/types are interpretation aids, not an ABI contract. + +Observed paths: + +- 01d067f0, CTCGraphicAppearance::OnCreate, obtains animation and optional + CAppearanceDef lighting configuration; 01d09e20 returns the main graphic at + component offset 0x6c. 0263a9d0 checks that graphic is animating-mesh type 4. +- 0252d9c0, CTCRandomAppearanceMorph::OnCreate, finds optional skeletal morph, + applies its stored seed and checks animating-mesh type 4. 0252dbd0 obtains + CCreatureDef's RandomAppearanceMorph, builds its morph set, transfers per-part + meshes/texture morphs, and applies skeletal morphs under the RAND identifier. +- 0289ac90 calls skeletal selection, body selection, then global texture + selection with the same seed reference. 0289b110 loops exactly three body + groups, skips empty groups, and draws one alternative from each nonempty group. + It processes that mesh's texture table and the body group's texture table. + Decapitation suppresses group zero only after those selection steps. Direct + original-PE disassembly confirms the group bound, shared seed pointer, + alternative-count RNG call and conditional push. See 0289b110.disasm.txt. +- 0252e720 iterates the resulting mesh parts and makes persistent primitives, + including an animation-from-file branch and a normal morphed animating-mesh + branch using the shared appearance/animation/position/orientation inputs. + Its recovered C has unreliable argument types; do not copy its guessed ABI + or infer attachment transforms without checking the relevant instructions. +- 026c4230 requires CReplaceableMeshDef on the separate replaceable-mesh path. + This is not evidence that all creatures use that path. + +A read-only census (`component-census.json`) validated all 517 retail CREATURE +runtime component lists and preserved game.bin/names.bin hashes. Counts overlap: +516 CTCGraphicAppearance, 290 CTCRandomAppearanceMorph, 199 +CTCGraphicAppearanceAnimatedMesh, 191 CTCSkeletalMorph, and four each CTCHeroMorph +and CTCHeroAttachableAppearanceModifiers. Root-only rendering therefore misses a +widely used component path. These are definition counts, including templates, +not counts of rendered or successfully assembled NPCs. + +The RNG primitive at 018cf370 updates a uint32 seed with multiply 0x24a1, add +0x24df and rotate-right 13, then returns seed modulo bound (zero bound returns +zero after still advancing the seed). Rotate thunk 0183fdab targets 018bad60. +`tools/audit_creature_appearance_rng.py` pins the executable SHA and executes only +these original x86 ranges in Unicorn, with 128-instruction/100ms bounds per call. +32 seeds, 11 bounds and four consecutive draws pass all 1,408 comparisons; +receipt `rng-original-x86.json` pins instruction hashes and unchanged executable. +This is an RNG primitive oracle, not an engine process or rendering acceptance. +Reproduce with pefile/unicorn installed: + +```text +python tools/audit_creature_appearance_rng.py --exe --output /rng-original-x86.json +``` + +Next: finish skeletal/texture selection ordering and original TNG Seed semantics, +then expose a checked read-only appearance plan using existing creature-prefix +abstractions. Validate selection against native execution before connecting +whole-creature preview/scene rendering. Do not concatenate every alternative, +reset the seed independently per body part, or treat one imported model as a +whole-creature result. Production rendering, native bank writers and retained +checkpoint ZIPs are unchanged by this investigation. diff --git a/tools/audit_creature_appearance_rng.py b/tools/audit_creature_appearance_rng.py new file mode 100644 index 00000000..22887693 --- /dev/null +++ b/tools/audit_creature_appearance_rng.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Compare the recovered appearance RNG with bounded original FableWin x86 execution. + +Read-only evidence tool; requires pefile and unicorn. This checks the RNG primitive, +not body-selection ordering, serialization, animation or rendered compatibility. +""" +import argparse +import hashlib +import json +from pathlib import Path +import struct + +import pefile +from unicorn import Uc, UC_ARCH_X86, UC_MODE_32 +from unicorn.x86_const import UC_X86_REG_EAX, UC_X86_REG_ECX, UC_X86_REG_EDX, UC_X86_REG_EIP, UC_X86_REG_ESP + + +EXPECTED_SHA = 'a9d6d0977d9d7fc8242da4292ae7ed92e926ca80364a95c003d9b9f689e41845' +ENTRY = 0x018cf370 +ROTATE_THUNK = 0x0183fdab +ROTATE = 0x018bad60 + + +def recovered(seed, bound): + word = (seed * 0x24a1 + 0x24df) & 0xffffffff + seed = ((word >> 13) | (word << 19)) & 0xffffffff + return seed, seed % bound if bound else 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--exe', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + raw = args.exe.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if digest != EXPECTED_SHA: + raise ValueError('unsupported executable identity; no execution attempted') + if args.output.resolve() == args.exe.resolve(): + raise ValueError('evidence output must differ from executable') + pe = pefile.PE(data=raw) + base = pe.OPTIONAL_HEADER.ImageBase + machine = Uc(UC_ARCH_X86, UC_MODE_32) + ranges = [(ENTRY, 87), (ROTATE_THUNK, 5), (ROTATE, 18)] + for address, length in ranges: + machine.mem_map(address & ~0xfff, 0x1000) + machine.mem_write(address, pe.get_data(address-base, length)) + stack, seed_at, stop = 0x10010000, 0x10020000, 0x10030000 + for address in (stack-0x1000, seed_at, stop): + machine.mem_map(address, 0x1000) + bounds = [0, 1, 2, 3, 7, 50, 100, 255, 4096, 0x7fffffff, 0xffffffff] + seeds = [0, 1, 2, 0x7fffffff, 0x80000000, 0xfffffffe, 0xffffffff] + seeds += [int.from_bytes(hashlib.sha256(str(i).encode('ascii')).digest()[:4], 'little') + for i in range(25)] + comparisons = 0 + samples = [] + for initial in seeds: + for bound in bounds: + seed = initial + for step in range(4): + expected_seed, expected_result = recovered(seed, bound) + machine.mem_write(stack-4, struct.pack(' Date: Fri, 23 Oct 2026 09:12:31 -0600 Subject: [PATCH 63/75] Audit complete native Creature appearance table layout --- docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md | 59 ++++++ tools/audit_creature_appearance_layout.py | 209 +++++++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 tools/audit_creature_appearance_layout.py diff --git a/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md index 44dd8846..d01c453d 100644 --- a/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md @@ -304,3 +304,62 @@ whole-creature preview/scene rendering. Do not concatenate every alternative, reset the seed independently per body part, or treat one imported model as a whole-creature result. Production rendering, native bank writers and retained checkpoint ZIPs are unchanged by this investigation. + +### Complete appearance layout evidence + +Follow-up targeted exports bring `byte-verification.json` to 23 distinct complete +function ranges, all matching the original PE. Added evidence includes +0252dba0 (seed assignment/rebuild), 0252e4e0 (unsigned `Seed` serialization), +0252e660 (editor `RandomAppearanceSeed`), 0252d680 (constructor), 0289b010 +(skeletal amplitude), 018bac90 (float RNG), 0289b880 (appearance binary loader) +and 028a55b0 (texture map loader). The constructor copies the current world seed +into its stored seed and advances the world generator. A placed creature with +no serialized Seed therefore cannot be presumed to have seed zero or a fixed +appearance independent of world creation order. Retail StartOakValeWest TNG +examples store signed decimal Seed values; preserve their uint32 bit patterns. + +The native loader asserts exactly three body groups. It reads each body +alternative's mesh, priority and texture map, then three body-level texture maps, +the skeletal table, a global texture map and the maximum texture-group ID. +The map loader verifies its byte-size field against the resized map, reads its +raw rows and restores a separate Dirty boolean. Original-PE and matching PDB +queries establish 12-byte texture rows: original texture key, NewTexture and +GroupID. `CBodyPartMesh` fields are MeshIndex, Priority and TextureMorphs; +`CSkeletalMorph` contains GroupID, Name and AllowVariation. Query outputs are +`CBodyPartMesh-pdb.txt`, `CTextureMorph-pdb.txt`, `CSkeletalMorph-pdb.txt` in the +same evidence directory (FableWin.pdb GUID 0B63010E-1E75-49E7-ADDA-5E6B10298BAB, +age 8). These are memory layouts corroborating the separately recovered +serialized loader, not permission to memcpy complete runtime C++ objects. + +`tools/audit_creature_appearance_layout.py` is an independent, bounded read-only +reader for this evidence. It validates counts, map byte sizes, booleans, names +pool boundaries and terminal control markers before reporting the layout. +`layout-independent-audit.json` passes all 330 retail CCreatureDef records, +1,745 body slots and 75 skeletal rows, with game.bin/names.bin hashes unchanged. +It agrees exactly with the previous body audit on 329 records. Component 10594 +has a four-row global texture map (source 214, alternatives 214/215/214/216) +and was previously refused at its control marker because that reader skips an +assumed empty global map. All body-level maps are empty in this corpus; 80 +components have nonzero texture-group counts. Every observed map byte size is +12 times its row count, and every observed Dirty flag is false. + +The independent audit also passes a synthetic populated global map and refuses +all 120 strict truncations and five malformed count/size/name inputs. Command: + +```text +python tools/audit_creature_appearance_layout.py --install --output /layout-independent-audit.json +``` + +The initial ad hoc `texture-map-census.json` was rejected: its Python cursor +update evaluated before a count read and misaligned every record. The corrected +exploratory report and the reusable audit agree on all 330 records. This was a +probe bug, not a native data defect. Do not use the rejected report as evidence. + +Remaining selection details: key-zero skeletal entries use a conditional draw +unless AllowVariation is true; nonzero skeletal groups draw before choosing a +candidate. Selected varying skeletal morphs consume a float RNG draw. Texture +selection groups rows by original-texture key and admits GroupID zero or the +chosen color group before drawing a replacement. Their exact thresholds, +sorting and draw order still need instruction/oracle validation before a core +appearance plan is integrated. Production reader/writer and renderer behavior +remain unchanged; this audit does not claim whole-creature rendering. diff --git a/tools/audit_creature_appearance_layout.py b/tools/audit_creature_appearance_layout.py new file mode 100644 index 00000000..ac09294f --- /dev/null +++ b/tools/audit_creature_appearance_layout.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Independent read-only audit of native CCreatureDef appearance serialization. + +Uses the recovered FableWin loader/PDB layout, including body/global texture maps. +This is evidence for a future checked core reader, not a writer or renderer gate. +""" +import argparse +from collections import Counter +import hashlib +import json +from pathlib import Path +import struct + +from audit_creature_body_models import body_slots +from test_fmp_links import crc0, read_bank + + +class Reader: + def __init__(self, data): + self.data, self.at, self.total = data, 3, 0 + + def take(self, size): + if size < 0 or self.at > len(self.data) or size > len(self.data)-self.at: + raise ValueError('truncated appearance layout') + start = self.at + self.at += size + return start + + def word(self): + return struct.unpack_from(' 4096 or self.total > 65536 or value > (len(self.data)-self.at)//minimum: + raise ValueError('invalid appearance count') + return value + + def textures(self): + count = self.count(12) + if self.word() != count*12: + raise ValueError('texture-map byte size does not match count') + rows = [] + for _ in range(count): + offset = self.at + rows.append(dict(offset=offset, original=self.word(), replacement=self.word(), group=self.word())) + return dict(rows=rows, dirty=self.flag()) + + +def decode(data, names): + reader = Reader(data) + reader.tag(crc0('WoundMorphs')) + wounds = reader.count(44) + reader.take(wounds*44) + reader.tag(0x01ab6cee) + reader.tag(crc0('RandomAppearanceMorph')) + groups = reader.count(4) + if groups != 3: + raise ValueError('native appearance requires exactly three body groups') + bodies, slots = [], [] + for body in range(groups): + alternatives = [] + for alternative in range(reader.count(17)): + offset = reader.at + mesh, priority = reader.word(), reader.word() + textures = reader.textures() + alternatives.append(dict(mesh=mesh, priority=priority, textures=textures)) + slots.append(dict(body_part=body, alternative=alternative, offset=offset, id=mesh)) + bodies.append(alternatives) + group_textures = [reader.textures() for _ in range(groups)] + skeleton = [] + for _ in range(reader.count(13)): + key, group, name = reader.word(), reader.word(), reader.word() + variation = reader.flag() + if name not in names: + raise ValueError('skeletal name is not a names-pool boundary') + skeleton.append(dict(key=key, group=group, name_offset=name, + name=names[name], allow_variation=variation)) + global_textures = reader.textures() + variants = reader.word() + reader.tag(crc0('Expressions')) + expressions = reader.count(12) + reader.take(expressions*12) + reader.tag(0x01ab6d32) + reader.tag(crc0('Stats')) + return dict(bodies=bodies, body_texture_maps=group_textures, + skeletal_morphs=skeleton, global_texture_map=global_textures, + texture_group_count=variants, slots=slots, + stats_value_offset=reader.at) + + +def names_pool(raw): + if len(raw) < 20: + raise ValueError('truncated names header') + count, = struct.unpack_from(' (len(raw)-20)//5: + raise ValueError('invalid names count') + names, at = {}, 20 + for _ in range(count): + end = raw.index(0, at+4) + names[at+4-20] = raw[at+4:end].decode('ascii') + at = end+1 + if at != len(raw): + raise ValueError('unsupported names tail') + return names + + +def boundary_checks(): + # Minimal native shape exercises a populated global map and a name boundary. + word = lambda *values: struct.pack('<'+'I'*len(values), *values) + data = b'\x01\0\x01'+word(crc0('WoundMorphs'), 0, 0x01ab6cee, + crc0('RandomAppearanceMorph'), 3, 0, 0, 0) + data += (word(0, 0)+b'\0')*3 + skeletal_count = len(data) + data += word(1, 0, 0, 4)+b'\1' + map_count = len(data) + data += word(1, 12, 214, 215, 0)+b'\0' + data += word(0, crc0('Expressions'), 0, 0x01ab6d32, crc0('Stats')) + result = decode(data, {4:'MORPH'}) + if result['global_texture_map']['rows'][0]['replacement'] != 215: + raise AssertionError('populated global map was not decoded') + for end in range(len(data)): + try: + decode(data[:end], {4:'MORPH'}) + except (ValueError, struct.error, IndexError): + pass + else: + raise AssertionError(f'truncated layout accepted at {end}') + corruptions = [('body count', 19, 2), ('skeletal count', skeletal_count, 0xffffffff), + ('name boundary', skeletal_count+12, 5), + ('map count', map_count, 0xffffffff), ('map byte size', map_count+4, 11)] + for label, offset, value in corruptions: + changed = bytearray(data) + struct.pack_into(' Date: Fri, 23 Oct 2026 09:29:11 -0600 Subject: [PATCH 64/75] Share checked Creature appearance layout across core authoring --- docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md | 72 ++++++++++++ docs/ROADMAP_1.0.md | 7 ++ .../include/forge/creatureappearance.hpp | 46 ++++++++ libs/forgecore/src/creatureappearance.cpp | 109 ++++++++++++++++++ libs/forgecore/src/fmplinks.cpp | 77 ++++--------- tests/test_creaturevisualclone.cpp | 3 +- tests/test_defrelocate.cpp | 58 ++++++++++ tests/test_fmplinks.cpp | 70 +++++++++-- tests/test_skinnedinput.cpp | 3 +- tools/audit_creature_appearance_layout.py | 21 ++++ tools/test_creature_body_stale_ui.py | 4 +- 11 files changed, 404 insertions(+), 66 deletions(-) create mode 100644 libs/forgecore/include/forge/creatureappearance.hpp create mode 100644 libs/forgecore/src/creatureappearance.cpp diff --git a/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md index d01c453d..c539d0be 100644 --- a/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md @@ -363,3 +363,75 @@ chosen color group before drawing a replacement. Their exact thresholds, sorting and draw order still need instruction/oracle validation before a core appearance plan is integrated. Production reader/writer and renderer behavior remain unchanged; this audit does not claim whole-creature rendering. + +### Shared checked core reader + +`forge::creatureappearance::read` now represents all three body groups, per-model, +per-body and global texture maps, skeletal rows, texture-group count, wound and +expression boundaries. It verifies counts/allocation budgets, 12-byte map sizes, +booleans and control tags before returning. Offsets retain original payload +positions. The input and opaque Stats/tail bytes are never rewritten; skeletal +names remain offsets until checked against the owning bank's pool. + +FMP prefix enumeration/lookup delegates to this reader instead of its previous +independent fixed-size skips. Body/global texture keys and replacements are +classified in the texture asset namespace, so equal game-entry/name numbers do +not cause namespace relocation. Validated map sizes/counts are structural words. +Malformed metadata remains fail-closed. No scene renderer or model selection +algorithm is connected yet. + +Source acceptance: + +- Four affected unit executables pass: fmplinks, defrelocate, creaturevisualclone + and skinnedinput. The new complete-map unit covers populated body/global maps, + truncations, malformed counts/sizes/booleans and unchanged source bytes. +- The optional `--creature-appearance-audit` mode of the relocation test reader + compares every exposed field and offset with the independent Python reader. + All 330 components match exactly, including component 10594's global table; + all original bank hashes remain unchanged. Receipt: + `C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/ca-tni66p7h/layout-core-comparison.json`. + This pins the audit executable/schema hashes and 330 exact row comparisons. +- A follow-up namespace test proves body/global texture and mesh IDs survive + collisions with game-entry maps while only the intended skeletal name offset + changes. Updated fmplinks executable passes in `ad-vyn23vpj/0.log`. +- GUI, forge and forge-tools shipping builds pass (`build/creature-appearance-shipping-build-20261023.log`). + +The first fmplinks run refused an old synthetic test that deliberately placed +777 in a texture table's byte-size field. The new reader correctly rejected the +invalid size, leaving the relocation scanner unable to classify that prefix. +The fixture now uses the valid native size (24 for two rows) and still tests a +candidate crossing that boundary. No scanner guard was weakened. Initial +receipt/log: `ca-7oaz_tcx/0.log`; successful four-unit window `ca-tni66p7h`. + +Older synthetic authoring fixtures used one or two body groups followed by three +empty body texture maps, unlike the actual native three-group contract. Their +payloads now contain three groups, retaining empty groups explicitly. Existing +body-model offsets and tested edit intent are unchanged. The old retained donor +packet is preserved. A distinct regenerated packet from the updated reader is +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/ad-vyn23vpj/skinned-input-donors-1a862nw7`. +All five donors pass authored remap, stored/compressed independent skeleton +readback and refusal checks; its receipt retains publication/Restore/Windows +rollback checks. This regeneration is warranted by the changed fixture shape, +not a new claim of whole-creature rendering or gameplay. + +The relevant 50-case GUI import regression passes at both sizes against the +regenerated packet. Receipt: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/ai-w9lnk8f4/creature-import-ui-4nstujwj/result.json`. +It pins candidate GUI SHA256 +`63155c0c4755700c29f5fae48275a967f1712aa9b092ed0fb84415aa9e20a06f`. +All eight affected body-edit Save/stale-bank/stale-names/stale-graphics cases +also pass at both sizes, preserving fixtures exactly and exercising Restore. +Receipt under `ab-mck4uft3/creature-body-stale-ui-*/result.json`. +No older donor packet or checkpoint archive was modified. + +Each source check window used a fresh process guard/exclusive RuntimeLease and +queued release after owned checks/hidden GUIs closed. Final release receipt is +`ab-mck4uft3/released-runtime-lease.json`, Native notification +`01a1695f-92b2-75a1-a57b-c8d55d5e0dbc`. No Forge runtime remains or gap is reserved. +Combined source hashes, binaries and receipts are pinned in +`build/creature-appearance-source-acceptance-20261023.json`. +The retained runtime-profile ZIP is still the previous immutable checkpoint; +the new shared reader has source acceptance, not extracted-package acceptance. +Do not repeat the accepted profile/placement gates for this unrelated change. +Next is native-checked selection/draw ordering and assembly transforms, not a +claim that the parsed layout already renders a complete NPC. diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index da3680e1..8ef1b63b 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -2,6 +2,13 @@ # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there +Subsequent source work adds a shared checked Creature appearance-table reader: +330 native components match an independent reader, including the populated +global texture map previously refused. Four affected units, five donor fixtures, +50 import GUI cases and eight body-edit cases pass. This is source acceptance; +the retained ZIP above is unchanged. Whole-creature selection and assembly are +still open. Evidence is in the linked Creature placement profiles handoff. + Ordered dialogue-tag controls pass source checks at both window sizes and an independent retail-copy preservation/Restore check. Signed stored positions remain explicit; new tag names are bounded ASCII and original legacy names survive other diff --git a/libs/forgecore/include/forge/creatureappearance.hpp b/libs/forgecore/include/forge/creatureappearance.hpp new file mode 100644 index 00000000..1a22fff2 --- /dev/null +++ b/libs/forgecore/include/forge/creatureappearance.hpp @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace forge::creatureappearance { +struct TextureMorph { + size_t offset = 0; + uint32_t original = 0, replacement = 0, group = 0; +}; +struct TextureMap { + std::vector rows; + bool dirty = false; +}; +struct BodyModel { + size_t offset = 0; + uint32_t mesh = 0; + int32_t priority = 0; + TextureMap textures; +}; +struct SkeletalMorph { + size_t offset = 0; + int32_t key = 0, group = 0; + uint32_t nameOffset = 0; + bool allowVariation = false; +}; +struct Prefix { + size_t wounds = 0, woundRows = 0, expressions = 0, expressionRows = 0; + std::array, 3> bodies; + std::array bodyTextures; + std::vector skeletal; + TextureMap globalTextures; + uint32_t textureGroupCount = 0; + size_t statsValueOffset = 0; + std::vector structuralWords; +}; +// Read a decoded CCreatureDef prefix through the Stats tag. Offsets refer to +// the original prefix; input/unknown Stats bytes are untouched. Counts, table +// byte sizes, booleans and control tags are checked before a result escapes. +// Name offsets still require validation against the owning bank's names pool. +// This is a serialized layout, not a selected/assembled or animated creature. +Prefix read(std::span bytes); +} // namespace forge::creatureappearance diff --git a/libs/forgecore/src/creatureappearance.cpp b/libs/forgecore/src/creatureappearance.cpp new file mode 100644 index 00000000..5358eba4 --- /dev/null +++ b/libs/forgecore/src/creatureappearance.cpp @@ -0,0 +1,109 @@ +#include "forge/creatureappearance.hpp" +#include "forge/defdecode.hpp" +#include +#include +#include +#include + +namespace forge::creatureappearance { +namespace { +[[noreturn]] void fail(const char* reason) { + throw std::runtime_error(std::string("Creature appearance: ") + reason); +} +class Reader { + std::span bytes_; + size_t total_ = 0; +public: + size_t at = 3; + std::vector structuralWords; + explicit Reader(std::span bytes) : bytes_(bytes) {} + size_t take(size_t length) { + if (at > bytes_.size() || length > bytes_.size()-at) fail("truncated prefix"); + const auto start = at; + at += length; + return start; + } + uint32_t word() { + const auto start = take(4); + uint32_t value = 0; + for (size_t n = 0; n < 4; ++n) value |= uint32_t(bytes_[start+n]) << (8*n); + return value; + } + bool flag() { + const auto value = bytes_[take(1)]; + if (value > 1) fail("invalid serialized boolean"); + return value != 0; + } + void tag(uint32_t expected) { + structuralWords.push_back(at); + if (word() != expected) fail("unsupported control tag"); + } + size_t count(size_t minimum) { + structuralWords.push_back(at); + const size_t value = word(); + if (value > 4096 || total_ > 65536-value || value > (bytes_.size()-at)/minimum) + fail("invalid count"); + total_ += value; + return value; + } + TextureMap textures() { + const auto size = count(12); + structuralWords.push_back(at); + if (word() != size*12) fail("texture map byte size disagrees with count"); + TextureMap result; + for (size_t n = 0; n < size; ++n) { + TextureMorph row; + row.offset = at; + row.original = word(); row.replacement = word(); row.group = word(); + result.rows.push_back(row); + } + result.dirty = flag(); + return result; + } +}; +} // namespace + +Prefix read(std::span bytes) { + // FableWin 0289b880 reads three body groups, their texture maps, skeletal + // rows, global texture map and color-group count. 028a55b0 checks 12-byte + // texture rows. See HANDOFF_CREATURE_PROFILES_2026-10-23.md for byte/PDB evidence. + Reader reader(bytes); + Prefix result; + reader.tag(defdecode::fieldTag("WoundMorphs")); + result.wounds = reader.count(44); result.woundRows = reader.at; + reader.take(result.wounds*44); + reader.tag(0x01ab6cee); + reader.tag(defdecode::fieldTag("RandomAppearanceMorph")); + if (reader.count(4) != result.bodies.size()) fail("expected three body groups"); + for (auto& body : result.bodies) { + const auto size = reader.count(17); + for (size_t n = 0; n < size; ++n) { + BodyModel model; + model.offset = reader.at; + model.mesh = reader.word(); model.priority = std::bit_cast(reader.word()); + model.textures = reader.textures(); + body.push_back(std::move(model)); + } + } + for (auto& map : result.bodyTextures) map = reader.textures(); + const auto size = reader.count(13); + for (size_t n = 0; n < size; ++n) { + SkeletalMorph morph; + morph.offset = reader.at; + morph.key = std::bit_cast(reader.word()); + morph.group = std::bit_cast(reader.word()); + morph.nameOffset = reader.word(); morph.allowVariation = reader.flag(); + result.skeletal.push_back(morph); + } + result.globalTextures = reader.textures(); + result.textureGroupCount = reader.word(); + reader.tag(defdecode::fieldTag("Expressions")); + result.expressions = reader.count(12); result.expressionRows = reader.at; + reader.take(result.expressions*12); + reader.tag(0x01ab6d32); + reader.tag(defdecode::fieldTag("Stats")); + result.statsValueOffset = reader.at; + result.structuralWords = std::move(reader.structuralWords); + return result; +} +} // namespace forge::creatureappearance diff --git a/libs/forgecore/src/fmplinks.cpp b/libs/forgecore/src/fmplinks.cpp index 0fbcc298..f350b99f 100644 --- a/libs/forgecore/src/fmplinks.cpp +++ b/libs/forgecore/src/fmplinks.cpp @@ -1,5 +1,6 @@ #include "forge/fmplinks.hpp" #include "forge/defdecode.hpp" +#include "forge/creatureappearance.hpp" #include #include #include @@ -208,69 +209,31 @@ struct CreaturePrefix { size_t wounds = 0, woundRows = 0, expressions = 0, expressionRows = 0; struct Model { size_t mesh = 0; std::vector textures; }; std::vector> bodies; - std::vector skeletons; + std::vector skeletons, otherTextures; std::vector structuralWords; }; CreaturePrefix creaturePrefix(std::span bytes) { - // Native CCreatureDef leading counted-pointer blocks, described by - // ChocolateBox FFF38E28/237DCBE3/F0F1CA81. Walk boundaries; never search - // opaque bytes for an apparent reference tag. - size_t cursor = 3; + const auto decoded = creatureappearance::read(bytes); CreaturePrefix result; - auto skip = [&](size_t length) { - if (cursor > bytes.size() || length > bytes.size() - cursor) - fail("truncated creature prefix layout"); - cursor += length; - }; - auto word = [&]() { const auto value = read32(bytes, cursor); cursor += 4; return value; }; - auto tag = [&](uint32_t expected) { - const auto at = cursor; - if (word() != expected) fail("unsupported creature prefix control layout"); - result.structuralWords.push_back(at); - }; - auto count = [&](size_t minimum) { - const auto at = cursor; - const size_t value = word(); - if (value > 4096 || cursor > bytes.size() || value > (bytes.size() - cursor) / minimum) - fail("invalid creature prefix count"); - result.structuralWords.push_back(at); - return value; - }; - tag(defdecode::fieldTag("WoundMorphs")); - result.wounds = count(44); result.woundRows = cursor; skip(result.wounds * 44); - tag(0x01ab6ceeu); - tag(defdecode::fieldTag("RandomAppearanceMorph")); - const size_t bodies = count(4); - size_t allModels = 0, allTextures = 0; - for (size_t body = 0; body < bodies; ++body) { - const size_t models = count(17); allModels += models; - if (allModels > 4096) fail("invalid creature prefix model count"); - result.bodies.emplace_back(); - for (size_t model = 0; model < models; ++model) { - CreaturePrefix::Model layout; layout.mesh = cursor; - skip(8); - const size_t textures = count(12); allTextures += textures; - if (allTextures > 4096) fail("invalid creature prefix texture count"); - skip(4); // serialized array metadata, retained verbatim - for (size_t texture = 0; texture < textures; ++texture) { - layout.textures.push_back(cursor); skip(12); - } - skip(1); result.bodies.back().push_back(std::move(layout)); + result.wounds = decoded.wounds; result.woundRows = decoded.woundRows; + result.expressions = decoded.expressions; result.expressionRows = decoded.expressionRows; + result.structuralWords = decoded.structuralWords; + for (const auto& body : decoded.bodies) { + auto& models = result.bodies.emplace_back(); + for (const auto& model : body) { + CreaturePrefix::Model converted; + converted.mesh = model.offset; + for (const auto& texture : model.textures.rows) converted.textures.push_back(texture.offset); + models.push_back(std::move(converted)); } } - skip(3 + 24); - const size_t skeletal = count(13); - for (size_t index = 0; index < skeletal; ++index) { - result.skeletons.push_back(cursor + 8); skip(13); - } - skip(1 + 12); - tag(defdecode::fieldTag("Expressions")); - result.expressions = count(12); result.expressionRows = cursor; - skip(result.expressions * 12); - tag(0x01ab6d32u); - tag(defdecode::fieldTag("Stats")); + for (const auto& morph : decoded.skeletal) result.skeletons.push_back(morph.offset + 8); + for (const auto& map : decoded.bodyTextures) + for (const auto& row : map.rows) result.otherTextures.push_back(row.offset); + for (const auto& row : decoded.globalTextures.rows) result.otherTextures.push_back(row.offset); return result; } + bool identicalGameOwners(const big::File& package, const std::string& owner) { const auto* records = package.findBank("GameBINEntries"); if (!records) return false; @@ -305,6 +268,10 @@ CreaturePrefixLayout creaturePrefixLayout(std::span bytes) { result.references.push_back({texture + 4, 0, Destination::TextureEntry}); } } + for (const auto texture : prefix.otherTextures) { + result.references.push_back({texture, 0, Destination::TextureEntry}); + result.references.push_back({texture + 4, 0, Destination::TextureEntry}); + } for (const auto skeleton : prefix.skeletons) result.references.push_back({skeleton, 0, Destination::NameOffset}); for (size_t row = 0; row < prefix.expressions; ++row) diff --git a/tests/test_creaturevisualclone.cpp b/tests/test_creaturevisualclone.cpp index 492661cf..9c543c90 100644 --- a/tests/test_creaturevisualclone.cpp +++ b/tests/test_creaturevisualclone.cpp @@ -219,8 +219,9 @@ int main(int argc,char** argv)try{ { const auto bindingSchema=defschema::Schema::loadText(R"({"CCreatureDef":{"fields":[{"name":"InitialActions","type":"uint32"}]}})"); Bytes payload{1,0,1};field(payload,"WoundMorphs",0);word(payload,0x01ab6cee); - field(payload,"RandomAppearanceMorph",1);word(payload,1); + field(payload,"RandomAppearanceMorph",3);word(payload,1); const size_t modelAt=payload.size();word(payload,10);word(payload,0);word(payload,0);word(payload,0);payload.push_back(0); + word(payload,0);word(payload,0); payload.insert(payload.end(),27,0);word(payload,0);payload.insert(payload.end(),13,0); field(payload,"Expressions",0);word(payload,0x01ab6d32);field(payload,"Stats",0);field(payload,"InitialActions",0); auto baseline=source;baseline.setEntryData(3,payload); diff --git a/tests/test_defrelocate.cpp b/tests/test_defrelocate.cpp index 09678149..90f97784 100644 --- a/tests/test_defrelocate.cpp +++ b/tests/test_defrelocate.cpp @@ -1,6 +1,7 @@ #include "forge/defrelocate.hpp" #include "forge/defdecode.hpp" #include "forge/fmplinks.hpp" +#include "forge/creatureappearance.hpp" #include #include #include @@ -98,6 +99,63 @@ int audit(char** argv) { } int main(int argc, char** argv) { + if(argc==4 && std::string_view(argv[1])=="--creature-appearance-audit") { + try { + using namespace forge; + using Json=nlohmann::json; + const auto folder=std::filesystem::path(argv[2])/"data"/"CompiledDefs"; + const auto schema=defschema::Schema::load(argv[3]); + const auto source=bin::File::open(folder/"names.bin",folder/"game.bin"); + std::map names; + uint64_t offset=4; + for(const auto& [crc,name]:source.originalNames()) { + if(offset>UINT32_MAX)throw std::runtime_error("names offset overflow"); + names.emplace(uint32_t(offset),name);offset+=name.size()+5; + } + const auto mapJson=[](const creatureappearance::TextureMap& map) { + auto rows=Json::array(); + for(const auto& row:map.rows)rows.push_back({{"offset",row.offset},{"original",row.original}, + {"replacement",row.replacement},{"group",row.group}}); + return Json{{"rows",rows},{"dirty",map.dirty}}; + }; + auto rows=Json::array(); + for(size_t i=0;isecond},{"allow_variation",morph.allowVariation}}); + } + row["global_texture_map"]=mapJson(layout.globalTextures); + row["texture_group_count"]=layout.textureGroupCount; + row["stats_value_offset"]=layout.statsValueOffset; + }catch(const std::exception& e){row["refused"]=e.what();} + rows.push_back(std::move(row)); + } + std::cout< #include #include @@ -52,6 +53,60 @@ int main() { auto rejects = [&](auto action, const char* what) { try { action(); check(false, what); } catch (const std::runtime_error&) {} }; + { + Bytes data{1,0,1}; + word(data,forge::defdecode::fieldTag("WoundMorphs"));word(data,0);word(data,0x01ab6cee); + word(data,forge::defdecode::fieldTag("RandomAppearanceMorph")); + const auto groupsAt=data.size();word(data,3); + word(data,1);word(data,9);word(data,7); + word(data,1);word(data,12);word(data,100);word(data,101);word(data,2);data.push_back(1); + word(data,0);word(data,0); + word(data,1);const auto byteSizeAt=data.size();word(data,12); + word(data,200);word(data,201);word(data,0);data.push_back(0); + for(int n=0;n<2;++n){word(data,0);word(data,0);data.push_back(0);} + word(data,1);word(data,2);word(data,2);word(data,4);data.push_back(1); + word(data,2);word(data,24); + word(data,300);word(data,301);word(data,0); + word(data,300);word(data,302);word(data,2);const auto flagAt=data.size();data.push_back(0); + word(data,3);word(data,forge::defdecode::fieldTag("Expressions"));word(data,1); + word(data,0);word(data,0);word(data,400);word(data,0x01ab6d32); + word(data,forge::defdecode::fieldTag("Stats")); + const auto original=data; + const auto parsed=forge::creatureappearance::read(data); + check(parsed.bodies[0].size()==1 && parsed.bodies[1].empty() && parsed.bodies[2].empty() && + parsed.bodies[0][0].mesh==9 && parsed.bodies[0][0].priority==7 && + parsed.bodies[0][0].textures.dirty && parsed.bodies[0][0].textures.rows[0].group==2 && + parsed.bodyTextures[0].rows[0].replacement==201 && + parsed.globalTextures.rows.size()==2 && parsed.globalTextures.rows[1].replacement==302 && + parsed.skeletal.size()==1 && parsed.skeletal[0].nameOffset==4 && parsed.skeletal[0].allowVariation && + parsed.textureGroupCount==3 && parsed.statsValueOffset==data.size() && data==original, + "complete appearance reader preserves all body/global maps, skeletal fields and input bytes"); + const auto locations=forge::fmplinks::creaturePrefixLayout(data); + check(locations.references.size()==11 && locations.bodyModels.size()==1, + "body/global texture pairs join existing verified mesh/name/expression reference domains"); + const auto schema=forge::defschema::Schema::loadText( + R"({"CCreatureDef":{"fields":[{"name":"InitialActions","type":"uint32"}]}})"); + auto payload=data;word(payload,0); + word(payload,forge::defdecode::fieldTag("InitialActions"));word(payload,0); + auto expected=payload; + const auto nameAt=parsed.skeletal[0].offset+8; + for(size_t n=0;n<4;++n)expected[nameAt+n]=uint8_t(444u>>(8*n)); + const forge::bin::Entry entry{"CCreatureDef","ALL_TEXTURE_MAPS",0,payload}; + const forge::defrelocate::Maps maps{{{9,90},{200,900},{300,999}},{{4,444}}}; + check(forge::defrelocate::relocate(entry,schema,maps)==expected && entry.data==payload, + "body/global asset namespace collisions survive relocation while only skeletal name changes"); + for(size_t length=0;length>{ + {groupsAt,2},{groupsAt,4},{groupsAt,UINT32_MAX},{byteSizeAt,11},{byteSizeAt-4,UINT32_MAX}}) { + auto changed=data; + for(size_t n=0;n<4;++n)changed[at+n]=uint8_t(value>>(8*n)); + rejects([&]{forge::creatureappearance::read(changed);},"invalid native body count/map size refuses"); + } + auto badFlag=data;badFlag[flagAt]=2; + rejects([&]{forge::creatureappearance::read(badFlag);},"invalid texture map boolean refuses"); + } const auto valid = metadata(); const auto links = forge::fmplinks::readMetadata(valid); check(links.size() == 1 && links[0].path == "123._" && links[0].target == "TARGET", "forward refs, arrays and null padding"); @@ -222,12 +277,12 @@ int main() { const size_t skinAt = prefix.size(); for (uint32_t value = 70; value < 79; ++value) word(prefix, value); word(prefix, 0x01ab6cee); word(prefix, forge::defdecode::fieldTag("RandomAppearanceMorph")); - word(prefix, 2); word(prefix, 1); const size_t meshAt = prefix.size(); + word(prefix, 3); word(prefix, 1); const size_t meshAt = prefix.size(); word(prefix, 9); word(prefix, 10); - word(prefix, 2); word(prefix, 2); + word(prefix, 2); word(prefix, 24); const size_t textureAt = prefix.size(); for (int n = 0; n < 6; ++n) word(prefix, 100+n); - prefix.push_back(1); word(prefix, 0); + prefix.push_back(1); word(prefix, 0); word(prefix, 0); prefix.insert(prefix.end(), 27, 0); word(prefix, 1); const size_t skeletonAt = prefix.size() + 8; prefix.insert(prefix.end(), 13, 0); prefix.insert(prefix.end(), 13, 0); @@ -248,7 +303,9 @@ int main() { auto shiftedBody=prefix; const size_t bodyCountAt=meshAt-4; shiftedBody.insert(shiftedBody.begin()+bodyCountAt,4,0); - shiftedBody[bodyCountAt-4]=3; + // Move the last empty group before the populated group, retaining three. + shiftedBody.erase(shiftedBody.begin()+textureAt+24+1+8, + shiftedBody.begin()+textureAt+24+1+12); const auto shiftedLayout=forge::fmplinks::creaturePrefixLayout( forge::defdecode::decode({"CCreatureDef","SHIFTED_BODY",0,shiftedBody},prefixSchema).prefix); check(shiftedLayout.bodyModels.size()==1 && shiftedLayout.bodyModels[0].bodyPart==1 && @@ -308,10 +365,9 @@ int main() { rejects([&] { forge::defrelocate::relocate(unsupported,prefixSchema,prefixMaps); }, "unsupported creature prefix refuses without partial namespace relocation"); auto counted = prefixRecord; - for (size_t n = 0; n < 4; ++n) counted.data[meshAt+12+n] = uint8_t(777u >> (8*n)); - const forge::defrelocate::Maps crossing{{{777u << 8,222222}}, {}}; + const forge::defrelocate::Maps crossing{{{24u << 8,222222}}, {}}; check(forge::defrelocate::relocate(counted,prefixSchema,crossing) == counted.data, - "candidate crossing validated creature array count boundary stays unchanged"); + "candidate crossing validated creature array byte-size boundary stays unchanged"); const std::vector prefixLinks{ {"4294151720.Skin Effects Array.0.Link to ID5", "SKIN"}, {"4042377857.unknown.1.Animations", "ANIMATION"}}; diff --git a/tests/test_skinnedinput.cpp b/tests/test_skinnedinput.cpp index 9a8e3fe3..a49a063c 100644 --- a/tests/test_skinnedinput.cpp +++ b/tests/test_skinnedinput.cpp @@ -85,8 +85,9 @@ static int donorGate(char** argv) { field(bytes,"Components",0);field(bytes,"Self",donorRoot?1:0);field(bytes,"Other",99);return bytes; }; Bytes body{1,0,1};field(body,"WoundMorphs",0);word(body,0x01ab6cee); - field(body,"RandomAppearanceMorph",1);word(body,1); + field(body,"RandomAppearanceMorph",3);word(body,1); for(uint32_t value:{id,0u,0u,0u}) word(body,value);body.push_back(0); + word(body,0);word(body,0); body.insert(body.end(),27,0);word(body,0);body.insert(body.end(),13,0); field(body,"Expressions",0);word(body,0x01ab6d32);field(body,"Stats",0); field(body,"Link",1);field(body,"Other",99); diff --git a/tools/audit_creature_appearance_layout.py b/tools/audit_creature_appearance_layout.py index ac09294f..da07a02c 100644 --- a/tools/audit_creature_appearance_layout.py +++ b/tools/audit_creature_appearance_layout.py @@ -10,6 +10,7 @@ import json from pathlib import Path import struct +import subprocess from audit_creature_body_models import body_slots from test_fmp_links import crc0, read_bank @@ -156,6 +157,8 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--install', type=Path, required=True) parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--reader', type=Path, help='Optional built C++ audit executable; run only in a coordinated test window') + parser.add_argument('--schema', type=Path, default=Path(__file__).resolve().parents[1]/'docs/re_reference/def_schema.json') args = parser.parse_args() native = args.install.resolve(strict=True) if args.output.resolve().is_relative_to(native): @@ -191,12 +194,30 @@ def main(): counts['nonempty_global_texture_maps'] += bool(layout['global_texture_map']['rows']) counts['nonzero_texture_group_count'] += bool(layout['texture_group_count']) rows.append(dict(index=index, name=record['name'], **layout)) + core = None + if args.reader: + binary = args.reader.resolve(strict=True) + binary_hash = hashlib.sha256(binary.read_bytes()).hexdigest() + result = subprocess.run([str(binary), '--creature-appearance-audit', str(native), + str(args.schema.resolve(strict=True))], capture_output=True, text=True, timeout=120) + if result.returncode: + raise AssertionError(result.stderr) + actual = json.loads(result.stdout)['records'] + expected = [{k:v for k,v in row.items() if k != 'legacy_refusal'} for row in rows] + if actual != expected: + mismatch = next(((a,e) for a,e in zip(actual,expected) if a != e), None) + raise AssertionError(('C++/independent appearance mismatch', len(actual), len(expected), mismatch)) + if hashlib.sha256(binary.read_bytes()).hexdigest() != binary_hash: + raise AssertionError('C++ audit binary changed') + core = dict(reader=str(binary), sha256=binary_hash, compared=len(actual), exact_match=True, + schema_sha256=hashlib.sha256(args.schema.read_bytes()).hexdigest()) if any(hashlib.sha256(p.read_bytes()).hexdigest() != hashes[str(p)] for p in inputs): raise AssertionError('input banks changed') report = dict(input_hashes=hashes, input_hashes_unchanged=True, producer_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), boundary_checks=checks, counts=dict(counts), records=rows, refused=refused, + core_comparison=core, scope='Read-only serialized appearance layout; selection/rendering unverified.') args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2)+'\n', encoding='utf-8') diff --git a/tools/test_creature_body_stale_ui.py b/tools/test_creature_body_stale_ui.py index 5b3873e1..ec19612d 100644 --- a/tools/test_creature_body_stale_ui.py +++ b/tools/test_creature_body_stale_ui.py @@ -25,8 +25,8 @@ def fixture(root, repo, model_id=10): def word(value):return struct.pack(' Date: Fri, 23 Oct 2026 09:55:47 -0600 Subject: [PATCH 65/75] Select Creature appearance with native-checked seed and morph ordering --- docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md | 108 ++++ docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md | 4 + docs/HANDOFF_NAV.md | 3 + docs/ROADMAP_1.0.md | 5 + .../include/forge/creatureappearance.hpp | 19 + libs/forgecore/src/creatureappearance.cpp | 78 +++ tests/test_defrelocate.cpp | 24 +- tests/test_fmplinks.cpp | 47 ++ tools/audit_creature_body_selection.py | 467 ++++++++++++++++++ 9 files changed, 754 insertions(+), 1 deletion(-) create mode 100644 docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md create mode 100644 tools/audit_creature_body_selection.py diff --git a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md new file mode 100644 index 00000000..bbae4c8a --- /dev/null +++ b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md @@ -0,0 +1,108 @@ +# Creature selection and assembly continuation + +The checked core appearance reader now has a pure deterministic selection API, +`forge::creatureappearance::select`. All 2,640 corpus selections match execution +of the original native selection routines. It is not connected to scene/Models +rendering yet. Whole-creature assembly, animation and live compatibility remain +open. The retained checkpoint ZIP is unchanged. + +Checkout/branch remain the definition-clone GUI lane. The preceding reader change +is aca1c11; [CI 39978691290](https://github.com/BuffJesus/FableForge/actions/runs/39978691290) +passed build, unit tests, WARP GUI smoke and CLI usage. Package/upload were skipped. +Earlier writer/GUI/checkpoint acceptance and original input identities are in +[the profiles handoff](HANDOFF_CREATURE_PROFILES_2026-10-23.md). + +## Selected data and boundaries + +`select(prefix, storedSeed, decapitated)` returns chosen body/alternative indices, +mesh IDs, ordered texture replacements, skeletal name offsets/amplitudes, global +texture replacements and the final seed. Input/seed are not mutated. Asset/name +resolution, rendering and animation are separate responsibilities. + +The order follows original FableWin routines, not independent per-part seeds: + +1. Skeletal groups in their stored clean key order. Key zero considers each row: + AllowVariation skips the coin, otherwise a draw of 50 or greater includes it. + Nonzero keys select a candidate only when the coin is greater than 50. + Selected varying rows consume a unit-range float draw. +2. Draw the body texture color group when the group count is nonzero. +3. For each nonempty body group, draw one alternative, then its texture choices, + then that body group's texture choices. Decapitation omits group zero only + after consuming all its normal draws. +4. Select global texture replacements with color group zero. + +Texture rows are grouped by original texture ID. Rows with GroupID zero or the +chosen color group are eligible; one eligible row is selected per source key. +The API preserves model/body/global replacement order rather than guessing a +renderer flattening rule. A texture key with no compatible row throws before a +selection escapes. Dirty-map tie sorting and unordered clean maps also refuse; +no unverified sort order or partial fallback is presented as a native result. + +GFRandom advances uint32 state by multiply 0x24a1, add 0x24df, rotate-right 13, +then modulo the bound. GFFloatRandom(1) advances the same state and yields exactly +`float(seed & 0xffff) / 65536`, including a possible zero amplitude. + +## Original instruction oracle and source acceptance + +`tools/audit_creature_body_selection.py` pins the original FableWin SHA256 and +executes bounded original x86 in Unicorn. `--full-morph-set` includes native +GetRandomMorphSet (0289ac90), skeletal selection (0289ad40), skeletal addition +(0289b010), body selection (0289b110), texture selection (0289b300), integer RNG +(018cf370), float RNG (018bac90) and their actual jump thunks. It retains explicit +bridges for containers/strings/output capture and a unit-range x87 remainder +bridge for the external CRT call. This is neither native allocation/ABI nor +engine process/rendering acceptance. Bridge addresses and original byte hashes +are recorded for review; clean maps are the supported oracle fixture contract. + +Successive evidence stages under `build/creature-assembly-native-20261023` keep +the distinction between body-only execution with a texture helper bridge, +body+native texture execution, and full native selection. Do not use an earlier +bridged stage to claim coverage of a later native routine. + +Final receipt: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/cs-6zq12yea/selection-native-core.json`. +All 330 CCreatureDef records, four seeds and both head states pass 2,640 native +versus independent Python comparisons. The compiled C++ selector matches those +2,640 complete results exactly: final seeds, parts, ordered textures, skeletal +amplitudes and global textures. Original executable/game.bin/names.bin hashes +remain unchanged. The receipt pins C++ executable/schema and producer hashes. + +An additional 56 original-x86 synthetic cases cover empty layouts, key-zero and +nonzero skeletal groups, first-coin values 49/50/51 (seeds 32/43/114), texture +layering, color choices and head removal. Core unit assertions pin native vectors +for these threshold/seed/amplitude behaviors and unsupported-table refusals. +The first key-zero probe exposed a missing neutral iterator-increment bridge; +the bridge was added, and the complete final edge matrix passed. + +Both scoped executables (`fmplinks`, `defrelocate`) pass. Builds are logged in +`build/creature-appearance-selection-{build,final-build}-20261023.log`. +Combined source hashes/receipt: `build/creature-selection-source-acceptance-20261023.json`. +This adds an unused pure selector; the accepted reader/import GUI gates were not +repeated. Shipping GUI and checkpoint acceptance still refer to their pinned +prior binaries, not an assembled-NPC preview. + +The short test window used a fresh process guard/exclusive RuntimeLease. All +owned tests ended and the lease released (`cs-6zq12yea/released-runtime-lease.json`); +Native availability queued `01a16974-9f79-7bd0-a3f6-a6578579e165`. No Forge gap is +reserved. Continue fresh claims for subsequent named test/GUI/game launches. + +## Next required integration + +Resolve CCreatureDef through the existing checked CDefs ownership graph and use +the runtime component list to distinguish random-morph from ordinary animated +appearance. Do not assume every Creature uses body substitution. Read an existing +TNG `CTCRandomAppearanceMorph.Seed` with its signed decimal uint32 bit pattern. +Absent Seed comes from world construction state, not a universal zero; an Assets +preview needs an explicit preview seed rather than a false native-parity claim. + +Trace how UpdateMorphSet/CEngineTextureMorphSet compose duplicate texture sources +before flattening ordered replacements. Recover the shared part transforms and +skeletal morph application from OnAppearanceDraw/engine primitives before joining +meshes in a scene. Preserve the tested pack asset context and distinguish a bind +pose preview from animation. Do not treat one imported triangle as a complete NPC. + +Then validate complete representative NPCs, missing-part/refusal behavior, texture +identity and transforms in owned scratch/leased GUI windows. Keep gameplay checks +separate. Dirty-map tie order remains an explicit unsupported selection case until +native sorting behavior is established; reader/writer byte preservation is not a +substitute for that selection contract. diff --git a/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md index c539d0be..cc9af04a 100644 --- a/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_PROFILES_2026-10-23.md @@ -435,3 +435,7 @@ the new shared reader has source acceptance, not extracted-package acceptance. Do not repeat the accepted profile/placement gates for this unrelated change. Next is native-checked selection/draw ordering and assembly transforms, not a claim that the parsed layout already renders a complete NPC. + +Selection continuation now lives in [Creature assembly](HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md): +the pure core selector matches 2,640 original-native corpus results, with 56 native +edge probes and two scoped units passing. Rendering/assembly is not connected yet. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index a4b9c1a7..ec9ffacf 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -2,6 +2,9 @@ [Creature authoring history](HANDOFF_CREATURE_GUI_2026-10-23.md) retains the earlier import and material evidence; older checkpoint ZIPs remain unchanged. +[Creature assembly continuation](HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md) records the +checked reader/selector source milestone and remaining presentation integration. + # Navigation lane, 2026-09-27 Checkout: `D:/Code/FableForge-nav`, branch `feat/nav-thing-lines`, starting commit diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 8ef1b63b..74b2e0e3 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -9,6 +9,11 @@ global texture map previously refused. Four affected units, five donor fixtures, the retained ZIP above is unchanged. Whole-creature selection and assembly are still open. Evidence is in the linked Creature placement profiles handoff. +The subsequent pure selector matches 2,640 original-native corpus selections; +56 native edge probes and two scoped units pass. See +[Creature assembly continuation](HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md). +It is not connected to scene rendering or animation yet. + Ordered dialogue-tag controls pass source checks at both window sizes and an independent retail-copy preservation/Restore check. Signed stored positions remain explicit; new tag names are bounded ASCII and original legacy names survive other diff --git a/libs/forgecore/include/forge/creatureappearance.hpp b/libs/forgecore/include/forge/creatureappearance.hpp index 1a22fff2..577df4a1 100644 --- a/libs/forgecore/include/forge/creatureappearance.hpp +++ b/libs/forgecore/include/forge/creatureappearance.hpp @@ -43,4 +43,23 @@ struct Prefix { // Name offsets still require validation against the owning bank's names pool. // This is a serialized layout, not a selected/assembled or animated creature. Prefix read(std::span bytes); + +struct TextureReplacement { uint32_t original = 0, replacement = 0; }; +struct BodyPart { + size_t body = 0, alternative = 0; + uint32_t mesh = 0; + std::vector textures; +}; +struct SkeletalValue { uint32_t nameOffset = 0; float amount = 1.0f; }; +struct Selection { + uint32_t seedAfter = 0; + std::vector parts; + std::vector skeletal; + std::vector globalTextures; +}; +// Deterministic native selection order from a supplied stored uint32 Seed. +// Does not infer a world seed, resolve names/assets, skin vertices or animate. +// Dirty/unsorted maps and unavailable color groups throw without mutating input. +// Dirty-map sorting still needs a native tie-order contract before support. +Selection select(const Prefix& prefix, uint32_t seed, bool decapitated = false); } // namespace forge::creatureappearance diff --git a/libs/forgecore/src/creatureappearance.cpp b/libs/forgecore/src/creatureappearance.cpp index 5358eba4..dcdd17d5 100644 --- a/libs/forgecore/src/creatureappearance.cpp +++ b/libs/forgecore/src/creatureappearance.cpp @@ -106,4 +106,82 @@ Prefix read(std::span bytes) { result.structuralWords = std::move(reader.structuralWords); return result; } + +Selection select(const Prefix& prefix, uint32_t seed, bool decapitated) { + const auto validate = [](const TextureMap& map) { + if (map.dirty) fail("dirty texture map sorting is not yet supported"); + for (size_t n=1;nmap.rows[n].original) fail("unordered clean texture map"); + }; + for (const auto& body:prefix.bodies) + for (const auto& model:body) validate(model.textures); + for (const auto& map:prefix.bodyTextures) validate(map); + validate(prefix.globalTextures); + for (size_t n=1;nprefix.skeletal[n].key) fail("unordered skeletal map"); + const auto random = [&](uint32_t bound) { + seed=std::rotr(uint32_t(seed*0x24a1u+0x24dfu),13); + return bound ? seed%bound : 0u; + }; + const auto textures = [&](const TextureMap& map, uint32_t color, + std::vector& output) { + for (size_t first=0;first=50)skeletal(prefix.skeletal[n]); + } else if(random(100)>50) { + const auto choice=random(uint32_t(end-first)); + skeletal(prefix.skeletal[first+choice]); + } + first=end; + } + const auto color=prefix.textureGroupCount ? random(prefix.textureGroupCount)+1 : 0; + for(size_t body=0;body divisor 65536 and uint32 dividend. This +# exact range needs one fprem (exponent difference <=16), not a general CRT stub. +REMAINDER = (0x035835b8, bytes.fromhex('dd44240c dd442404 d9f8 ddd9 c3')) + + +def textures(rows, color, seed): + """Recovered helper bridge; shared by the oracle and comparison model.""" + if any(a['original'] > b['original'] for a,b in zip(rows, rows[1:])): + raise ValueError('bridge requires clean, ordered texture maps') + result, missing = [], 0 + for key in dict.fromkeys(row['original'] for row in rows): + eligible = [row for row in rows if row['original']==key and row['group'] in (0,color)] + if not eligible: + # Native takes its diagnostic path and continues without an RNG draw. + missing += 1 + continue + seed, choice = recovered(seed, len(eligible)) + result.append([key, eligible[choice]['replacement']]) + return seed, result, missing + + +def expected(layout, seed, decapitated): + color = 0 + if layout['texture_group_count']: + seed, color = recovered(seed, layout['texture_group_count']) + color += 1 + parts, calls = [], [] + for body, alternatives in enumerate(layout['bodies']): + if not alternatives: + continue + seed, alternative = recovered(seed, len(alternatives)) + model = alternatives[alternative] + combined = [] + for kind, rows in [('model',model['textures']['rows']), + ('body',layout['body_texture_maps'][body]['rows'])]: + seed, selected, missing = textures(rows, color, seed) + combined += selected + calls.append([body,kind,color,selected,missing]) + if not (decapitated and body==0): + parts.append(dict(mesh=model['mesh'], textures=combined)) + return dict(seed=seed, parts=parts, calls=calls) + + +def expected_morphs(layout, seed, decapitated): + rows, skeletal = layout['skeletal_morphs'], [] + if any(a['key']>b['key'] for a,b in zip(rows,rows[1:])): + raise ValueError('skeletal bridge requires ordered keys') + for key in dict.fromkeys(row['key'] for row in rows): + candidates = [r for r in rows if r['key']==key] + selected = [] + if key==0: + for row in candidates: + if row['allow_variation']: + selected.append(row) + else: + seed, draw = recovered(seed,100) + if draw>=50: + selected.append(row) + else: + seed, draw = recovered(seed,100) + if draw>50: + seed, choice = recovered(seed,len(candidates)) + selected.append(candidates[choice]) + for row in selected: + amount = 1.0 + if row['allow_variation']: + seed, _ = recovered(seed,0) + amount = (seed & 0xffff)/65536.0 + skeletal.append(dict(name_offset=row['name_offset'],amount=amount)) + result = expected(layout,seed,decapitated) + seed, global_textures, missing = textures(layout['global_texture_map']['rows'],0,result['seed']) + result.update(seed=seed,skeletal=skeletal,global_textures=global_textures) + result['calls'].append([-1,'global',0,global_textures,missing]) + return result + + +def edge_layouts(): + empty = lambda: dict(rows=[],dirty=False) + base = dict(bodies=[[],[],[]],body_texture_maps=[empty() for _ in range(3)], + skeletal_morphs=[],global_texture_map=empty(),texture_group_count=0) + yield 'empty', copy.deepcopy(base) + filled = copy.deepcopy(base) + for body in (0,1,2): + for alternative in (0,1): + filled['bodies'][body].append(dict(mesh=10+10*body+alternative,priority=1, + textures=dict(dirty=False,rows=[dict(original=100,replacement=200+alternative,group=0), + dict(original=100,replacement=300+alternative,group=1), + dict(original=100,replacement=400+alternative,group=2)]))) + filled['body_texture_maps'][body] = dict(dirty=False,rows=[dict(original=100,replacement=500+body,group=0)]) + filled['global_texture_map'] = dict(dirty=False,rows=[dict(original=100,replacement=v,group=0) for v in (600,601)]) + filled['texture_group_count'] = 2 + zero = copy.deepcopy(filled) + zero['skeletal_morphs'] = [dict(key=0,group=0,name_offset=4,allow_variation=False), + dict(key=0,group=0,name_offset=8,allow_variation=True)] + yield 'zero-key', zero + grouped = copy.deepcopy(filled) + grouped['skeletal_morphs'] = [dict(key=1,group=1,name_offset=4,allow_variation=False), + dict(key=1,group=1,name_offset=8,allow_variation=True), + dict(key=2,group=2,name_offset=12,allow_variation=True)] + yield 'nonzero-keys', grouped + yield 'layered-global', filled + + +class BodyOracle: + stack, seed_at, stop = 0x10010000, 0x10020000, 0x10030000 + obj, output = 0x20000000, 0x20000100 + + def __init__(self, pe, native_textures=False, full_morph_set=False): + self.machine = Uc(UC_ARCH_X86, UC_MODE_32) + self.pe = pe + self.native_textures = native_textures + self.full_morph_set = full_morph_set + self.ranges = RANGES + (TEXTURE_RANGES if native_textures else []) + (MORPH_RANGES if full_morph_set else []) + self.stubs = STUBS | (ITERATORS if native_textures else {}) | (MORPH_STUBS if full_morph_set else {}) + pages = {address & ~0xfff for address, _ in self.ranges} | {a & ~0xfff for a in self.stubs} + if full_morph_set: + pages |= {REMAINDER[0] & ~0xfff,0x04023000} + for address in pages: + self.machine.mem_map(address, 0x1000) + for address, length in self.ranges: + self.machine.mem_write(address, pe.get_data(address-pe.OPTIONAL_HEADER.ImageBase, length)) + for address in self.stubs: + if native_textures and address==0x01801ebb: + continue + self.machine.mem_write(address, b'\xc3') + if full_morph_set: + self.machine.mem_write(*REMAINDER) + for address in (0x04023800,0x04023810): + self.machine.mem_write(address,pe.get_data(address-pe.OPTIONAL_HEADER.ImageBase,8)) + for address in (0, self.stack-0x1000, self.seed_at, self.stop): + self.machine.mem_map(address, 0x1000) + self.machine.mem_map(self.obj, 0x400000) + self.machine.hook_add(UC_HOOK_CODE, self.bridge) + + def word(self, address): + return struct.unpack('=0x80000000: + key -= 0x100000000 + rows = self.tables[this][2] + if kind=='key_count': + result, pop = sum(r[field]==key for r in rows),4 + else: + index = next((i for i,r in enumerate(rows) if + (r[field]>key if kind=='upper_bound' else r[field]>=key)),len(rows)) + self.iterators[arg(1)] = (this,index) + result, pop = arg(1), 8 + elif kind=='texture_pair': + self.store(this,arg(1),arg(2)) + pop = 8 + elif kind=='texture_append': + source = arg(1) + self.local[this].append([self.word(source),self.word(source+4)]) + pop = 4 + elif kind=='keys_construct': + pass + elif kind=='keys': + assert this==self.obj+0x28 + destination = arg(1) + keys = list(dict.fromkeys(row['key'] for row in self.tables[this][2])) + self.raw_rows[destination] = [] + for i,key in enumerate(keys): + pointer = self.obj+0x300000+4*i + self.store(pointer,key & 0xffffffff) + self.raw_rows[destination].append(pointer) + pop = 4 + elif kind=='iter_offset': + table, index = self.iterators[this] + self.iterators[arg(1)] = (table,index+arg(2)) + result, pop = arg(1),8 + elif kind=='skeletal_construct': + self.store(this,0,0,0) + elif kind=='def_string': + self.store(arg(1),self.word(this),0) + result, pop = arg(1),4 + elif kind=='string_copy': + machine.mem_write(this,bytes(machine.mem_read(arg(1),8))) + pop = 4 + elif kind=='skeletal_append': + assert this==self.output+0x14 + pointer = arg(1) + amount, = struct.unpack('b['original'] for a,b in zip(rows,rows[1:])): + raise ValueError('container bridge requires clean, ordered maps') + for row in rows: + self.store(cursor,row['original'],row['replacement'],row['group']) + self.raw_rows[table].append(cursor) + cursor += 12 + if self.full_morph_set: + self.tables[self.obj+0x28] = (-1,'skeletal',layout['skeletal_morphs']) + self.raw_rows[self.obj+0x28] = [] + for row in layout['skeletal_morphs']: + self.store(cursor,row['key'] & 0xffffffff,row['group'] & 0xffffffff, + row['name_offset'],int(row['allow_variation'])) + self.raw_rows[self.obj+0x28].append(cursor) + cursor += 16 + self.tables[self.obj+0x40] = (-1,'global',layout['global_texture_map']['rows']) + self.raw_rows[self.obj+0x40] = [] + for row in layout['global_texture_map']['rows']: + self.store(cursor,row['original'],row['replacement'],row['group']) + self.raw_rows[self.obj+0x40].append(cursor) + cursor += 12 + self.local[self.output+0x28] = [] + self.store(0,0) + self.store(self.obj+0x58,layout['texture_group_count']) + self.store(self.seed_at,seed) + self.store(self.stack-16,self.stop,self.seed_at,self.output,int(decapitated)) + self.machine.reg_write(UC_X86_REG_ESP,self.stack-16) + self.machine.reg_write(UC_X86_REG_ECX,self.obj) + self.machine.reg_write(UC_X86_REG_EDX,0) + self.machine.emu_start(0x0289ac90 if self.full_morph_set else BODY,self.stop,timeout=500000,count=100000) + if (self.machine.reg_read(UC_X86_REG_ESP)!=self.stack or + self.machine.reg_read(UC_X86_REG_EIP)!=self.stop): + raise AssertionError('original body routine did not return within bound') + result = dict(seed=self.word(self.seed_at),parts=self.parts,calls=self.calls) + if self.full_morph_set: + result.update(skeletal=self.skeletal,global_textures=self.local[self.output+0x28]) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--exe',type=Path,required=True) + parser.add_argument('--install',type=Path,required=True) + parser.add_argument('--output',type=Path,required=True) + parser.add_argument('--native-textures',action='store_true',help='Execute original texture selector with neutral container bridges') + parser.add_argument('--full-morph-set',action='store_true',help='Also execute original skeletal/global selection and unit-range float draws') + parser.add_argument('--reader',type=Path,help='Optional C++ selector audit executable; requires a coordinated test window') + parser.add_argument('--schema',type=Path,default=Path(__file__).resolve().parents[1]/'docs/re_reference/def_schema.json') + args = parser.parse_args() + raw = args.exe.read_bytes() + if hashlib.sha256(raw).hexdigest()!=EXPECTED_SHA: + raise ValueError('unsupported executable identity') + native = args.install.resolve(strict=True) + if args.output.resolve().is_relative_to(native) or args.output.resolve()==args.exe.resolve(): + raise ValueError('evidence must be outside native inputs') + files = [native/'data/CompiledDefs'/name for name in ('game.bin','names.bin')] + hashes = {str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in files} + names = names_pool(files[1].read_bytes()) + if args.full_morph_set: + args.native_textures = True + if args.reader and not args.full_morph_set: + raise ValueError('C++ comparison requires --full-morph-set') + machine = BodyOracle(pefile.PE(data=raw),args.native_textures,args.full_morph_set) + counts, rows = Counter(), [] + seeds = [0,1,0x12345678,0xffffffff] + for index, record in enumerate(read_bank(native)): + if record['definition']!='CCreatureDef': + continue + layout = decode(record['data'], names) + for seed in seeds: + for decapitated in (False,True): + actual = machine.run(layout,seed,decapitated) + prediction = (expected_morphs if args.full_morph_set else expected)(layout,seed,decapitated) + if actual!=prediction: + raise AssertionError((index,seed,decapitated,actual,prediction)) + counts['cases'] += 1 + counts['texture_bridge_calls'] += len(actual['calls']) + counts['missing_texture_groups'] += sum(c[4] for c in actual['calls']) + rows.append(dict(index=index,seed=seed,decapitated=decapitated, + seed_after=actual['seed'],meshes=[p['mesh'] for p in actual['parts']], + selection={k:v for k,v in actual.items() if k!='calls'})) + counts['components'] += 1 + edge_cases = [] + if args.full_morph_set: + edge_seeds = seeds + [next(seed for seed in range(10000) if recovered(seed,100)[1]==value) + for value in (49,50,51)] + for name,layout in edge_layouts(): + for seed in dict.fromkeys(edge_seeds): + for decapitated in (False,True): + actual = machine.run(layout,seed,decapitated) + prediction = expected_morphs(layout,seed,decapitated) + if actual!=prediction: + raise AssertionError(('edge',name,seed,decapitated,actual,prediction)) + edge_cases.append(dict(profile=name,seed=seed,decapitated=decapitated, + selection={k:v for k,v in actual.items() if k!='calls'})) + counts['edge_cases'] = len(edge_cases) + core = None + if args.reader: + binary = args.reader.resolve(strict=True) + digest = hashlib.sha256(binary.read_bytes()).hexdigest() + result = subprocess.run([str(binary),'--creature-appearance-selection-audit',str(native), + str(args.schema.resolve(strict=True))],capture_output=True,text=True,timeout=120) + if result.returncode: + raise AssertionError(result.stderr) + actual = json.loads(result.stdout)['records'] + wanted = [{k:v for k,v in row.items() if k not in ('seed_after','meshes')} for row in rows] + if actual!=wanted: + mismatch = next(((a,e) for a,e in zip(actual,wanted) if a!=e),None) + raise AssertionError(('native/C++ selection mismatch',len(actual),len(wanted),mismatch)) + if hashlib.sha256(binary.read_bytes()).hexdigest()!=digest: + raise AssertionError('C++ reader changed during audit') + core = dict(binary=str(binary),sha256=digest,compared=len(actual),exact_match=True, + schema_sha256=hashlib.sha256(args.schema.read_bytes()).hexdigest()) + if args.exe.read_bytes()!=raw or any(hashlib.sha256(p.read_bytes()).hexdigest()!=hashes[str(p)] for p in files): + raise AssertionError('native input changed') + report = dict(passed=True,counts=dict(counts),input_hashes=hashes,exe_sha256=EXPECTED_SHA, + producer_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + original_ranges=[dict(address=f'{a:08x}',length=n, + sha256=hashlib.sha256(machine.pe.get_data(a-machine.pe.OPTIONAL_HEADER.ImageBase,n)).hexdigest()) for a,n in machine.ranges], + bridge_addresses={f'{a:08x}':name for a,name in machine.stubs.items() + if not(args.native_textures and name=='textures')},cases=rows,edge_cases=edge_cases, + native_textures=args.native_textures, + full_morph_set=args.full_morph_set, + core_comparison=core, + remainder_bridge=REMAINDER[1].hex() if args.full_morph_set else None, + scope=('Original body/texture control flow and RNG; neutral clean-container bridges only.' if args.native_textures else + 'Original body control flow and RNG; container/texture helpers bridged.')+ + (' Original skeletal/global selection plus a unit-range x87 remainder bridge.' if args.full_morph_set else + ' No skeletal selection acceptance.')+ + ' No dirty-map sort, native allocation/ABI or rendering acceptance.') + args.output.parent.mkdir(parents=True,exist_ok=True) + args.output.write_text(json.dumps(report,indent=2)+'\n',encoding='utf-8') + print(json.dumps(dict(passed=True,counts=dict(counts),output=str(args.output)))) + + +if __name__=='__main__': + main() From 5e7a68cf841932b590b02928800dcac9d288b457 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:11:20 -0600 Subject: [PATCH 66/75] Compose Creature texture replacements by native source keys --- docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md | 53 +++++++++++++++++++ .../include/forge/creatureappearance.hpp | 8 +++ libs/forgecore/src/creatureappearance.cpp | 20 +++++++ tests/test_fmplinks.cpp | 22 ++++++++ 4 files changed, 103 insertions(+) diff --git a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md index bbae4c8a..31f3c9ce 100644 --- a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md @@ -88,6 +88,59 @@ reserved. Continue fresh claims for subsequent named test/GUI/game launches. ## Next required integration +### Texture composition source continuation + +`composeTextures(part, global)` now produces one type-3 replacement per original +texture ID, retaining first-source insertion order and the last replacement for +that source. It does not chase replacement IDs or mix parts. An empty part map +composes the globals for the ordinary base-mesh path. The selector's ordered +output remains intact; composition is a separate, currently unused API. + +Original FableWin `UpdateMorphSet` (0252dbd0) adds global texture entries after +each selected part's entries; with no substituted parts it adds globals to the +base mesh morph set. `FindPEngineMorph` (01e5f4d0) compares the incoming original +ID to the first word of each existing engine morph. `AddTextureMorph` (01e5f6d0) +finds or creates that source entry, clears its blend list when the incoming blend +type is 3, then appends the descriptor. The consequential branch is confirmed at +01e5f743 (`cmp [eax+4],3`), 01e5f747 (skip clear for other types), 01e5f74f (clear) +and 01e5f75e (append). This helper only represents these selected type-3 lists; +it does not implement the general compositor's other blend types. + +Read-only exports and PE-byte comparisons are under +`build/creature-assembly-native-20261023`: `composition-*`, `lookup-*`, individual +address `.c` files, `texture-composition-disassembly.txt`, and +`byte-verification.json` (37 verified function ranges). Executable identity is +unchanged from the selector oracle above. This is static instruction evidence, +not a newly executed native composition oracle or rendering acceptance. + +Material follow-up: both `GetDiffuseTexture` (02e73b40) and `GetBumpTexture` +(02e810f0) consult `GetComposedTexture` (02ee4d10) using their original material +texture ID. That routine performs one source-key map lookup. For a single blend +entry it obtains `GetBaseTextureIndex` (02ee4f00), which returns the first blend +descriptor's texture ID, and the caller loads that texture. `IsMultipleTextures` +(02ee4ec0) tests whether the blend list has more than one entry. There is no +recursive lookup of the returned replacement in this path. Therefore do not +restrict the composition rule to diffuse IDs when adding material integration; +the bump path uses it too. Other texture channels remain untraced. These exports +are in `materials-*` and `single-texture-*` in the same evidence directory. + +The affected unit target builds (`build/creature-texture-composition-build-20261023.log`). +Its added assertions distinguish duplicate-source override, identity override, +cyclic destinations, part isolation and the no-parts global path. The first launch attempt +refused active Native CPU tests before claiming a lease. After a clear guard, +the scoped executable passed in a fresh bounded lease. Receipt: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/ct-m6nzxgb3/result.json`; +executable SHA256 `63f65b3912576a57ed47d2ac6c2fa40508f52d1beb8e81f506c1a1c29b9de9a3`. +The owned process closed and the lease released; availability queued +01a16987-9cc3-7351-8979-62d4326cef22. No Forge reservation remains. + +The morphed animating-mesh constructor (01e0a8f0) forwards position/orientation, +animation and scale inputs to the standard animating-mesh constructor (01e0aa00), +which copies the interpolators. There is no added part-local transform in this +wrapper. Confirm the shared caller arguments and skeletal application before +using this observation to combine geometry. Material-channel lookup, skeletal +deformation, renderer integration and representative complete previews remain open. + Resolve CCreatureDef through the existing checked CDefs ownership graph and use the runtime component list to distinguish random-morph from ordinary animated appearance. Do not assume every Creature uses body substitution. Read an existing diff --git a/libs/forgecore/include/forge/creatureappearance.hpp b/libs/forgecore/include/forge/creatureappearance.hpp index 577df4a1..a4346384 100644 --- a/libs/forgecore/include/forge/creatureappearance.hpp +++ b/libs/forgecore/include/forge/creatureappearance.hpp @@ -62,4 +62,12 @@ struct Selection { // Dirty/unsorted maps and unavailable color groups throw without mutating input. // Dirty-map sorting still needs a native tie-order contract before support. Selection select(const Prefix& prefix, uint32_t seed, bool decapitated = false); +// Compose selected type-3 replacements for one part, followed by globals. +// Last entry for each original ID wins; replacement IDs are never chased. +// Output retains first-source insertion order and does not mutate either input. +// For an appearance with no substituted parts, pass an empty part map to obtain +// the global replacements for the ordinary base mesh. +std::vector composeTextures( + std::span part, + std::span global); } // namespace forge::creatureappearance diff --git a/libs/forgecore/src/creatureappearance.cpp b/libs/forgecore/src/creatureappearance.cpp index dcdd17d5..55d7dff9 100644 --- a/libs/forgecore/src/creatureappearance.cpp +++ b/libs/forgecore/src/creatureappearance.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace forge::creatureappearance { @@ -184,4 +185,23 @@ Selection select(const Prefix& prefix, uint32_t seed, bool decapitated) { result.seedAfter=seed; return result; } +std::vector composeTextures( + std::span part, + std::span global) { + std::vector result; + std::unordered_map sources; + const auto append = [&](std::span layer) { + for (const auto& replacement : layer) { + const auto [entry, inserted] = sources.emplace(replacement.original, result.size()); + if (inserted) result.push_back(replacement); + else result[entry->second].replacement = replacement.replacement; + } + }; + // FableWin UpdateMorphSet (0252dbd0), FindPEngineMorph (01e5f4d0) + // and AddTextureMorph (01e5f6d0): key by original ID, clear the blend + // list on type 3, append the replacement. Global entries are added last. + append(part); + append(global); + return result; +} } // namespace forge::creatureappearance diff --git a/tests/test_fmplinks.cpp b/tests/test_fmplinks.cpp index 3d3ae6e6..dd92115f 100644 --- a/tests/test_fmplinks.cpp +++ b/tests/test_fmplinks.cpp @@ -91,6 +91,28 @@ int main() { const auto empty=select(Prefix{},seed,true); check(empty.seedAfter==seed && empty.parts.empty() && empty.skeletal.empty() && empty.globalTextures.empty(), "empty appearance consumes no draws"); + const auto composed=composeTextures(selected.parts[0].textures,selected.globalTextures); + check(composed.size()==1 && composed[0].original==100 && composed[0].replacement==600 && + selected.parts[0].textures.size()==2, + "global type-3 replacement supersedes model/body layers without mutating selection"); + const std::vector layers{{10,20},{20,30},{10,40},{40,10}}; + const std::vector globals{{20,50},{60,60},{10,10}}; + const auto cycle=composeTextures(layers,{}); + check(cycle.size()==3 && cycle[0].replacement==40 && cycle[1].replacement==30 && + cycle[2].replacement==10, + "replacement destinations are not chased through other source keys or cycles"); + const auto keyed=composeTextures(layers,globals); + check(keyed.size()==4 && keyed[0].original==10 && keyed[0].replacement==10 && + keyed[1].original==20 && keyed[1].replacement==50 && + keyed[2].original==40 && keyed[2].replacement==10 && + keyed[3].original==60 && keyed[3].replacement==60, + "source keys retain insertion order, identity overrides and independent cyclic destinations"); + const auto otherPart=composeTextures(std::vector{{10,99}},{}); + const auto baseOnly=composeTextures({},globals); + check(otherPart.size()==1 && otherPart[0].replacement==99 && + baseOnly.size()==3 && baseOnly[0].original==20 && baseOnly[2].replacement==10 && + composeTextures({},{}).empty(), + "part maps stay isolated and empty-part base appearance receives globals"); auto invalid=plan;invalid.globalTextures.dirty=true; rejects([&]{select(invalid,seed);},"unverified dirty-map tie order refuses"); invalid=plan;invalid.globalTextures.rows={{0,200,600,0},{0,100,601,0}}; From 9a58982df76e7614a1401bdd7c52e584cee2924c Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:22:28 -0600 Subject: [PATCH 67/75] Share read-only Creature visual ownership resolution --- docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md | 51 +++++++++++ libs/forgecore/include/forge/defedit.hpp | 10 +++ libs/forgecore/src/defedit.cpp | 35 +++++--- tests/test_creaturevisualclone.cpp | 30 +++++++ tools/audit_creature_visual_resolution.py | 92 ++++++++++++++++++++ 5 files changed, 206 insertions(+), 12 deletions(-) create mode 100644 tools/audit_creature_visual_resolution.py diff --git a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md index 31f3c9ce..09c76f5f 100644 --- a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md @@ -12,6 +12,9 @@ passed build, unit tests, WARP GUI smoke and CLI usage. Package/upload were skip Earlier writer/GUI/checkpoint acceptance and original input identities are in [the profiles handoff](HANDOFF_CREATURE_PROFILES_2026-10-23.md). +Selector CI 39982494520 subsequently passed build, units, WARP GUI smoke and +CLI usage. Package/upload were skipped. This does not add rendering acceptance. + ## Selected data and boundaries `select(prefix, storedSeed, decapitated)` returns chosen body/alternative indices, @@ -88,6 +91,35 @@ reserved. Continue fresh claims for subsequent named test/GUI/game launches. ## Next required integration +### Shared ownership resolver in progress + +`defedit::resolveCreatureVisuals` extracts the existing clone graph checks into +a read-only API: root/body/appearance indices, selected CDefs row numbers and +component count. The clone now consumes that proof. Original-parent and correctly +typed unique-row requirements stay intact. Component payload cleanliness, +runtime profile, selection, assets and relocation validity remain separate. + +The affected target builds (`build/creature-visual-resolver-build-20261023.log`). +Direct unit assertions cover shared ownership, private-clone resolution and +malformed roots/indices/owners. An unrelated untyped relocation candidate may +pass graph resolution while still refusing clone, preserving that distinction. +The unit and new `--visual-graph-audit` mode are pending execution. Texture +finished its checkpoint, then Native completed/released its agreed three-capture +gap (queue 01a1698d-aee9-7032-8623-a959046b538b). The immediate guard still found +neighboring TLC regular-frame-timing/particle CPU tests, so Forge claimed no lease +and launched nothing. Native was told to drop its courtesy hold (queue +01a1698e-9622-73f1-93cb-fcbc7f87857b); no Forge reservation remains. Those CPU tests +belong neighboring TLC work, not Native. Retry only a clear, fresh guarded window. + +`tools/audit_creature_visual_resolution.py` independently reads the serialized CDefs +triples and original-parent rows. Its offline run finds 502 accepted and 15 +refused roots among all 517 retail Creatures, with input hashes unchanged: +`build/creature-visual-graph-independent-20261023.json`. The refused set contains +the null root, incomplete templates and cooperative spirit definitions; it must +not be turned into a guessed complete NPC. Optional `--reader` compares every +result with the compiled resolver and requires an already guarded launch window. +The independent model checks graph structure; C++ also checks reflected decoding. + ### Texture composition source continuation `composeTextures(part, global)` now produces one type-3 replacement per original @@ -141,6 +173,25 @@ wrapper. Confirm the shared caller arguments and skeletal application before using this observation to combine geometry. Material-channel lookup, skeletal deformation, renderer integration and representative complete previews remain open. +### Skeletal application trace + +Further read-only exports (`skeletal-*` in the native evidence directory) add +three original-byte-matched functions, bringing that verification list to 40. +`AddSkeletalMorph` (026625e0) stores a group/name/value entry and marks the cached +morph dirty. `GetPSkeletalMorph` (02662c80) recalculates only while dirty, clears +the flag and returns the cached counted pointer. `CalculateSkeletalMorph` +(02662960) resets the combined object, resolves each stored name through +`CSkeletalMorphResourceManager::GetMorphTarget`, constructs a parameter for that +target, adds it, sets its current value, then notifies parameter updates. The +random-appearance caller clears/adds only group `RAND` (0x52414e44), leaving +other producers relevant to the final shape. + +This confirms that selected name/amplitude pairs need resolved target resources; +they are not an extra shared object scale. Target loading, combination and bone +application still need evidence and implementation. The next narrow native entry +points are GetMorphTarget (023e36e0), PreloadBoneConfigs (023e3f10), and +C3DSkeletalMorphTarget::CombineMorph (0321ce20). No rendering parity is claimed. + Resolve CCreatureDef through the existing checked CDefs ownership graph and use the runtime component list to distinguish random-morph from ordinary animated appearance. Do not assume every Creature uses body substitution. Read an existing diff --git a/libs/forgecore/include/forge/defedit.hpp b/libs/forgecore/include/forge/defedit.hpp index 973d83ee..4199169e 100644 --- a/libs/forgecore/include/forge/defedit.hpp +++ b/libs/forgecore/include/forge/defedit.hpp @@ -96,6 +96,16 @@ size_t cloneCreature(bin::File& file, const defschema::Schema& schema, struct CreatureVisualCloneResult { size_t rootIndex, bodyIndex, appearanceIndex; }; +struct CreatureVisualGraph { + size_t rootIndex, bodyIndex, appearanceIndex; + size_t componentCount, bodyRow, appearanceRow; +}; +// Resolve the two visual CDefs rows without mutation. Requires a clean CREATURE +// root, exactly one correctly typed row per visual component, and an original +// parent whose corresponding row points to the same component. This validates +// graph ownership, not component payloads, runtime profile or asset availability. +CreatureVisualGraph resolveCreatureVisuals(const bin::File& file, + const defschema::Schema& schema, size_t rootIndex); // Append a private root/CCreatureDef/CAppearanceDef graph in memory. Explicit // names must be unique printable ASCII (127-byte maximum) and introduce no // compiled-name CRC ambiguity. Only the two selected visual CDefs rows receive diff --git a/libs/forgecore/src/defedit.cpp b/libs/forgecore/src/defedit.cpp index 7d36edef..082a0733 100644 --- a/libs/forgecore/src/defedit.cpp +++ b/libs/forgecore/src/defedit.cpp @@ -425,29 +425,26 @@ size_t cloneEntry(bin::File& file,std::string_view donorName,std::string newName return file.cloneEntry(selected,std::move(newName)); } -CreatureVisualCloneResult cloneCreatureVisuals(bin::File& file,const defschema::Schema& schema, - size_t donorIndex,std::string rootName,std::string bodyName,std::string appearanceName) { +CreatureVisualGraph resolveCreatureVisuals(const bin::File& file, + const defschema::Schema& schema,size_t donorIndex) { const auto& entries=file.entries(); - // CSubDefInfo persists signed-long definition/original-parent indices. - if(entries.size()>size_t(INT32_MAX)-2u) - throw std::runtime_error("private creature clone: append indices exceed the signed reference domain"); auto read=[](const auto& data,size_t offset) { if(offset>data.size() || data.size()-offset<4) - throw std::runtime_error("private creature clone: truncated component word"); + throw std::runtime_error("creature visual graph: truncated component word"); return uint32_t(data[offset]) | uint32_t(data[offset+1])<<8 | uint32_t(data[offset+2])<<16 | uint32_t(data[offset+3])<<24; }; auto rootLayout=[&](size_t index) { if(index>=entries.size() || entries[index].definition!="CREATURE") - throw std::runtime_error("private creature clone: donor/original parent must be a Creature index"); + throw std::runtime_error("creature visual graph: donor/original parent must be a Creature index"); const auto& entry=entries[index];const auto decoded=defdecode::decode(entry,schema); if(!decoded.clean() || defdecode::encode(decoded)!=entry.data || entry.data.size()<5 || decoded.fields.empty() || decoded.fields.front().name!="Components" || std::any_of(decoded.fields.begin(),decoded.fields.end(),[](const auto& field){return !field.tagOk;})) - throw std::runtime_error("private creature clone: unverified Creature component layout"); + throw std::runtime_error("creature visual graph: unverified Creature component layout"); const size_t count=size_t(entry.data[3]) | size_t(entry.data[4])<<8; if(!count || decoded.prefixLen!=5+12*count) - throw std::runtime_error("private creature clone: empty or malformed Creature listing"); + throw std::runtime_error("creature visual graph: empty or malformed Creature listing"); return count; }; const auto count=rootLayout(donorIndex); @@ -462,17 +459,31 @@ CreatureVisualCloneResult cloneCreatureVisuals(bin::File& file,const defschema:: ++matches;rows[k]=row;donors[k+1]=read(root.data,9+12*row); } if(matches!=1 || donors[k+1]>=entries.size() || entries[donors[k+1]].definition!=kinds[k]) - throw std::runtime_error("private creature clone: requires one correctly typed "+std::string(kinds[k])+" row"); + throw std::runtime_error("creature visual graph: requires one correctly typed "+std::string(kinds[k])+" row"); const auto owner=read(root.data,13+12*rows[k]); const auto ownerCount=rootLayout(owner);size_t ownerMatches=0; for(size_t row=0;rowsize_t(INT32_MAX)-2u) + throw std::runtime_error("private creature clone: append indices exceed the signed reference domain"); + const auto graph=resolveCreatureVisuals(file,schema,donorIndex); + const auto count=graph.componentCount; + const auto& root=entries[donorIndex]; + const std::array rows{graph.bodyRow,graph.appearanceRow}; + const std::array donors{graph.rootIndex,graph.bodyIndex,graph.appearanceIndex}; std::array names{std::move(rootName),std::move(bodyName),std::move(appearanceName)}; std::array keys{}; for(size_t i=0;i{"","1","BAD\nNAME",std::string(128,'A'),std::string("BAD\xff",4)}) @@ -140,6 +163,10 @@ int main(int argc,char** argv)try{ auto bad=source;auto data=root(true); if(mutation.first==3)data[3]=uint8_t(mutation.second);else put(data,mutation.first,mutation.second); bad.setEntryData(1,data);refuses(bad,clone); + if(mutation.first!=61) + refuses(bad,[&](auto& candidate){defedit::resolveCreatureVisuals(candidate,schema,1);}); + else require(defedit::resolveCreatureVisuals(bad,schema,1).bodyIndex==3, + "graph resolution must not claim relocation validation of unrelated fields"); } auto bad=source;auto data=root(false);put(data,9,6);bad.setEntryData(0,data);refuses(bad,clone); bad=source;data=body();put(data,15,1);bad.setEntryData(3,data);refuses(bad,clone); @@ -149,6 +176,9 @@ int main(int argc,char** argv)try{ const auto original=source.entries();auto working=source;const auto result=clone(working); require(result.rootIndex==7&&result.bodyIndex==8&&result.appearanceIndex==9,"private append slots differ"); + const auto privateGraph=defedit::resolveCreatureVisuals(working,schema,result.rootIndex); + require(privateGraph.bodyIndex==result.bodyIndex&&privateGraph.appearanceIndex==result.appearanceIndex, + "private clone ownership resolves through shared graph checks"); auto rootExpected=root(true);put(rootExpected,9,8);put(rootExpected,13,7);put(rootExpected,21,9);put(rootExpected,25,7);put(rootExpected,53,7); auto bodyExpected=body();put(bodyExpected,7,7);auto appearanceExpected=appearance();put(appearanceExpected,7,7); require(working.entries()[7].data==rootExpected,"private root ownership/delta differs"); diff --git a/tools/audit_creature_visual_resolution.py b/tools/audit_creature_visual_resolution.py new file mode 100644 index 00000000..ab9dfe73 --- /dev/null +++ b/tools/audit_creature_visual_resolution.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Read-only Creature CDefs ownership audit; optional compiled resolver comparison. + +The independent model checks serialized triples, types and original parents. +The C++ resolver additionally requires clean reflected decoding. Invoke --reader +only inside an already guarded runtime lease, as for other named test tools. +""" +import argparse +import hashlib +import json +from pathlib import Path +import struct +import subprocess + +from test_fmp_links import crc0, read_bank + + +def resolve(entries, index): + def table(slot): + if not 0 <= slot < len(entries) or entries[slot]['definition'] != 'CREATURE': + raise ValueError('root/owner is not a Creature') + data = entries[slot]['data'] + if len(data) < 5: + raise ValueError('truncated root') + count, = struct.unpack_from('= len(entries) or entries[target]['definition'] != kind: + raise ValueError('visual target has wrong type/index') + ownership = [r for r in table(owner) if r[0] == crc0(kind)] + if len(ownership) != 1 or ownership[0][1] != target: + raise ValueError('original parent does not identify the same visual') + result[label], result[label + '_row'] = target, row + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--install', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--reader', type=Path) + parser.add_argument('--schema', type=Path) + args = parser.parse_args() + if args.reader and not args.schema: + parser.error('--reader requires --schema') + inputs = [args.install/'data/CompiledDefs'/name for name in ['game.bin', 'names.bin']] + inputs += [p for p in [args.reader, args.schema] if p] + if args.output.resolve() in [p.resolve() for p in inputs]: + raise ValueError('output must differ from every input') + digest = lambda path: hashlib.sha256(path.read_bytes()).hexdigest() + hashes = {str(p.resolve()): digest(p) for p in inputs} + entries, records = read_bank(args.install), [] + for index, entry in enumerate(entries): + if entry['definition'] != 'CREATURE': + continue + try: + records.append(resolve(entries, index)) + except ValueError: + records.append(dict(root=index, refused=True)) + if args.reader: + child = subprocess.run([str(args.reader.resolve()), '--visual-graph-audit', + str(args.install.resolve()), str(args.schema.resolve())], + capture_output=True, text=True, check=True, timeout=60) + actual = json.loads(child.stdout)['records'] + actual = [dict(root=r['root'], refused=True) if 'refused' in r else r for r in actual] + if actual != records: + raise AssertionError('compiled and independent visual graphs differ') + if hashes != {str(p.resolve()): digest(p) for p in inputs}: + raise AssertionError('an audit input changed') + report = dict(records=records, input_hashes=hashes, roots=len(records), + accepted=sum('refused' not in r for r in records), + refused=sum('refused' in r for r in records), compiled_comparison=bool(args.reader), + inputs_unchanged=True, scope='CDefs graph ownership only; not runtime selection/rendering') + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2)+'\n', encoding='utf-8') + print(f"Visual graphs: {report['accepted']} accepted, {report['refused']} refused; {args.output}") + + +if __name__ == '__main__': + main() From 776c4f6d9f1d42fe8fb653d9d280e84adee08a41 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:26:12 -0600 Subject: [PATCH 68/75] Record Creature graph acceptance and skeletal resource evidence --- docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md | 33 +++++++++++++++----- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md index 09c76f5f..ba63e72e 100644 --- a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md @@ -103,13 +103,16 @@ The affected target builds (`build/creature-visual-resolver-build-20261023.log`) Direct unit assertions cover shared ownership, private-clone resolution and malformed roots/indices/owners. An unrelated untyped relocation candidate may pass graph resolution while still refusing clone, preserving that distinction. -The unit and new `--visual-graph-audit` mode are pending execution. Texture -finished its checkpoint, then Native completed/released its agreed three-capture -gap (queue 01a1698d-aee9-7032-8623-a959046b538b). The immediate guard still found -neighboring TLC regular-frame-timing/particle CPU tests, so Forge claimed no lease -and launched nothing. Native was told to drop its courtesy hold (queue -01a1698e-9622-73f1-93cb-fcbc7f87857b); no Forge reservation remains. Those CPU tests -belong neighboring TLC work, not Native. Retry only a clear, fresh guarded window. +After the neighboring CPU tests closed, the scoped unit passed and the compiled +`--visual-graph-audit` matched the independent reader on all 517 roots: 502 accepted +and 15 refused. All source bank hashes stayed unchanged. Receipt: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/vg-g9shta90/result.json`; +executable SHA256 `1e0da7013b333f8b020be529103836a14199a8da414f650cc12e80f52ec5be44`. +`build/creature-visual-resolver-acceptance-20261023.json` pins source/schema hashes. +The fresh bounded lease released after both owned calls closed; availability +queued 01a16995-29f8-7b52-a87a-6db3ce8736ab. No Forge reservation remains. Earlier +refusals occurred before claim; guard hits belonged neighboring TLC CPU work, +not Native. Native's agreed three-capture gap was honored through explicit release. `tools/audit_creature_visual_resolution.py` independently reads the serialized CDefs triples and original-parent rows. Its offline run finds 502 accepted and 15 @@ -118,6 +121,8 @@ refused roots among all 517 retail Creatures, with input hashes unchanged: the null root, incomplete templates and cooperative spirit definitions; it must not be turned into a guessed complete NPC. Optional `--reader` compares every result with the compiled resolver and requires an already guarded launch window. +The original `audit_creature_visual_graph.py` observation workflow is preserved; +its synthetic checks pass. The independent model checks graph structure; C++ also checks reflected decoding. ### Texture composition source continuation @@ -210,3 +215,17 @@ identity and transforms in owned scratch/leased GUI windows. Keep gameplay check separate. Dirty-map tie order remains an explicit unsupported selection case until native sorting behavior is established; reader/writer byte preservation is not a substitute for that selection contract. + +Resource follow-up exports (`skeletal-resources-*`, `bone-config-loader-*`) bring +the original-byte verification list to 44 functions. GetMorphTarget resolves a +name under the bone-config directory and caches the target; PreloadBoneConfigs +enumerates `*.bncfg`. The actual target loader (0287d630) skips to the +`#Start_Bone_data` section, reads bone names and three float components, and +initializes the target from those rows. Earlier creature/group-setting text is +not read as bone rows. Parser token/error behavior and scale combination still +need instruction checks before a new core reader or deformation implementation. + +A read-only census of all 75 stored skeletal references finds 15 unique resource +names, all present under retail `data/Bones`; their file hashes are retained in +`build/creature-assembly-native-20261023/bone-resource-census.json`. This validates +resource availability, not parsing, pose or complete rendering. From e73afb8ecf29ad91e1f73822f65a8e3b254c60a9 Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:31:06 -0600 Subject: [PATCH 69/75] Record native bone scale combination and config corpus shape --- docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md | 31 ++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md index ba63e72e..a455d2e1 100644 --- a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md @@ -229,3 +229,34 @@ A read-only census of all 75 stored skeletal references finds 15 unique resource names, all present under retail `data/Bones`; their file hashes are retained in `build/creature-assembly-native-20261023/bone-resource-census.json`. This validates resource availability, not parsing, pose or complete rendering. + +## Bone scale continuation + +Read-only `bone-combine-*` and `bone-keys-*` exports bring the original-byte +verification list to 51 functions. The vector methods invoked by CombineMorph +are explicit subtraction (019072b0), scalar multiplication (018c0af0) and +in-place addition (01ded8a0). Caller disassembly at 0321cf35 loads the incoming +bone's scale at +0xc, subtracts identity, multiplies by the supplied amount and +adds it to the accumulated bone scale. A newly inserted bone starts at identity. +The recovered rule is `accumulated += (incoming - 1) * amount`, componentwise; +it is not multiplication of the incoming scale vectors. Float storage/rounding +and complete original-machine combination probes remain to be checked. + +Target Initialise (0321ccc0) excludes rows whose three components are all exactly +one, then sorts. Bone construction (0321cba0) stores CCharString::GetCRC at +8; +CompareCRCs (0321cc20) compares that field as unsigned. GetCRC (0186d2a0) passes +the raw string byte length/data and initial CRC zero to CCRC::Calc. This is a +name-keyed target, not a direct local skeleton index. Ties/collisions and actual +bone application are not yet supported by a new implementation. + +`bone-config-shape-census.json` independently checks the 15 referenced retail +resources: 254 named XYZ rows, six exact identity rows, finite float32 values, +and no repeated bone CRC within a target. Each has the observed CRLF start marker +and semicolon-separated row shape. Hashes/source order remain recorded. This is +an offline grammar census, not execution of CStringParser or a core reader. + +Next: implement a bounded read-only reader for this verified subset, retain its +source names/rows separately from prepared target rows, refuse duplicate CRCs +until native tie behavior is known, compare all 15 resources with independent +readback, then verify weighted combination and skeleton application. Do not +present undeformed joined parts as the selected complete Creature. From 7b5ac4b058828aabf4464b5d01c06302cf08f81f Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:35:43 -0600 Subject: [PATCH 70/75] Read checked Creature bone configs and prepare native target rows --- CMakeLists.txt | 3 + docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md | 40 +++++++-- docs/ROADMAP_1.0.md | 5 ++ libs/forgecore/include/forge/boneconfig.hpp | 29 ++++++ libs/forgecore/src/boneconfig.cpp | 94 ++++++++++++++++++++ tests/test_boneconfig.cpp | 52 +++++++++++ 6 files changed, 218 insertions(+), 5 deletions(-) create mode 100644 libs/forgecore/include/forge/boneconfig.hpp create mode 100644 libs/forgecore/src/boneconfig.cpp create mode 100644 tests/test_boneconfig.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 18d3e879..a9b0effb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,6 +99,9 @@ add_test(NAME fableforge_defdecode_tests COMMAND fableforge_defdecode_tests) add_executable(fableforge_fmplinks_tests tests/test_fmplinks.cpp) target_link_libraries(fableforge_fmplinks_tests PRIVATE forgecore_slice) add_test(NAME fableforge_fmplinks_tests COMMAND fableforge_fmplinks_tests) +add_executable(fableforge_boneconfig_tests tests/test_boneconfig.cpp) +target_link_libraries(fableforge_boneconfig_tests PRIVATE forgecore_slice) +add_test(NAME fableforge_boneconfig_tests COMMAND fableforge_boneconfig_tests) add_executable(fableforge_defrelocate_tests tests/test_defrelocate.cpp) target_link_libraries(fableforge_defrelocate_tests PRIVATE forgecore_slice) add_test(NAME fableforge_defrelocate_tests COMMAND fableforge_defrelocate_tests) diff --git a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md index a455d2e1..8347c12f 100644 --- a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md @@ -255,8 +255,38 @@ and no repeated bone CRC within a target. Each has the observed CRLF start marke and semicolon-separated row shape. Hashes/source order remain recorded. This is an offline grammar census, not execution of CStringParser or a core reader. -Next: implement a bounded read-only reader for this verified subset, retain its -source names/rows separately from prepared target rows, refuse duplicate CRCs -until native tie behavior is known, compare all 15 resources with independent -readback, then verify weighted combination and skeleton application. Do not -present undeformed joined parts as the selected complete Creature. +## Bone-config reader source acceptance + +`forge::boneconfig::read` now reads the verified CRLF subset into source-ordered +names, CRCs, float32 XYZ scales and byte offsets. It locates exact start/end bone +markers and leaves unknown header/group-setting text and the input untouched. +The parser accepts one semicolon-terminated row per line, bounds input to 1 MiB, +rows to 4096 and printable ASCII bone names to 255 bytes, and refuses malformed +component counts, nonfinite/out-of-range decimals, NUL input and duplicate CRCs. +It does not claim general CStringParser grammar compatibility. + +`prepare` separately validates the names/CRCs/scales, excludes exact identity +rows, and sorts by unsigned CRC. Source rows, including identities and their +offsets, stay intact. Duplicate keys/collisions refuse until native tie behavior +is established. This is an unused read-only core API, not skeleton deformation. + +The new `fableforge_boneconfig_tests` target builds and passes, including malformed +rows, identity pruning, source preservation, unsigned ordering and CRC collision +refusals. All 15 referenced retail files match independent readback: 254 source +rows and 248 prepared target rows, exact offsets, names, CRCs and float32 bits. +Every resource hash stays unchanged. The acceptance packet is +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/bc-nubgz_lw/result.json`; +binary SHA256 `2e7817e06cc172fa5c377978f3f3acb70ce4525c15a6775d6c7d42e09e4b79fc`. +Source hashes/receipt: `build/bone-config-reader-acceptance-20261023.json`. +Build log: `build/bone-config-reader-build-20261023.log`. The test's read-only +`--read file` mode emits both row sets for further independent comparisons. + +These checks exercised a fresh bounded guarded lease; all owned calls closed and +the lease released. Availability queued 01a1699d-5f81-7813-b2a9-0d948a0b80af; +receipt `bc-nubgz_lw/released-runtime-lease.json`. No Forge reservation remains. +No shipping GUI or package was rebuilt for this unused core reader. + +Next: verify weighted combination with original instructions and implement the +mapping/application to mesh skeleton bones. Continue scene/Assets integration +through the shared ownership resolver and runtime-profile checks. Do not present +undeformed joined parts as the selected complete Creature. diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 74b2e0e3..51c466a4 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -14,6 +14,11 @@ The subsequent pure selector matches 2,640 original-native corpus selections; [Creature assembly continuation](HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md). It is not connected to scene rendering or animation yet. +The read-only bone-config reader now matches all 15 resources referenced by the +stored skeletal choices: 254 source rows and 248 prepared rows, with source hashes +unchanged. Its scoped unit passes. Weighted combination, skeleton application and +complete Creature rendering remain open in the assembly continuation above. + Ordered dialogue-tag controls pass source checks at both window sizes and an independent retail-copy preservation/Restore check. Signed stored positions remain explicit; new tag names are bounded ASCII and original legacy names survive other diff --git a/libs/forgecore/include/forge/boneconfig.hpp b/libs/forgecore/include/forge/boneconfig.hpp new file mode 100644 index 00000000..ad9f0c8a --- /dev/null +++ b/libs/forgecore/include/forge/boneconfig.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace forge::boneconfig { +struct Bone { + size_t offset = 0; + std::string name; + uint32_t crc = 0; + std::array scale{1,1,1}; +}; +struct Config { + size_t bodyBegin = 0, bodyEnd = 0; + std::vector rows; // source order, including identity rows +}; +// Read the verified CRLF .bncfg subset: one named XYZ scale per line between +// exact bone-data markers. Unknown header/group text and input remain untouched. +// Bounded to 1 MiB, 4096 rows, 255-byte printable ASCII names, finite decimals. +// This is not the general native CStringParser grammar or skeleton deformation. +Config read(std::string_view text); +// Native target preparation: remove exact identity scales, sort by unsigned CRC. +// Duplicate CRCs (including collisions) refuse; native tie behavior is unverified. +std::vector prepare(const Config& config); +} // namespace forge::boneconfig diff --git a/libs/forgecore/src/boneconfig.cpp b/libs/forgecore/src/boneconfig.cpp new file mode 100644 index 00000000..aad9dee3 --- /dev/null +++ b/libs/forgecore/src/boneconfig.cpp @@ -0,0 +1,94 @@ +#include "forge/boneconfig.hpp" +#include "forge/bin.hpp" +#include +#include +#include +#include +#include + +namespace forge::boneconfig { +namespace { +[[noreturn]] void fail(const char* reason) { + throw std::runtime_error(std::string("bone config: ")+reason); +} +std::string_view trim(std::string_view value) { + const auto first=value.find_first_not_of(" \t"); + if(first==value.npos)return {}; + return value.substr(first,value.find_last_not_of(" \t")-first+1); +} +float decimal(std::string_view value) { + value=trim(value); + if(value.empty())fail("missing scale component"); + if(value.front()=='+') { + value.remove_prefix(1); + if(!value.empty() && (value.front()=='+' || value.front()=='-'))fail("invalid scale sign"); + } + if(value.empty() || value.find_first_not_of("0123456789.eE+-")!=value.npos) + fail("expected finite decimal scale"); + float result=0; + const auto parsed=std::from_chars(value.data(),value.data()+value.size(),result); + if(parsed.ec!=std::errc{} || parsed.ptr!=value.data()+value.size() || !std::isfinite(result)) + fail("invalid or out-of-range scale"); + return result; +} +void validate(const Bone& bone) { + if(bone.name.empty() || bone.name.size()>255 || trim(bone.name)!=bone.name || + std::any_of(bone.name.begin(),bone.name.end(),[](unsigned char ch) { + return ch<32 || ch>126 || ch==':' || ch==';'; + }))fail("unsupported bone name"); + if(bone.crc!=bin::nameCrc(bone.name))fail("bone name/CRC mismatch"); + if(!std::all_of(bone.scale.begin(),bone.scale.end(),[](float v){return std::isfinite(v);})) + fail("nonfinite scale"); +} +} +Config read(std::string_view text) { + constexpr std::string_view start="#Start_Bone_data\r\n", end="#End_bone_data\r\n"; + if(text.size()>1024*1024 || text.find('\0')!=text.npos)fail("oversized or NUL-containing input"); + const auto begin=text.find(start); + if(begin==text.npos || (begin && text[begin-1]!='\n') || text.find(start,begin+start.size())!=text.npos) + fail("missing, embedded or repeated start marker"); + const auto finish=text.find(end,begin+start.size()); + if(finish==text.npos || text.find(end,finish+end.size())!=text.npos) + fail("missing or repeated end marker"); + Config result{begin+start.size(),finish,{}}; + std::unordered_set keys; + for(size_t at=result.bodyBegin;atfinish)fail("expected CRLF bone row"); + const auto line=text.substr(at,next-at); + if(!trim(line).empty()) { + if(result.rows.size()>=4096)fail("too many bone rows"); + const auto colon=line.find(':'); + if(colon==line.npos)fail("missing bone separator"); + Bone bone;bone.offset=at;bone.name=std::string(line.substr(0,colon)); + bone.crc=bin::nameCrc(bone.name); + auto values=trim(line.substr(colon+1)); + if(values.empty() || values.back()!=';')fail("missing row terminator"); + values.remove_suffix(1); + for(size_t axis=0;axis<3;++axis) { + const auto comma=values.find(','); + if((axis<2)==(comma==values.npos))fail("expected exactly three scale components"); + bone.scale[axis]=decimal(values.substr(0,comma)); + if(axis<2)values.remove_prefix(comma+1); + } + validate(bone); + if(!keys.insert(bone.crc).second)fail("duplicate bone CRC is not supported"); + result.rows.push_back(std::move(bone)); + } + at=next+2; + } + return result; +} +std::vector prepare(const Config& config) { + if(config.rows.size()>4096)fail("too many bone rows"); + std::unordered_set keys; + std::vector result; + for(const auto& bone:config.rows) { + validate(bone); + if(!keys.insert(bone.crc).second)fail("duplicate bone CRC is not supported"); + if(bone.scale!=std::array{1,1,1})result.push_back(bone); + } + std::sort(result.begin(),result.end(),[](const Bone& a,const Bone& b){return a.crc +#include +#include +#include + +static void require(bool value,const char* message) {if(!value)throw std::runtime_error(message);} +template static void refuses(F action) { + try {action();}catch(const std::runtime_error&) {return;} + throw std::runtime_error("malformed bone config accepted"); +} +static nlohmann::json rows(const std::vector& bones) { + auto result=nlohmann::json::array(); + for(const auto& bone:bones)result.push_back({{"offset",bone.offset},{"name",bone.name}, + {"crc",bone.crc},{"scale",bone.scale}}); + return result; +} +int main(int argc,char** argv)try { + using namespace forge::boneconfig; + if(argc==3&&std::string_view(argv[1])=="--read") { + std::ifstream in(argv[2],std::ios::binary);require(bool(in),"cannot open input"); + const std::string text(std::istreambuf_iterator(in),{}); + const auto config=read(text); + std::cout<{1.1f,.75f,-.2f}&&target.size()==1&& + target[0].crc==forge::bin::nameCrc("Bip01 R Foot")&&config.rows.size()==2, + "source rows/offsets or native identity pruning differ"); + require(read(start+end).rows.empty(),"empty bone section differs"); + const auto ordered=prepare(read(start+"Z: 2,2,2;\r\nA: 3,3,3;\r\n"+end)); + require(ordered.size()==2&&ordered[0].crc{"",start,start+"X: 1,2,3;\n"+end, + start+"X: 1,2;\r\n"+end,start+"X: 1,2,3,4;\r\n"+end, + start+"X: 1,2,3\r\n"+end,start+"X: nan,2,3;\r\n"+end, + start+"X: 1e99,2,3;\r\n"+end,start+"X: +-1,2,3;\r\n"+end, + start+" X: 1,2,3;\r\n"+end,start+"X: 1,2,3;\r\nX: 2,2,2;\r\n"+end, + start+"plumless: 2,2,2;\r\nbuckeroo: 3,3,3;\r\n"+end, + start+end+start+end,std::string(1024*1024+1,'x'),start+"X"+std::string(1,'\0')+": 1,2,3;\r\n"+end}) + refuses([&]{read(bad);}); + auto damaged=config;damaged.rows[0].crc^=1;refuses([&]{prepare(damaged);}); + damaged=config;damaged.rows.push_back(config.rows[1]);refuses([&]{prepare(damaged);}); + std::cout<<"Bone config: PASS\n";return 0; +} catch(const std::exception& error) {std::cerr< Date: Fri, 23 Oct 2026 10:40:53 -0600 Subject: [PATCH 71/75] Verify native bone scale arithmetic and mesh key mapping --- docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md | 38 +++++++ tools/audit_bone_scale_math.py | 101 +++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 tools/audit_bone_scale_math.py diff --git a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md index 8347c12f..e9768d46 100644 --- a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md @@ -290,3 +290,41 @@ Next: verify weighted combination with original instructions and implement the mapping/application to mesh skeleton bones. Continue scene/Assets integration through the shared ownership resolver and runtime-profile checks. Do not present undeformed joined parts as the selected complete Creature. + +## Positive scale arithmetic and mesh mapping evidence + +`tools/audit_bone_scale_math.py` now executes the original vector subtraction, +scalar multiplication, in-place addition, constructor and constructor thunk in +bounded Unicorn x86. It uses no container, parameter or arithmetic bridges. All +3,048 comparisons pass: the 254 referenced source rows, four weights (0, float32 +0.1, 0.5, 1) and three accumulated XYZ scales. Output float32 bits match the +recovered expression with separate float32 storage after subtraction, +multiplication and addition. This verifies an arithmetic update, not complete +CombineMorph container execution, multiple-parameter control flow or skinning. + +Receipt: `C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/bm-frz06g7y/result.json`. +That directory retains the census and producer; the report pins producer/input +hashes and all original instruction ranges. The native executable and census +stay unchanged. Original code executes in an emulator inside the offline Python +tool; no frontend/game/test executable was launched and no runtime lease claimed. + +Read-only `bone-application-*` and `bone-mapping-*` exports bring the original-byte +verification list to 56 functions. PeekCombinedMorph (0321c960) visits stored +parameters in order, using the positive target for nonnegative current values +and the negative target with negated magnitude for negative values. The present +random-appearance selector only supplies nonnegative values; do not generalize +its target handling to other morph producers. + +InitialiseBoneScalings (02e79430) obtains the combined target, sizes an array to +the mesh's bone count, then walks local bone slots and resolves each mesh bone's +name CRC through GetBoneScaling (0321d210). That lookup uses binary search of the +sorted target keys and writes identity XYZ when a key is absent. GetBoneScalings +(02d7b8a0) caches this array by mesh ID; without a skeletal morph it returns null. +This confirms mapping by bone name CRC, not by target row number or global +animation-track ID. Actual use in skeleton transforms/vertex skinning remains +unverified. + +Next: implement positive selected-target combination with explicit float32 +rounding and stable parameter order, validate C++ against the arithmetic oracle, +map combined values into decoded mesh bone slots, then trace and implement their +application to bind/animation transforms. Whole-Creature presentation remains open. diff --git a/tools/audit_bone_scale_math.py b/tools/audit_bone_scale_math.py new file mode 100644 index 00000000..e374eace --- /dev/null +++ b/tools/audit_bone_scale_math.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Bounded original-x86 vector arithmetic used by positive skeletal combination. + +No container/parameter/animation bridges: runs subtraction, multiplication, +=, +their constructor and its original thunk. This is arithmetic evidence only, not +execution of the complete CombineMorph routine or skeleton deformation. +""" +import argparse +import hashlib +import json +from pathlib import Path +import struct + +import pefile +from unicorn import Uc, UC_ARCH_X86, UC_MODE_32 +from unicorn.x86_const import UC_X86_REG_EAX, UC_X86_REG_ECX, UC_X86_REG_EIP, UC_X86_REG_ESP +from audit_creature_appearance_rng import EXPECTED_SHA + + +def f32(value): + return struct.unpack(' Date: Fri, 23 Oct 2026 10:45:49 -0600 Subject: [PATCH 72/75] Combine selected bone scales and map mesh-local CRC slots --- docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md | 46 +++++++++++++++-- libs/forgecore/include/forge/boneconfig.hpp | 12 +++++ libs/forgecore/src/boneconfig.cpp | 53 ++++++++++++++++++++ tests/test_boneconfig.cpp | 46 +++++++++++++++++ tools/audit_bone_scale_math.py | 30 ++++++++++- 5 files changed, 182 insertions(+), 5 deletions(-) diff --git a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md index e9768d46..246067f2 100644 --- a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md @@ -324,7 +324,45 @@ This confirms mapping by bone name CRC, not by target row number or global animation-track ID. Actual use in skeleton transforms/vertex skinning remains unverified. -Next: implement positive selected-target combination with explicit float32 -rounding and stable parameter order, validate C++ against the arithmetic oracle, -map combined values into decoded mesh bone slots, then trace and implement their -application to bind/animation transforms. Whole-Creature presentation remains open. +## Positive combination and slot mapping source acceptance + +`boneconfig::combine` now combines prepared targets in parameter order with +explicit float32 stores after subtraction, multiplication and addition. It +supports selected random-appearance amplitudes in [0,1], retains first-source +name/offset provenance and emits CRC-sorted combined rows. Zero weights retain +identity entries, as the native insertion path does. Negative-target selection +is outside this API. Nonfinite/unsupported amplitudes, unsorted/duplicate input, +cross-target CRC/name collisions and overflow refuse without mutating inputs. +Work is bounded to 4096 targets and 65536 total source rows. + +`mapScales` resolves the combined rows into supplied mesh-local CRC order, with +identity for missing keys. Repeated mesh slots with the same CRC resolve to the +same scale; target row number/global track ID is never used as a local slot. +It does not apply scales to matrices or vertices yet. + +The affected unit target builds and passes. Cases distinguish additive deltas +from multiplying scales, preserve non-associative float32 parameter order, retain +zero-weight identities, map missing/repeated mesh keys and refuse malformed +targets/collisions/overflow. The extended original-x86 oracle compares the C++ +helper on all 3,048 native arithmetic cases, matching output bits exactly. +The C++ mode emits only computed bits from source/base/weight inputs; native +expected results are not fed to it. + +Receipt: `C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/bs-afuxk45_/native-core.json`. +`result.json` in that directory pins the source hashes; binary SHA256 +`4be6ee7f524440c9a2c047af21bd5c87a5fe0aa2bfd3de98da6144a7c840ab72`. +Build log: `build/bone-config-combine-build-20261023.log`; source receipt: +`build/bone-config-combine-acceptance-20261023.json`. This is positive arithmetic +and slot-mapping acceptance, not complete native container/parameter execution, +negative morph handling, mesh transform/animation or rendering acceptance. + +The fresh bounded lease released after owned checks closed; receipt: +`bs-afuxk45_/released-runtime-lease.json`. Claim notification queued +01a169a6-3cc4-7e61-963e-6871c0522522. Release notification failed twice because +the peer queue reached its 100-submission limit; the actual lease is absent, +and no Forge runtime or reservation remains. Do not retry-spam or mutate the +queue database. The release is also reported in the user-facing turn. + +Next: trace and implement scale application to bind/animation transforms, wire +selection/combination and decoded mesh bones through the shared Context/preview +path, then qualify representative complete Creatures. Whole presentation remains open. diff --git a/libs/forgecore/include/forge/boneconfig.hpp b/libs/forgecore/include/forge/boneconfig.hpp index ad9f0c8a..093a61d8 100644 --- a/libs/forgecore/include/forge/boneconfig.hpp +++ b/libs/forgecore/include/forge/boneconfig.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -26,4 +27,15 @@ Config read(std::string_view text); // Native target preparation: remove exact identity scales, sort by unsigned CRC. // Duplicate CRCs (including collisions) refuse; native tie behavior is unverified. std::vector prepare(const Config& config); +struct WeightedTarget { std::span rows; float amount = 1; }; +// Combine prepared targets in supplied parameter order. Supports the nonnegative +// random-appearance range [0,1]; negative-target selection remains separate. +// Each arithmetic step rounds to float32, matching the native vector methods. +// Output is CRC-sorted; names/offsets come from the first contributing target. +// Cross-target CRC/name collisions, malformed targets and nonfinite results refuse. +std::vector combine(std::span targets); +// Resolve combined scales into mesh-local bone order by name CRC. Missing keys +// produce identity, including when the combined target is empty. No skinning. +std::vector> mapScales(std::span combined, + std::span meshBoneCrcs); } // namespace forge::boneconfig diff --git a/libs/forgecore/src/boneconfig.cpp b/libs/forgecore/src/boneconfig.cpp index aad9dee3..521ef4df 100644 --- a/libs/forgecore/src/boneconfig.cpp +++ b/libs/forgecore/src/boneconfig.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -91,4 +92,56 @@ std::vector prepare(const Config& config) { std::sort(result.begin(),result.end(),[](const Bone& a,const Bone& b){return a.crc rows) { + if(rows.size()>65536)fail("too many combined bones"); + for(size_t n=0;n=rows[n].crc)fail("expected unique CRC-sorted target rows"); + } +} +} +std::vector combine(std::span targets) { + if(targets.size()>4096)fail("too many weighted targets"); + size_t count=0; + std::map accumulated; + for(const auto& target:targets) { + if(!std::isfinite(target.amount) || target.amount<0 || target.amount>1) + fail("unsupported selected-target amplitude"); + validateSorted(target.rows); + if(target.rows.size()>65536-count)fail("too many source bones"); + count+=target.rows.size(); + for(const auto& bone:target.rows) { + auto [entry,inserted]=accumulated.try_emplace(bone.crc,bone); + if(inserted)entry->second.scale={1,1,1}; + else if(entry->second.name!=bone.name)fail("cross-target bone CRC collision"); + for(size_t axis=0;axis<3;++axis) { + // Each original vector method stores its result before the next + // call. Volatile float stores also prevent multiply-add fusion. + volatile float delta=bone.scale[axis]-1.0f; + volatile float weighted=delta*target.amount; + volatile float result=entry->second.scale[axis]+weighted; + if(!std::isfinite(result))fail("combined scale overflow"); + entry->second.scale[axis]=result; + } + } + } + std::vector result; + result.reserve(accumulated.size()); + for(auto& entry:accumulated)result.push_back(std::move(entry.second)); + return result; +} +std::vector> mapScales(std::span combined, + std::span meshBoneCrcs) { + validateSorted(combined); + if(meshBoneCrcs.size()>65536)fail("too many mesh bones"); + std::vector> result; + result.reserve(meshBoneCrcs.size()); + for(const auto crc:meshBoneCrcs) { + const auto found=std::lower_bound(combined.begin(),combined.end(),crc, + [](const Bone& bone,uint32_t key){return bone.crccrc==crc ? found->scale : std::array{1,1,1}); + } + return result; +} } // namespace forge::boneconfig diff --git a/tests/test_boneconfig.cpp b/tests/test_boneconfig.cpp index f3cbd2a3..1e4e85f2 100644 --- a/tests/test_boneconfig.cpp +++ b/tests/test_boneconfig.cpp @@ -1,7 +1,9 @@ #include "forge/boneconfig.hpp" #include "forge/bin.hpp" +#include #include #include +#include #include #include @@ -18,6 +20,19 @@ static nlohmann::json rows(const std::vector& bones) { } int main(int argc,char** argv)try { using namespace forge::boneconfig; + if(argc==3&&std::string_view(argv[1])=="--scale-math-audit") { + std::ifstream in(argv[2]);require(bool(in),"cannot open arithmetic vectors"); + const auto cases=nlohmann::json::parse(in);auto output=nlohmann::json::array(); + for(const auto& item:cases) { + Bone base{0,"Bone",forge::bin::nameCrc("Bone"),item.at("base").get>()}; + Bone source=base;source.scale=item.at("source").get>(); + const std::array targets{{{{&base,1},1},{{&source,1},item.at("weight").get()}}}; + const auto combined=combine(targets); + output.push_back(std::array{std::bit_cast(combined[0].scale[0]), + std::bit_cast(combined[0].scale[1]),std::bit_cast(combined[0].scale[2])}); + } + std::cout<(in),{}); @@ -48,5 +63,36 @@ int main(int argc,char** argv)try { refuses([&]{read(bad);}); auto damaged=config;damaged.rows[0].crc^=1;refuses([&]{prepare(damaged);}); damaged=config;damaged.rows.push_back(config.rows[1]);refuses([&]{prepare(damaged);}); + const auto one=prepare(read(start+"Bone: 2,3,4;\r\n"+end)); + const auto two=prepare(read(start+"Bone: 3,2,.5;\r\n"+end)); + const std::array layers{{{one,.5f},{two,.25f}}}; + const auto combined=combine(layers); + require(combined.size()==1&&combined[0].scale==std::array{2,2.25f,2.375f}, + "weighted targets must add identity-relative deltas in parameter order"); + const std::array mesh{forge::bin::nameCrc("Missing"),one[0].crc,one[0].crc}; + const auto mapped=mapScales(combined,mesh); + require(mapped.size()==3&&mapped[0]==std::array{1,1,1}&&mapped[1]==combined[0].scale&& + mapped[2]==mapped[1]&&mapScales({},mesh)[1]==mapped[0], + "mesh-local slots map by CRC with identity for missing keys"); + const std::array zero{{{one,0}}}; + require(combine(zero)[0].scale==std::array{1,1,1},"zero weight retains identity bone entry"); + const std::array negative{{{one,-.1f}}};refuses([&]{combine(negative);}); + auto unsorted=one;unsorted.push_back(one[0]); + const std::array duplicate{{{unsorted,1}}};refuses([&]{combine(duplicate);}); + refuses([&]{mapScales(unsorted,mesh);}); + Bone collisionA{0,"plumless",forge::bin::nameCrc("plumless"),{2,2,2}}; + Bone collisionB{0,"buckeroo",forge::bin::nameCrc("buckeroo"),{3,3,3}}; + const std::array collisions{{{{&collisionA,1},1},{{&collisionB,1},1}}}; + refuses([&]{combine(collisions);}); + Bone big{0,"Bone",one[0].crc,{16777216,16777216,16777216}}; + Bone minus=big;minus.scale={-16777216,-16777216,-16777216}; + Bone small=big;small.scale={2,2,2}; + const std::array orderA{{{{&big,1},1},{{&minus,1},1},{{&small,1},1}}}; + const std::array orderB{{{{&big,1},1},{{&small,1},1},{{&minus,1},1}}}; + require(combine(orderA)[0].scale[0]==1&&combine(orderB)[0].scale[0]==0, + "float32 accumulation must preserve parameter order"); + Bone huge=big;huge.scale.fill(std::numeric_limits::max()); + const std::array overflow{{{{&huge,1},1},{{&huge,1},1}}}; + refuses([&]{combine(overflow);}); std::cout<<"Bone config: PASS\n";return 0; } catch(const std::exception& error) {std::cerr< Date: Fri, 23 Oct 2026 10:53:34 -0600 Subject: [PATCH 73/75] Trace native displacement hierarchy and independent bone scaling --- docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md | 46 ++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md index 246067f2..51fd29f4 100644 --- a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md @@ -366,3 +366,49 @@ queue database. The release is also reported in the user-facing turn. Next: trace and implement scale application to bind/animation transforms, wire selection/combination and decoded mesh bones through the shared Context/preview path, then qualify representative complete Creatures. Whole presentation remains open. + +## Native transform-stage continuation + +Source commit de80f2f CI run 39989492089 was inspected while its Windows job +120537245636 was still building; no later CI success is claimed here. Native +owned the shared runtime lease during this investigation. Forge stayed offline, +with no frontend/test launch or lease claim. Peer release notification remains +subject to the queue-capacity limitation recorded above. + +Read-only `scale-callers-*` / `scale-consumers-*` follow GetBoneScalings through +thunk 017b3b12 to GetBoneTransformations (02e79140). That routine passes scales to +CalcBlendedMeshTransformations, then applies each mesh bone's transposed +object-to-bone transform. The matrix cache is per render frame. This is not a +post-skin multiplication of exported vertex coordinates. + +`animation-scales-*`, `morph-stages-*` and `keyframe-scale-*` exports bring the +PE-byte verification list to 68 functions. Both CalcMeshTransformations +(0337de00) and CalcBlendedMeshTransformations (033800c0) use this order: + +1. Combine animation keyframes. +2. ApplyBoneMorphDisplacement (033823a0), when morph scales exist. +3. ApplyParentTransformations (033825f0). +4. ApplyBoneMorphScaling (03382ed0), when morph scales exist. + +Displacement walks local bones. For a non-root bone it requires the parent slot +to precede the child, then multiplies the child's local keyframe position +componentwise by the immediate parent's morph scale. Root positions are left +alone. The parent stage composes each local keyframe matrix with the already +computed parent matrix. The inspected 4x4 keyframe wrappers call quaternion +matrix conversion with the position; they do not explicitly apply stored scaling. +Do not reuse the head-pose inverse-bind decomposition as native Creature TRS +behavior without resolving this distinction. + +The final morph-scaling stage acts on each completed bone matrix separately. +Its scalar path scales the upper 3x3 rows for ordinary native layout or columns +for transposed layout; translation stays unchanged. The alternate aligned path +also restores the saved translation components. Therefore putting these morph +scales into local keyframe scale before hierarchy accumulation would incorrectly +compound ancestor scales through descendants. These facts are static source/ +instruction evidence, not an executed complete pose fixture. + +Next required check: an original-instruction two-/three-bone fixture separating +root position, parent-scaled child displacement, rotated parents and independent +post-hierarchy scale, plus transposed/nontransposed consistency and inverse-bind +order. Then implement the corresponding Creature pose calculation, reusing the +existing animation reader and skinning primitives where their contracts fit. From c81be322fbfb8e84e06b122b0f0a9129e4e82d3c Mon Sep 17 00:00:00 2001 From: BuffJesus <69093219+BuffJesus@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:02:21 -0600 Subject: [PATCH 74/75] Execute native Creature pose stages across hierarchy and scale fixtures --- docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md | 41 +++++- tools/audit_creature_pose_stages.py | 146 +++++++++++++++++++ 2 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 tools/audit_creature_pose_stages.py diff --git a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md index 51fd29f4..6f1c332d 100644 --- a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md @@ -369,8 +369,9 @@ path, then qualify representative complete Creatures. Whole presentation remains ## Native transform-stage continuation -Source commit de80f2f CI run 39989492089 was inspected while its Windows job -120537245636 was still building; no later CI success is claimed here. Native +Source commit de80f2f CI run 39989492089 completed successfully: build, units, +WARP GUI without an install and CLI usage passed; package/upload were skipped. +This covers the combined core source, not assembled-Creature GUI acceptance. Native owned the shared runtime lease during this investigation. Forge stayed offline, with no frontend/test launch or lease claim. Peer release notification remains subject to the queue-capacity limitation recorded above. @@ -412,3 +413,39 @@ root position, parent-scaled child displacement, rotated parents and independent post-hierarchy scale, plus transposed/nontransposed consistency and inverse-bind order. Then implement the corresponding Creature pose calculation, reusing the existing animation reader and skinning primitives where their contracts fit. + +## Executed native pose-stage fixture + +`tools/audit_creature_pose_stages.py` executes original displacement, parent and +scaling stages plus actual quaternion/matrix/keyframe/vector callees. The original +PE file stays unchanged; its mapped emulator image receives fixture data and CPU +branch selection. Only mesh bone count, local-bone lookup and parent lookup are +bridged. Receivers/indices are checked with distinct manager/mesh identities. +Code bytes are not patched. Executed ranges and producer/input hashes are pinned. + +All 240 cases pass: one-/two-/three-bone chains, siblings and multiple roots; +identity/nonuniform scales; six quaternion patterns including zero and non-unit; +both matrix layouts and both selected CPU math branches. Exactly representable +fixtures match numeric matrix values exactly (signed zero is ignored). Child +displacement, hierarchy and independent final scaling are checked separately. +Stored keyframe scaling is deliberately nonidentity (7,8,9) and remains intact; +these 4x4 matrix routines ignore it. + +The first rotated probe exposed an incorrect quaternion sign convention in the +initial independent model. Native ToMatrixTransposed (03382b80) confirms the +off-diagonal signs and factor `2 / squared_norm`, with a norm-zero fallback. +The corrected model passes the full matrix. The failed probe and resolution are +retained; this was an oracle-model error, not an engine defect. Do not silently +reuse the common glTF quaternion formula for raw native keyframes. + +Final packet: `C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/ps-muf0p5y6/result.json`. +It retains the producer and first-probe diagnostic. Intermediate 160/240-case +packets remain under ps-3v36ysp4 and ps-ii0pkvc9. Leaf exports (`pose-leaves-*`) +bring PE-byte verification to 78 functions. These are offline emulated instructions; +no frontend, game or named unit executable launched and no runtime lease claimed. + +The fixture does not execute animation sampling/blending, inverse-bind composition +or vertex skinning. Next: pin inverse-bind matrix order with the actual native +operation, then implement Creature pose calculation against these vectors, +compare C++ and qualify representative decoded meshes. Keep raw native quaternion +conventions explicit at the conversion boundary. Whole presentation remains open. diff --git a/tools/audit_creature_pose_stages.py b/tools/audit_creature_pose_stages.py new file mode 100644 index 00000000..411756a0 --- /dev/null +++ b/tools/audit_creature_pose_stages.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Execute original displacement/hierarchy/scaling on bounded synthetic skeletons. + +Only mesh count, local bone lookup and parent lookup are bridged. Quaternion, +matrix, keyframe and vector operations execute original bytes. No game launches. +This does not execute animation sampling, inverse-bind application or skinning. +""" +import argparse +import hashlib +import itertools +import json +from pathlib import Path +import struct + +import pefile +from unicorn import Uc, UC_ARCH_X86, UC_MODE_32, UC_HOOK_CODE +from unicorn.x86_const import UC_X86_REG_EAX, UC_X86_REG_ECX, UC_X86_REG_EIP, UC_X86_REG_ESP +from audit_creature_appearance_rng import EXPECTED_SHA + + +def multiply(a,b): + return [sum(a[r*4+k]*b[k*4+c] for k in range(4)) for r in range(4) for c in range(4)] + + +def local_matrix(q,p): + x,y,z,w=q + # Native ToMatrixTransposed (03382b80), including its norm-zero fallback: + # its off-diagonal signs differ from the common glTF column-vector formula. + norm=x*x+y*y+z*z+w*w + factor=2/norm if norm else 1 + return [1-factor*(y*y+z*z),factor*(x*y+z*w),factor*(x*z-y*w),p[0], + factor*(x*y-z*w),1-factor*(x*x+z*z),factor*(y*z+x*w),p[1], + factor*(x*z+y*w),factor*(y*z-x*w),1-factor*(x*x+y*y),p[2],0,0,0,1] + + +def transpose(m): + return [m[c*4+r] for r in range(4) for c in range(4)] + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--exe',type=Path,required=True) + parser.add_argument('--output',type=Path,required=True) + args=parser.parse_args() + if args.output.resolve()==args.exe.resolve():raise ValueError('output aliases input') + raw=args.exe.read_bytes();digest=hashlib.sha256(raw).hexdigest() + if digest!=EXPECTED_SHA:raise ValueError('unsupported executable; no execution attempted') + pe=pefile.PE(data=raw);base=pe.OPTIONAL_HEADER.ImageBase + image=pe.get_memory_mapped_image();size=(len(image)+4095)&~4095 + machine=Uc(UC_ARCH_X86,UC_MODE_32);machine.mem_map(base,size);machine.mem_write(base,image) + machine.mem_map(0,4096);machine.mem_map(0x10000000,0x20000) + frames,scales,matrices,mesh,bones=0x10000100,0x10001000,0x10002000,0x10003000,0x10004000 + manager=0x10005000 + stack,stop=0x10018000,0x1001f000 + parents=[];executed={};calls={} + bridges={0x017dc463:'mesh bone count',0x017b6f5b:'mesh local bone lookup',0x017d63b0:'bone parent lookup'} + + def hook(uc,address,length,unused): + if address in bridges: + sp=uc.reg_read(UC_X86_REG_ESP) + this=uc.reg_read(UC_X86_REG_ECX) + if address!=0x017d63b0 and this!=mesh:raise AssertionError('wrong mesh receiver at bridge') + if address==0x017dc463:value,extra=len(parents),0 + elif address==0x017b6f5b: + index,=struct.unpack('=len(parents):raise AssertionError('bone lookup out of fixture') + value,extra=bones+index*32,4 + else: + index=(this-bones)//32 + if not 0<=index Date: Fri, 23 Oct 2026 11:12:22 -0600 Subject: [PATCH 75/75] Checkpoint Creature pose implementation and paused-session handoff --- CMakeLists.txt | 3 + docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md | 5 + docs/HANDOFF_CREATURE_POSE_2026-10-23.md | 112 ++++++++++++++++++ docs/HANDOFF_NAV.md | 3 + docs/ROADMAP_1.0.md | 4 + libs/forgecore/include/forge/creaturepose.hpp | 25 ++++ libs/forgecore/src/creaturepose.cpp | 81 +++++++++++++ tests/test_creaturepose.cpp | 68 +++++++++++ tools/audit_creature_pose_stages.py | 19 ++- 9 files changed, 317 insertions(+), 3 deletions(-) create mode 100644 docs/HANDOFF_CREATURE_POSE_2026-10-23.md create mode 100644 libs/forgecore/include/forge/creaturepose.hpp create mode 100644 libs/forgecore/src/creaturepose.cpp create mode 100644 tests/test_creaturepose.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a9b0effb..1179fb9e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,6 +102,9 @@ add_test(NAME fableforge_fmplinks_tests COMMAND fableforge_fmplinks_tests) add_executable(fableforge_boneconfig_tests tests/test_boneconfig.cpp) target_link_libraries(fableforge_boneconfig_tests PRIVATE forgecore_slice) add_test(NAME fableforge_boneconfig_tests COMMAND fableforge_boneconfig_tests) +add_executable(fableforge_creaturepose_tests tests/test_creaturepose.cpp) +target_link_libraries(fableforge_creaturepose_tests PRIVATE forgecore_slice) +add_test(NAME fableforge_creaturepose_tests COMMAND fableforge_creaturepose_tests) add_executable(fableforge_defrelocate_tests tests/test_defrelocate.cpp) target_link_libraries(fableforge_defrelocate_tests PRIVATE forgecore_slice) add_test(NAME fableforge_defrelocate_tests COMMAND fableforge_defrelocate_tests) diff --git a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md index 6f1c332d..3a887982 100644 --- a/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md +++ b/docs/HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md @@ -1,5 +1,10 @@ # Creature selection and assembly continuation +Paused at the user's request. Resume from the +[Creature pose checkpoint](HANDOFF_CREATURE_POSE_2026-10-23.md): native inverse-bind +order is now verified; the new C++ pose target builds but its execution checks and +retail bind diagnostics are pending. No Forge runtime lease remains. + The checked core appearance reader now has a pure deterministic selection API, `forge::creatureappearance::select`. All 2,640 corpus selections match execution of the original native selection routines. It is not connected to scene/Models diff --git a/docs/HANDOFF_CREATURE_POSE_2026-10-23.md b/docs/HANDOFF_CREATURE_POSE_2026-10-23.md new file mode 100644 index 00000000..48b2c752 --- /dev/null +++ b/docs/HANDOFF_CREATURE_POSE_2026-10-23.md @@ -0,0 +1,112 @@ +# Paused Creature pose continuation, 2026-10-23 + +The user asked to stop and prepare the next session. The continuing FableForge +goal is paused, not complete. Resume here, then consult the detailed +[assembly evidence](HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md). + +## Checkout and accepted baseline + +Use `C:/Users/Cornelio/AppData/Local/FableForgeDevelopment/definition-clone-gui-20261010`, +branch `feat/definition-clone-gui-20261010`. Preserve the separate dirty +`D:/Code/FableForge` checkout and Native work. For Git commands set: + +```powershell +$env:GIT_OBJECT_DIRECTORY='C:/Users/Cornelio/AppData/Local/FableForgeDevelopment/definition-clone-git-objects' +$env:GIT_ALTERNATE_OBJECT_DIRECTORIES='D:/Code/FableForge/.git/objects' +``` + +Read root AGENTS and the project's dev/format/RE skills. No subagents were used. +Keep PR #2 draft; no merge, stable release or replacement of immutable ZIPs. +The retained package is still the profiles checkpoint built from 37c1ed1, not +the subsequent source work. Whole-Creature rendering/gameplay remain open. + +Source through de80f2f is accepted for the shared appearance reader/selector, +ownership resolver, bone-config reader, positive scale combination and CRC slot +mapping. CI 39989492089 passed build, units, WARP GUI without an install and CLI; +package/upload skipped. Local receipts and exact boundaries are in the assembly +handoff. c81be32 adds the executed native pose-stage oracle, not renderer integration. + +## Current checkpoint: builds, execution pending + +New core files `forge/creaturepose.hpp` and `creaturepose.cpp` implement: + +- Native quaternion/position frames, with parent-scaled child displacement. +- Rigid parent hierarchy followed by independent per-bone geometry scaling. +- `global * inverseBind` skin matrices and finite/count/parent/affine checks. +- A decoded-mesh bind adapter reading preserved 48-byte native keyframes; + stored keyframe scaling is ignored by this path, as in the native 4x4 routines. + +The new `fableforge_creaturepose_tests` target **builds but has not been run**. +Build log: `build/creature-pose-build-20261023.log`. Its modes are: + +```text +fableforge_creaturepose_tests.exe +fableforge_creaturepose_tests.exe --oracle +fableforge_creaturepose_tests.exe --bind-mesh +``` + +No-argument cases cover displacement/independent scale, malformed inputs and raw +bind-record preservation. Oracle mode compares globals for 240 native cases and +skin matrices for 120 transposed cases. `--bind-mesh` reports bone/vertex counts +and neutral skin-matrix deviation; this retail diagnostic has not been executed. +`build/creature-pose-retail-samples-20261023.json` selects 12 spread body meshes +from the 139 distinct nonzero mesh IDs in all stored appearance alternatives. +No retail mesh result or C++ pose acceptance may be inferred from the build. + +The C++ matrix/quaternion arithmetic has not been qualified on arbitrary retail +frames. Synthetic oracle comparisons are the next gate, not a general numerical +or animation-parity claim. The API is unused by scene/Assets rendering. + +## Latest original-instruction evidence + +`tools/audit_creature_pose_stages.py` was extended through actual native +AlignedPreMultiplyTransposed (02e8e0f0). All 240 stage cases pass; 120 transposed +cases also confirm `posedGlobal * inverseBind` order. Packet: +`C:/Users/Cornelio/AppData/Local/FableForgeEvidence/20261023/pi-f_r5r0vx/result.json`. +It retains the matching producer. Working copy: +`build/creature-assembly-native-20261023/pose-inverse-bind-native.json`. + +Only mesh count, local-bone lookup and parent lookup are bridged; actual math +instructions execute in Unicorn. This is offline emulation, not a native game +launch. Quaternion signs differ from the common glTF formula. The earlier wrong +model and correction remain recorded in the prior pose packets. Both CPU math +paths, both matrix layouts, zero/non-unit quaternions, independent roots, +siblings and nonuniform scales are covered. Animation sampling and vertex +skinning are still outside this oracle. + +## Coordination and first actions on resume + +No Forge runtime/controller or owned lease remains. Native was actively taking +fresh per-capture V86 windows at the last inspection; never treat that historical +observation as a current hold or permission. Read the latest queue/lease/process +state. Do not close user-owned games or reclaim another owner's lease. + +Shared lease: `D:/Documents/FableTLC/.agents/runtime-lease.json`. +Use the existing RuntimeLease helper and immediate conservative process guard +before every named executable launch. Neighboring TLC CPU tests also trip it. +Use short bounded windows and release only the owned token with a receipt. + +Native thread `01a168a5-fbee-72f0-87fe-39450caddb64` reached 100 queued submissions; +release notification for the earlier bone-combination check failed despite the +actual successful lease release. Read-only count still showed 100 at the latest +inspection. Do not retry-spam or mutate the queue DB. Resolve current transport +availability before relying on notification delivery. This session took no new +runtime lease for the unexecuted Creature pose checks. + +Next actions: + +1. Reinspect the working tree and build the scoped pose target below normal, + Ninja `-j2`, if source/binary identities require it. +2. In a fresh coordinated window, run the unit and `--oracle` against the packet + above. Preserve the first failure; fix actual convention/layout differences + instead of weakening the native expectation. +3. Run representative retail bind diagnostics. Steam inputs are under + `C:/Programs/Steam/steamapps/common/Fable The Lost Chapters`; the bank is + `data/graphics/graphics.big`. Pin source hashes and investigate deviations + before claiming pose correctness on decoded meshes. +4. Reuse the existing animation reader and suitable skinning primitives, then + connect selection/texture composition/morph resources through the shared + ownership resolver and runtime profile into complete Creature presentation. + Preserve pack asset context, use explicit preview seeds, and distinguish bind + preview from animation. Missing parts or unsupported morphs must not silently + become a partial NPC presented as complete. diff --git a/docs/HANDOFF_NAV.md b/docs/HANDOFF_NAV.md index ec9ffacf..2ffdd217 100644 --- a/docs/HANDOFF_NAV.md +++ b/docs/HANDOFF_NAV.md @@ -5,6 +5,9 @@ [Creature assembly continuation](HANDOFF_CREATURE_ASSEMBLY_2026-10-23.md) records the checked reader/selector source milestone and remaining presentation integration. +Paused-session resume point: [Creature pose checkpoint](HANDOFF_CREATURE_POSE_2026-10-23.md). +The pose target builds; execution and retail diagnostics remain pending. + # Navigation lane, 2026-09-27 Checkout: `D:/Code/FableForge-nav`, branch `feat/nav-thing-lines`, starting commit diff --git a/docs/ROADMAP_1.0.md b/docs/ROADMAP_1.0.md index 51c466a4..047a3b61 100644 --- a/docs/ROADMAP_1.0.md +++ b/docs/ROADMAP_1.0.md @@ -2,6 +2,10 @@ # Albion Atlas -> 1.0: what it should do, what it can't, and how we get there +Paused-session resume point: [Creature pose checkpoint](HANDOFF_CREATURE_POSE_2026-10-23.md). +Native pose/inverse-bind fixtures pass; the new C++ pose target builds but has not +been executed or connected to complete Creature rendering. + Subsequent source work adds a shared checked Creature appearance-table reader: 330 native components match an independent reader, including the populated global texture map previously refused. Four affected units, five donor fixtures, diff --git a/libs/forgecore/include/forge/creaturepose.hpp b/libs/forgecore/include/forge/creaturepose.hpp new file mode 100644 index 00000000..16e191cc --- /dev/null +++ b/libs/forgecore/include/forge/creaturepose.hpp @@ -0,0 +1,25 @@ +#pragma once +#include +#include +#include +#include "forge/meshpreview.hpp" + +namespace forge::creaturepose { +using Matrix=std::array; // row-major storage, column-vector points +struct Frame { + std::array rotation{0,0,0,1}; // raw native quaternion convention + std::array position{}; +}; +struct Pose { + std::vector global; // hierarchy followed by independent bone scaling + std::vector skin; // global * inverseBind +}; +// Morph displacement, rigid hierarchy, independent scale, then inverse bind. +// Parents must precede children; negative parent slots are roots. Empty scales +// means identity. Does not sample animation or modify input frames/geometry. +Pose evaluate(std::span frames,std::span parents, + std::span> scales,std::span inverseBinds); +// Read raw native rotation/position from the mesh's preserved 48-byte bind rows. +// Stored keyframe scaling is ignored, matching the native 4x4 pose stages. +Pose bindPose(const meshpreview::Geometry& mesh,std::span> scales={}); +} // namespace forge::creaturepose diff --git a/libs/forgecore/src/creaturepose.cpp b/libs/forgecore/src/creaturepose.cpp new file mode 100644 index 00000000..08f7246d --- /dev/null +++ b/libs/forgecore/src/creaturepose.cpp @@ -0,0 +1,81 @@ +#include "forge/creaturepose.hpp" +#include +#include +#include +#include + +namespace forge::creaturepose { +namespace { +[[noreturn]] void fail(const char* reason) {throw std::runtime_error(std::string("creature pose: ")+reason);} +template bool finite(const T& values) { + return std::all_of(values.begin(),values.end(),[](float value){return std::isfinite(value);}); +} +Matrix multiply(const Matrix& a,const Matrix& b) { + Matrix result{}; + for(size_t r=0;r<4;++r)for(size_t c=0;c<4;++c) { + double value=0; + for(size_t k=0;k<4;++k)value+=double(a[r*4+k])*b[k*4+c]; + result[r*4+c]=float(value); + } + if(!finite(result))fail("matrix overflow"); + return result; +} +Matrix local(const Frame& frame,const std::array& position) { + if(!finite(frame.rotation)||!finite(position))fail("nonfinite frame"); + const auto [x,y,z,w]=frame.rotation; + const double norm=double(x)*x+double(y)*y+double(z)*z+double(w)*w; + const float factor=norm ? float(2/norm) : 1; + if(!std::isfinite(float(norm))||!std::isfinite(factor))fail("unsupported quaternion magnitude"); + Matrix result{1-factor*(y*y+z*z),factor*(x*y+z*w),factor*(x*z-y*w),position[0], + factor*(x*y-z*w),1-factor*(x*x+z*z),factor*(y*z+x*w),position[1], + factor*(x*z+y*w),factor*(y*z-x*w),1-factor*(x*x+y*y),position[2], + 0,0,0,1}; + if(!finite(result))fail("quaternion matrix overflow"); + return result; +} +} +Pose evaluate(std::span frames,std::span parents, + std::span> scales,std::span inverseBinds) { + const auto count=frames.size(); + if(count>65536 || parents.size()!=count || inverseBinds.size()!=count || + (!scales.empty()&&scales.size()!=count))fail("skeleton/pose size mismatch"); + for(size_t i=0;i=0&&size_t(parents[i])>=i)fail("parent must precede child"); + if(!finite(inverseBinds[i])||(!scales.empty()&&!finite(scales[i])))fail("nonfinite bone input"); + if(inverseBinds[i][12]!=0||inverseBinds[i][13]!=0||inverseBinds[i][14]!=0||inverseBinds[i][15]!=1) + fail("inverse bind must be affine"); + } + Pose result;result.global.resize(count);result.skin.resize(count); + for(size_t i=0;i=0&&!scales.empty()) + for(size_t axis=0;axis<3;++axis)position[axis]*=scales[size_t(parent)][axis]; + const auto matrix=local(frames[i],position); + result.global[i]=parent<0 ? matrix : multiply(result.global[size_t(parent)],matrix); + } + // Scale only after every hierarchy matrix is complete: descendants must not + // inherit ancestor geometry scale through the accumulated parent matrix. + for(size_t i=0;i> scales) { + const auto count=mesh.bones.size(); + if(mesh.boneCount!=count || count>65536)fail("decoded skeleton count mismatch"); + if(count && (!mesh.skeletonBlocks || mesh.skeletonBlocks->keyframes.size()!=count*48)) + fail("native bind keyframes unavailable"); + std::vector frames(count);std::vector parents(count);std::vector binds(count); + for(size_t i=0;ikeyframes.data()+i*48; + std::memcpy(frames[i].rotation.data(),source,16); + std::memcpy(frames[i].position.data(),source+16,12); + parents[i]=mesh.bones[i].parent;binds[i]=mesh.bones[i].inverseBind; + } + return evaluate(frames,parents,scales,binds); +} +} // namespace forge::creaturepose diff --git a/tests/test_creaturepose.cpp b/tests/test_creaturepose.cpp new file mode 100644 index 00000000..a350d775 --- /dev/null +++ b/tests/test_creaturepose.cpp @@ -0,0 +1,68 @@ +#include "forge/creaturepose.hpp" +#include +#include +#include +#include +#include +#include +#include + +using namespace forge::creaturepose; +static constexpr Matrix identity{1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1}; +static void require(bool ok,const char* message){if(!ok)throw std::runtime_error(message);} +templatestatic void refuses(F action){try{action();}catch(const std::runtime_error&){return;}throw std::runtime_error("invalid pose accepted");} +int main(int argc,char** argv)try { + if(argc==4&&std::string_view(argv[1])=="--bind-mesh") { + const auto id=std::stoull(argv[3]);require(id<=UINT32_MAX,"mesh ID overflow"); + const auto mesh=forge::meshpreview::readLod0(argv[2],uint32_t(id)); + const auto pose=bindPose(mesh);float deviation=0; + for(const auto& matrix:pose.skin)for(size_t i=0;i<16;++i) + deviation=std::max(deviation,std::abs(matrix[i]-identity[i])); + std::cout< frames; + for(const auto& frame:record.at("frames"))frames.push_back({frame.at("rotation").get>(),frame.at("position").get>()}); + const auto parents=record.at("parents").get>(); + const auto scales=record.at("scales").get>>(); + const auto binds=record.at("inverse_binds").get>(); + const auto pose=evaluate(frames,parents,scales,binds); + require(pose.global==record.at("final").get>(),"native global pose differs"); + if(record.at("transposed").get()) { + require(pose.skin==record.at("skin").get>(),"native inverse-bind composition differs");++skins; + } + ++cases; + } + std::cout< frames{{{0,0,0,1},{3,4,5}},{{0,0,0,1},{1,2,3}}}; + std::vector parents{-1,0};std::vector binds(2,identity); + const std::vector> scales{{2,3,4},{.5,.75,1.25}}; + const auto pose=evaluate(frames,parents,scales,binds); + require(pose.global[0][3]==3&&pose.global[1][3]==5&&pose.global[1][7]==10&&pose.global[1][11]==17&& + pose.global[1][0]==.5&&pose.global[1][5]==.75&&pose.global[1][10]==1.25&&pose.skin==pose.global, + "parent displacement or independent geometry scale differs"); + auto invalidParents=parents;invalidParents[1]=1;refuses([&]{evaluate(frames,invalidParents,scales,binds);}); + refuses([&]{evaluate(frames,parents,std::span(scales).first(1),binds);}); + auto invalidBinds=binds;invalidBinds[0][12]=1;refuses([&]{evaluate(frames,parents,scales,invalidBinds);}); + auto invalidFrames=frames;invalidFrames[0].rotation[0]=std::numeric_limits::quiet_NaN(); + refuses([&]{evaluate(invalidFrames,parents,scales,binds);}); + require(evaluate({}, {}, {}, {}).skin.empty(),"empty skeleton differs"); + forge::meshpreview::Geometry mesh;mesh.boneCount=2;mesh.bones.resize(2);mesh.skeletonBlocks.emplace(); + mesh.skeletonBlocks->keyframes.resize(96); + for(size_t i=0;i<2;++i) { + mesh.bones[i].parent=parents[i];mesh.bones[i].inverseBind=binds[i]; + const std::array row{0,0,0,1,frames[i].position[0],frames[i].position[1],frames[i].position[2],0,7,8,9,0}; + std::memcpy(mesh.skeletonBlocks->keyframes.data()+48*i,row.data(),48); + } + const auto before=mesh.skeletonBlocks->keyframes; + require(bindPose(mesh,scales).skin==pose.skin&&mesh.skeletonBlocks->keyframes==before, + "raw bind frames must preserve input and ignore stored scaling"); + mesh.skeletonBlocks->keyframes.pop_back();refuses([&]{bindPose(mesh,scales);}); + std::cout<<"Creature pose: PASS\n";return 0; +} catch(const std::exception& error){std::cerr<