From 4695187e493fa69a62f3d526979b3e5941c234f0 Mon Sep 17 00:00:00 2001 From: lr90 Date: Fri, 30 Oct 2026 23:25:34 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix(adapter):=20=E6=8C=89=20embedding=20tok?= =?UTF-8?q?en=20=E9=A2=84=E7=AE=97=E5=88=86=E7=89=87=E5=B9=B6=E5=9B=BA?= =?UTF-8?q?=E5=AE=9A=E9=87=8D=E8=AF=95=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/aml-benchmark.yml | 2 + .github/workflows/aml-ci.yml | 1 + agent_memory_leaderboard/.gitignore | 3 + agent_memory_leaderboard/Dockerfile | 1 + agent_memory_leaderboard/README.md | 49 ++++ agent_memory_leaderboard/pyproject.toml | 1 + .../requirements-experiment.lock | 113 ++++++++- agent_memory_leaderboard/requirements.lock | 111 +++++++- agent_memory_leaderboard/src/app.py | 2 + .../src/assets/NOTICE.txt | 31 +++ agent_memory_leaderboard/src/config.py | 7 + .../src/embedding_tokens.py | 67 +++++ agent_memory_leaderboard/src/ingestion.py | 40 ++- agent_memory_leaderboard/src/journal.py | 47 +++- agent_memory_leaderboard/src/service.py | 39 ++- .../tests/test_ingestion.py | 2 +- .../tests/test_journal_concurrency.py | 15 +- .../tests/test_journal_metrics.py | 8 +- agent_memory_leaderboard/tests/test_ready.py | 5 +- .../tests/test_token_chunks.py | 237 ++++++++++++++++++ agent_memory_leaderboard/uv.lock | 148 +++++++++++ 21 files changed, 906 insertions(+), 23 deletions(-) create mode 100644 agent_memory_leaderboard/src/assets/NOTICE.txt create mode 100644 agent_memory_leaderboard/src/embedding_tokens.py create mode 100644 agent_memory_leaderboard/tests/test_token_chunks.py diff --git a/.github/workflows/aml-benchmark.yml b/.github/workflows/aml-benchmark.yml index 2c201d2..13360c8 100644 --- a/.github/workflows/aml-benchmark.yml +++ b/.github/workflows/aml-benchmark.yml @@ -71,6 +71,8 @@ jobs: - name: Install locked benchmark dependencies timeout-minutes: 5 run: uv sync --locked --extra evaluation + - name: Prepare pinned embedding tokenizer + run: uv run --no-sync python -m src.embedding_tokens - name: Run Add/Search and selected evaluation mode id: execute timeout-minutes: 120 diff --git a/.github/workflows/aml-ci.yml b/.github/workflows/aml-ci.yml index 58c0953..9f23625 100644 --- a/.github/workflows/aml-ci.yml +++ b/.github/workflows/aml-ci.yml @@ -29,6 +29,7 @@ jobs: enable-cache: true cache-dependency-glob: agent_memory_leaderboard/uv.lock - run: uv sync --locked --extra evaluation + - run: uv run --no-sync python -m src.embedding_tokens - run: uv run --no-sync ruff check src tests benchmarks scripts/benchmark_ci.py ../deploy/scripts ../deploy/tests - run: uv run --no-sync pytest -q - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4 diff --git a/agent_memory_leaderboard/.gitignore b/agent_memory_leaderboard/.gitignore index 0054d94..f37bdff 100644 --- a/agent_memory_leaderboard/.gitignore +++ b/agent_memory_leaderboard/.gitignore @@ -22,3 +22,6 @@ state/*.json *.sqlite3-journal *.sqlite3-wal *.sqlite3-shm + +# Pinned public tokenizer downloaded and verified during setup/build. +src/assets/tokenizer.json diff --git a/agent_memory_leaderboard/Dockerfile b/agent_memory_leaderboard/Dockerfile index 6db964e..3177327 100644 --- a/agent_memory_leaderboard/Dockerfile +++ b/agent_memory_leaderboard/Dockerfile @@ -9,6 +9,7 @@ RUN if [ "$INSTALL_EXPERIMENT_DEPS" = "true" ]; then \ fi RUN useradd --uid 1000 --create-home adapter && mkdir /app/var && chown adapter /app/var COPY src ./src +RUN python -m src.embedding_tokens USER adapter EXPOSE 8000 CMD ["uvicorn", "src.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8000"] diff --git a/agent_memory_leaderboard/README.md b/agent_memory_leaderboard/README.md index cefd9fe..3d6b460 100644 --- a/agent_memory_leaderboard/README.md +++ b/agent_memory_leaderboard/README.md @@ -41,6 +41,7 @@ personal API key cannot provide isolation for multiple leaderboard users. ```sh uv sync --locked +uv run python -m src.embedding_tokens export MEMORIA_URL=http://127.0.0.1:8100 export MEMORIA_MASTER_KEY='your-memoria-master-key' export ADAPTER_API_KEY='a-separate-leaderboard-access-key' @@ -626,3 +627,51 @@ The self-run benchmark client can generate bounded, private local per-question Add/Search/Answer/Eval reports for independently sourced synthetic or permitted public data. This is disabled by default and adds no official-request body logs. See [the offline demonstration, source declaration and limits](benchmarks/EVALUATION.md#opt-in-local-content-diagnostics). + +### Token-aware ingestion and retry compatibility + +New Add requests use both the existing 32 KiB UTF-8 bound and a BGE-M3 token +budget. `ADAPTER_EMBEDDING_TOKEN_BUDGET` defaults to **7680** (allowed range +128–8192); the count includes the transmitted role/time prefix and special +tokens. Original Unicode is sliced without tokenizer decoding, truncation or +summarization. Short messages that fit both limits remain unchanged. Splitting +increases write/embedding/visibility-check work for long messages. + +The tokenizer is pinned to `BAAI/bge-m3` revision +`5617a9f61b028005a4858fdac845db406aefb181`, with SHA-256 +`21106b6d7dab2952c1d496fb21d5dc9db75c28ed361a05f5020bbba27810dd08`. +The upstream [BGE-M3 model/tokenizer](https://huggingface.co/BAAI/bge-m3) is +MIT-licensed. The tokenizer runtime is locked with the Python dependencies. +This configuration targets BGE-M3's 8192-token input limit; it is not a generic +limit for other embedding models. Changing the backend model requires checking +its tokenizer and adding a compatible, versioned rendering policy first. + +Run `uv run python -m src.embedding_tokens` after installing dependencies and +before local startup/tests. This downloads only the public tokenizer, verifies +its checksum and stores it in ignored `src/assets/tokenizer.json`. The image +build does this automatically. `ADAPTER_TOKENIZER_PATH` may point to a preloaded +copy of the **same** asset. Runtime requests never download assets; missing or +mismatched assets fail application startup. For offline builds, preload the +verified asset in the build context. An existing verified copy is reused. + +Each request's journal row pins its rendering version, tokenizer digest and +budget in the same durable transaction as its first write intent. Changing the +default budget only affects new requests. Existing journals migrate old rows to +byte-only version 1, retaining their chunk boundaries, positions and idempotency +keys. Completed retries return without re-rendering or backend calls. Unknown +policies fail before sending a write. The number of durable journal commits +remains three per new chunk; policy lookup adds one read per Add. + +**This does not automatically repair old oversized requests.** An incomplete v1 +request still uses its original byte-only chunks, even if one exceeds the model +limit. Never clear its journal or reinterpret existing positions to force a +retry. Coordinate an explicit replay/migration plan for such requests. Keep one +adapter worker/replica and stop the old version before upgrading. Do not roll +back to a byte-only binary with unfinished v2 requests: old code cannot honor +pinned token boundaries. Preserve the journal and fixed model/tokenizer for the +whole run. Ordinary Search queries are unchanged; the write-visibility search +uses each newly bounded chunk. + +Local source manifests use the new default rendering version for attribution. +They must not be treated as an exact reproduction of old v1 runs or runs using +a different token budget. diff --git a/agent_memory_leaderboard/pyproject.toml b/agent_memory_leaderboard/pyproject.toml index bc38be1..38c86f1 100644 --- a/agent_memory_leaderboard/pyproject.toml +++ b/agent_memory_leaderboard/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "httpx>=0.28,<1", "pydantic>=2.10,<3", "uvicorn>=0.34,<1", + "tokenizers==0.22.2", ] [project.optional-dependencies] diff --git a/agent_memory_leaderboard/requirements-experiment.lock b/agent_memory_leaderboard/requirements-experiment.lock index 53da773..f8ba2ea 100644 --- a/agent_memory_leaderboard/requirements-experiment.lock +++ b/agent_memory_leaderboard/requirements-experiment.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv export --locked --no-dev --extra evaluation --format requirements-txt --output-file requirements-experiment.lock +# uv export --locked --no-dev --extra evaluation --no-emit-project --format requirements-txt --output-file requirements-experiment.lock annotated-doc==0.0.5 \ --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb @@ -151,17 +151,50 @@ charset-normalizer==3.5.2 \ click==8.5.0 \ --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 - # via uvicorn + # via + # huggingface-hub + # uvicorn +colorama==0.4.6 ; sys_platform == 'win32' \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + # via tqdm fastapi==0.142.2 \ --hash=sha256:06366626f2e70576367714d9ab2fe8472e6c8456dba69b399f9f797ab5e92570 \ --hash=sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b # via agent-memory-leaderboard +filelock==4.2.4 \ + --hash=sha256:301241438c57936ef93f4243a94f2559022268bfcf9b375a9a72eb0f8db8358e \ + --hash=sha256:c6d8663535e1650ee430507b4ffbac7b464dd64b7647ea0112a9f406499d163d + # via huggingface-hub +fsspec==2026.9.0 \ + --hash=sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe \ + --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f + # via huggingface-hub h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via # httpcore # uvicorn +hf-xet==1.8.0 ; platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ + --hash=sha256:07a06d7481702616726c2d8578cfb00052a5b3f21beed6a674592bc837a10330 \ + --hash=sha256:10ed23b51dacf7917d64f335c25048deb470efa755e9989b218fc0531386138e \ + --hash=sha256:1ebaf248c9bef8bb0c9d1d8d71db1fdf0cac14f922cc7a936a81adbf12eb5f01 \ + --hash=sha256:23c262e9f5e137b18dadbce60becfeea84156b0d43119c320bffe260d32cd2fa \ + --hash=sha256:29dab7c904cd1914a0439c6e1306c12c7497c8bd32290fbe563ebf51ca4b6d6d \ + --hash=sha256:3e8934784425781f0f779b91044df441f3a7af4dab47375d7e3aeaecbb8eecb4 \ + --hash=sha256:3ecfe2934448cca55a4c42732b60f985266aecb241905945219b3d388aceb7ba \ + --hash=sha256:42b66fdc688eaa4ff3eb77dc9275e5a0f1258ecc9fdd083ddaf43bb49bec925b \ + --hash=sha256:56735c2d4f416d7473fc986586b26a8756e067832ca4b7fdaad32a0d6f2ce740 \ + --hash=sha256:62b056aedb89ac809f78940baf465fe553b70c8b25ee56890fbab3039fffec1b \ + --hash=sha256:7edeb21c5d381903ee5f4857fc746fa7b52a4b374035e8811ecfb67ee27add0c \ + --hash=sha256:8800e3444770f2c630b361bc6d5c6ebb770b7830646cf632fbb79d0e5bbc9e7f \ + --hash=sha256:92188598dedc1e1f5609bd718d4db8687a6c89915553912571e2f587892eecec \ + --hash=sha256:a584a7b3268da62f23c3187cc7201ca01f7b610985a36b1b6119c66bccb0383c \ + --hash=sha256:c74e3751cbcdae9222c5170e8d788e83201a03dc5e5f4308a20f17a5b6023b56 \ + --hash=sha256:e00d8f60170cd3dd55d98306e41dbaf28b27ff1bcfefbfa84279d197221be83e \ + --hash=sha256:f42940869572dc2b05dfe06f4cc398c6291503e086ddc4eddfb7a6ab9409ad01 + # via huggingface-hub httpcore==1.0.9 \ --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 @@ -169,7 +202,13 @@ httpcore==1.0.9 \ httpx==0.28.1 \ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad - # via agent-memory-leaderboard + # via + # agent-memory-leaderboard + # huggingface-hub +huggingface-hub==1.33.0 \ + --hash=sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899 \ + --hash=sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e + # via tokenizers idna==3.20 \ --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c @@ -181,6 +220,10 @@ opentelemetry-api==1.45.0 \ --hash=sha256:711ede81773c8025c2c03dac0450bc89f3d30aea6eabcc815c570d4e35a963f7 \ --hash=sha256:80e068aba7cd56c8b58512d6a36f8d25cb1dfaa0c0a4cc1c938ccf9f362d9cb3 # via fastapi +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via huggingface-hub pydantic==2.13.5 \ --hash=sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73 \ --hash=sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08 @@ -254,6 +297,47 @@ pydantic-core==2.46.5 \ --hash=sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0 \ --hash=sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9 # via pydantic +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via huggingface-hub regex==2026.9.29 \ --hash=sha256:01000ddf0e3ffef97f2413ceb514f6313040106b6d18a03ee00a4fe35c1eb1db \ --hash=sha256:0166844493626c5015c6088ee15c9ca2fd060ca15b7641d1657da6a58432ae33 \ @@ -406,12 +490,35 @@ tiktoken==0.14.0 \ --hash=sha256:f2af4a336ea56d6c14f27741a0e1d8294a35dd0b038bcf990d232ebb54eb994b \ --hash=sha256:f702e0aeeb6506e57687e881c59e844ebe8f0a6a097ddafe20e3ab25f387be4e # via agent-memory-leaderboard +tokenizers==0.22.2 \ + --hash=sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e \ + --hash=sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001 \ + --hash=sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7 \ + --hash=sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd \ + --hash=sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4 \ + --hash=sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67 \ + --hash=sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a \ + --hash=sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5 \ + --hash=sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917 \ + --hash=sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c \ + --hash=sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a \ + --hash=sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc \ + --hash=sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92 \ + --hash=sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5 \ + --hash=sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48 \ + --hash=sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b + # via agent-memory-leaderboard +tqdm==4.70.1 \ + --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 \ + --hash=sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4 + # via huggingface-hub typing-extensions==4.16.0 \ --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 # via # anyio # fastapi + # huggingface-hub # opentelemetry-api # pydantic # pydantic-core diff --git a/agent_memory_leaderboard/requirements.lock b/agent_memory_leaderboard/requirements.lock index f496a44..9bb8f62 100644 --- a/agent_memory_leaderboard/requirements.lock +++ b/agent_memory_leaderboard/requirements.lock @@ -24,17 +24,50 @@ certifi==2026.7.22 \ click==8.5.0 \ --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 - # via uvicorn + # via + # huggingface-hub + # uvicorn +colorama==0.4.6 ; sys_platform == 'win32' \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + # via tqdm fastapi==0.142.2 \ --hash=sha256:06366626f2e70576367714d9ab2fe8472e6c8456dba69b399f9f797ab5e92570 \ --hash=sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b # via agent-memory-leaderboard +filelock==4.2.4 \ + --hash=sha256:301241438c57936ef93f4243a94f2559022268bfcf9b375a9a72eb0f8db8358e \ + --hash=sha256:c6d8663535e1650ee430507b4ffbac7b464dd64b7647ea0112a9f406499d163d + # via huggingface-hub +fsspec==2026.9.0 \ + --hash=sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe \ + --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f + # via huggingface-hub h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via # httpcore # uvicorn +hf-xet==1.8.0 ; platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ + --hash=sha256:07a06d7481702616726c2d8578cfb00052a5b3f21beed6a674592bc837a10330 \ + --hash=sha256:10ed23b51dacf7917d64f335c25048deb470efa755e9989b218fc0531386138e \ + --hash=sha256:1ebaf248c9bef8bb0c9d1d8d71db1fdf0cac14f922cc7a936a81adbf12eb5f01 \ + --hash=sha256:23c262e9f5e137b18dadbce60becfeea84156b0d43119c320bffe260d32cd2fa \ + --hash=sha256:29dab7c904cd1914a0439c6e1306c12c7497c8bd32290fbe563ebf51ca4b6d6d \ + --hash=sha256:3e8934784425781f0f779b91044df441f3a7af4dab47375d7e3aeaecbb8eecb4 \ + --hash=sha256:3ecfe2934448cca55a4c42732b60f985266aecb241905945219b3d388aceb7ba \ + --hash=sha256:42b66fdc688eaa4ff3eb77dc9275e5a0f1258ecc9fdd083ddaf43bb49bec925b \ + --hash=sha256:56735c2d4f416d7473fc986586b26a8756e067832ca4b7fdaad32a0d6f2ce740 \ + --hash=sha256:62b056aedb89ac809f78940baf465fe553b70c8b25ee56890fbab3039fffec1b \ + --hash=sha256:7edeb21c5d381903ee5f4857fc746fa7b52a4b374035e8811ecfb67ee27add0c \ + --hash=sha256:8800e3444770f2c630b361bc6d5c6ebb770b7830646cf632fbb79d0e5bbc9e7f \ + --hash=sha256:92188598dedc1e1f5609bd718d4db8687a6c89915553912571e2f587892eecec \ + --hash=sha256:a584a7b3268da62f23c3187cc7201ca01f7b610985a36b1b6119c66bccb0383c \ + --hash=sha256:c74e3751cbcdae9222c5170e8d788e83201a03dc5e5f4308a20f17a5b6023b56 \ + --hash=sha256:e00d8f60170cd3dd55d98306e41dbaf28b27ff1bcfefbfa84279d197221be83e \ + --hash=sha256:f42940869572dc2b05dfe06f4cc398c6291503e086ddc4eddfb7a6ab9409ad01 + # via huggingface-hub httpcore==1.0.9 \ --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 @@ -42,7 +75,13 @@ httpcore==1.0.9 \ httpx==0.28.1 \ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad - # via agent-memory-leaderboard + # via + # agent-memory-leaderboard + # huggingface-hub +huggingface-hub==1.33.0 \ + --hash=sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899 \ + --hash=sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e + # via tokenizers idna==3.20 \ --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c @@ -53,6 +92,10 @@ opentelemetry-api==1.45.0 \ --hash=sha256:711ede81773c8025c2c03dac0450bc89f3d30aea6eabcc815c570d4e35a963f7 \ --hash=sha256:80e068aba7cd56c8b58512d6a36f8d25cb1dfaa0c0a4cc1c938ccf9f362d9cb3 # via fastapi +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via huggingface-hub pydantic==2.13.5 \ --hash=sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73 \ --hash=sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08 @@ -126,16 +169,80 @@ pydantic-core==2.46.5 \ --hash=sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0 \ --hash=sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9 # via pydantic +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via huggingface-hub starlette==1.7.0 \ --hash=sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e \ --hash=sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d # via fastapi +tokenizers==0.22.2 \ + --hash=sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e \ + --hash=sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001 \ + --hash=sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7 \ + --hash=sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd \ + --hash=sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4 \ + --hash=sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67 \ + --hash=sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a \ + --hash=sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5 \ + --hash=sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917 \ + --hash=sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c \ + --hash=sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a \ + --hash=sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc \ + --hash=sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92 \ + --hash=sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5 \ + --hash=sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48 \ + --hash=sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b + # via agent-memory-leaderboard +tqdm==4.70.1 \ + --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 \ + --hash=sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4 + # via huggingface-hub typing-extensions==4.16.0 \ --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 # via # anyio # fastapi + # huggingface-hub # opentelemetry-api # pydantic # pydantic-core diff --git a/agent_memory_leaderboard/src/app.py b/agent_memory_leaderboard/src/app.py index 425bee5..49c0430 100644 --- a/agent_memory_leaderboard/src/app.py +++ b/agent_memory_leaderboard/src/app.py @@ -10,6 +10,7 @@ from .budget import request_budget from .config import Settings +from .embedding_tokens import load_tokenizer from .errors import AdapterError from .journal import Journal from .memoria import Memoria @@ -20,6 +21,7 @@ def create_app(settings: Settings | None = None, *, client: httpx.Client | None = None) -> FastAPI: settings = settings or Settings.from_env() + load_tokenizer() # Fail startup rather than accept traffic with missing/wrong assets. http = client or httpx.Client(follow_redirects=False, trust_env=False) # Keep probe connections and thread capacity independent of long Add/Search calls. probe_http = client or httpx.Client( diff --git a/agent_memory_leaderboard/src/assets/NOTICE.txt b/agent_memory_leaderboard/src/assets/NOTICE.txt new file mode 100644 index 0000000..dc32688 --- /dev/null +++ b/agent_memory_leaderboard/src/assets/NOTICE.txt @@ -0,0 +1,31 @@ +BGE-M3 tokenizer +Publisher: Beijing Academy of Artificial Intelligence (BAAI) +Source: https://huggingface.co/BAAI/bge-m3 +Revision: 5617a9f61b028005a4858fdac845db406aefb181 +The model card declares the MIT license. +Only tokenizer.json is downloaded; no model weights are bundled. + +The associated upstream FlagEmbedding project license notice follows: +https://github.com/FlagOpen/FlagEmbedding/blob/master/LICENSE + +MIT License + +Copyright (c) 2022 staoxiao + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/agent_memory_leaderboard/src/config.py b/agent_memory_leaderboard/src/config.py index 0361f29..b1ccec9 100644 --- a/agent_memory_leaderboard/src/config.py +++ b/agent_memory_leaderboard/src/config.py @@ -25,8 +25,14 @@ class Settings: journal_slow_ms: float = 100 journal_queue_timeout_seconds: float = 30 time_normalization_enabled: bool = False + embedding_token_budget: int = 7680 def __post_init__(self) -> None: + if ( + type(self.embedding_token_budget) is not int + or not 128 <= self.embedding_token_budget <= 8192 + ): + raise ValueError("ADAPTER_EMBEDDING_TOKEN_BUDGET must be an integer in [128, 8192]") if type(self.time_normalization_enabled) is not bool: raise ValueError("MEMORY_TIME_NORMALIZATION_ENABLED must be a boolean") url = urlsplit(self.memoria_url) @@ -92,6 +98,7 @@ def from_env(cls) -> "Settings": if journal_logging not in {"true", "false", "1", "0"}: raise ValueError("ADAPTER_JOURNAL_LOG_ALL_OPERATIONS must be true/false or 1/0") return cls( + embedding_token_budget=int(os.getenv("ADAPTER_EMBEDDING_TOKEN_BUDGET", "7680")), time_normalization_enabled=time_normalization in {"true", "1"}, memoria_url=os.environ["MEMORIA_URL"].rstrip("/"), memoria_master_key=os.environ["MEMORIA_MASTER_KEY"], diff --git a/agent_memory_leaderboard/src/embedding_tokens.py b/agent_memory_leaderboard/src/embedding_tokens.py new file mode 100644 index 0000000..d14512e --- /dev/null +++ b/agent_memory_leaderboard/src/embedding_tokens.py @@ -0,0 +1,67 @@ +"""Pinned BGE-M3 tokenizer. Download at build/setup time, never during requests.""" + +import hashlib +import os +import tempfile +import urllib.request +from functools import cache +from pathlib import Path + +from tokenizers import Tokenizer + +TOKENIZER_REVISION = "5617a9f61b028005a4858fdac845db406aefb181" +TOKENIZER_SHA256 = "21106b6d7dab2952c1d496fb21d5dc9db75c28ed361a05f5020bbba27810dd08" +TOKENIZER_URL = f"https://huggingface.co/BAAI/bge-m3/resolve/{TOKENIZER_REVISION}/tokenizer.json" +DEFAULT_TOKEN_BUDGET = 7680 + + +def tokenizer_path() -> Path: + return Path( + os.getenv("ADAPTER_TOKENIZER_PATH", Path(__file__).parent / "assets/tokenizer.json") + ) + + +@cache +def load_tokenizer() -> Tokenizer: + try: + data = tokenizer_path().read_bytes() + except OSError: + raise RuntimeError("Tokenizer asset missing; run python -m src.embedding_tokens") from None + if hashlib.sha256(data).hexdigest() != TOKENIZER_SHA256: + raise RuntimeError("Tokenizer asset checksum mismatch") + tokenizer = Tokenizer.from_str(data.decode("utf-8")) + tokenizer.no_truncation() + tokenizer.no_padding() + return tokenizer + + +def token_count(text: str) -> int: + # Count the exact transmitted text, including BGE-M3's special tokens. + return len(load_tokenizer().encode(text, add_special_tokens=True).ids) + + +def download_tokenizer() -> None: + target = tokenizer_path() + if target.exists() and hashlib.sha256(target.read_bytes()).hexdigest() == TOKENIZER_SHA256: + return + with urllib.request.urlopen(TOKENIZER_URL, timeout=120) as response: + data = response.read() + if hashlib.sha256(data).hexdigest() != TOKENIZER_SHA256: + raise RuntimeError("Downloaded tokenizer checksum mismatch") + target.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as output: + temporary = Path(output.name) + output.write(data) + temporary.chmod(0o644) + os.replace(temporary, target) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +if __name__ == "__main__": + download_tokenizer() + load_tokenizer() + print("Pinned BGE-M3 tokenizer ready (checksum verified)") diff --git a/agent_memory_leaderboard/src/ingestion.py b/agent_memory_leaderboard/src/ingestion.py index 9481535..8c84d56 100644 --- a/agent_memory_leaderboard/src/ingestion.py +++ b/agent_memory_leaderboard/src/ingestion.py @@ -2,9 +2,10 @@ from datetime import UTC, datetime, timedelta +from .embedding_tokens import DEFAULT_TOKEN_BUDGET, token_count from .models import text_content -RENDER_VERSION = 1 +RENDER_VERSION = 2 MAX_MEMORY_BYTES = 32768 @@ -18,7 +19,38 @@ def split_utf8(text: str, limit: int): remaining = remaining[len(part.encode("utf-8")) :] -def message_parts(message): +def split_tokens(text: str, header: str, budget: int): + """Slice original Unicode, never decode normalized tokenizer output. + + Prefix token counts need not be monotonic. The binary search only selects a + known-valid prefix; it need not find the largest possible one. Every emitted + prefix is counted again with its header and special tokens. + """ + offset = 0 + while offset < len(text): + remaining = text[offset:] + if token_count(header + remaining) <= budget: + yield remaining + return + lo, hi = 0, len(remaining) + while lo + 1 < hi: + middle = (lo + hi) // 2 + if token_count(header + remaining[:middle]) <= budget: + lo = middle + else: + hi = middle + if lo == 0: + raise ValueError("Token budget cannot fit the message prefix and a character") + # Prefer a nearby paragraph/word boundary, without dropping whitespace. + floor = lo * 3 // 4 + boundary = max(remaining.rfind(mark, floor, lo) + 1 for mark in ("\n", " ", "。")) + if boundary > floor and token_count(header + remaining[:boundary]) <= budget: + lo = boundary + yield remaining[:lo] + offset += lo + + +def message_parts(message, *, token_budget: int | None = DEFAULT_TOKEN_BUDGET): observed_at = None header = f"[{message.role}]\n" if message.timestamp is not None: @@ -29,4 +61,6 @@ def message_parts(message): ) header = f"[{message.role} at {observed_at}]\n" for part in split_utf8(text_content(message.content), MAX_MEMORY_BYTES - len(header.encode())): - yield header + part, observed_at + pieces = split_tokens(part, header, token_budget) if token_budget is not None else [part] + for piece in pieces: + yield header + piece, observed_at diff --git a/agent_memory_leaderboard/src/journal.py b/agent_memory_leaderboard/src/journal.py index 4a44096..87d378a 100644 --- a/agent_memory_leaderboard/src/journal.py +++ b/agent_memory_leaderboard/src/journal.py @@ -1,6 +1,7 @@ """Durable idempotency state, with process-safe locks on a shared local disk.""" import fcntl +import json import math import sqlite3 import threading @@ -69,6 +70,10 @@ def __init__( PRIMARY KEY (request_key, position) ) """) + request_columns = {row[1] for row in db.execute("PRAGMA table_info(requests)")} + if "render_policy" not in request_columns: + db.execute("""ALTER TABLE requests ADD COLUMN render_policy TEXT NOT NULL + DEFAULT '{"version":1}'""") columns = {row[1] for row in db.execute("PRAGMA table_info(writes)")} if "protocol" not in columns: db.execute("ALTER TABLE writes ADD COLUMN protocol TEXT NOT NULL DEFAULT 'legacy'") @@ -208,7 +213,7 @@ def lock(self, key: str): raise @staticmethod - def _begin(db, key: str, fingerprint: str) -> bool: + def _begin(db, key: str, fingerprint: str, render_policy: dict | None = None) -> bool: row = db.execute( "SELECT fingerprint, complete FROM requests WHERE key = ?", (key,) ).fetchone() @@ -220,13 +225,47 @@ def _begin(db, key: str, fingerprint: str) -> bool: "restore the original payload", ) return bool(row[1]) - db.execute("INSERT INTO requests(key, fingerprint) VALUES (?, ?)", (key, fingerprint)) + db.execute( + "INSERT INTO requests(key, fingerprint, render_policy) VALUES (?, ?, ?)", + (key, fingerprint, json.dumps(render_policy or {"version": 1}, sort_keys=True)), + ) return False def begin(self, key: str, fingerprint: str) -> bool: with self.connect("begin") as db: return self._begin(db, key, fingerprint) + def request_policy(self, key: str, fingerprint: str, default: dict) -> dict | None: + """Read under lock(key); the first prepare_write persists the chosen policy. + + Existing rows migrate to byte-only v1. Never reinterpret old positions + or rewrite old receipts when the default budget changes. None is a + completed replay (after checking the payload fingerprint). + """ + with self.connect("state") as db: + row = db.execute( + "SELECT fingerprint, complete, render_policy FROM requests WHERE key = ?", (key,) + ).fetchone() + if row is None: + return default + if row[0] != fingerprint: + raise AdapterError( + 400, + "request_id was already used with a different payload; " + "restore the original payload", + ) + if row[1]: + return None + try: + policy = json.loads(row[2]) + except (TypeError, ValueError): + raise AdapterError( + 503, "Invalid journal rendering policy; check adapter version" + ) from None + if not isinstance(policy, dict): + raise AdapterError(503, "Invalid journal rendering policy; check adapter version") + return policy + @staticmethod def _state(db, key: str, position: int) -> tuple[str, str | None, str]: row = db.execute( @@ -251,7 +290,7 @@ def _record(db, key, position, state, memory_id, protocol): ) def prepare_write( - self, key: str, fingerprint: str, position: int + self, key: str, fingerprint: str, position: int, *, render_policy: dict | None = None ) -> tuple[str, str | None, str] | None: """Register the request and persist send intent in one durable transaction. @@ -259,7 +298,7 @@ def prepare_write( Legacy unknown outcomes must never be converted to atomic receipts. """ with self.connect("prepare_write") as db: - if self._begin(db, key, fingerprint): + if self._begin(db, key, fingerprint, render_policy): return None state, memory_id, protocol = self._state(db, key, position) if protocol not in {LEGACY_PROTOCOL, IDEMPOTENT_PROTOCOL}: diff --git a/agent_memory_leaderboard/src/service.py b/agent_memory_leaderboard/src/service.py index 1167dfc..d2886a5 100644 --- a/agent_memory_leaderboard/src/service.py +++ b/agent_memory_leaderboard/src/service.py @@ -5,6 +5,7 @@ from .budget import catalog_retry_budget, request_budget from .budget import remaining as budget_remaining from .config import Settings +from .embedding_tokens import TOKENIZER_SHA256 from .errors import AdapterError from .ingestion import message_parts from .journal import LEGACY_PROTOCOL, Journal @@ -40,11 +41,24 @@ def add(self, request: AddRequest) -> AddResponse: key = digest([self.settings.memoria_url.rstrip("/"), owner, request.request_id]) fingerprint = digest(request.model_dump()) with self.journal.lock(key): + policy = self.journal.request_policy( + key, + fingerprint, + { + "version": 2, + "tokenizer_sha256": TOKENIZER_SHA256, + "max_tokens": self.settings.embedding_token_budget, + }, + ) # One-item lookahead identifies the last chunk without buffering # the entire request, so its stored marker and completion commit together. - bodies = chain(self.write_bodies(request, key), [None]) + bodies = chain( + self.write_bodies(request, key, policy) if policy is not None else [], [None] + ) for position, (body, following) in enumerate(pairwise(bodies)): - prepared = self.journal.prepare_write(key, fingerprint, position) + prepared = self.journal.prepare_write( + key, fingerprint, position, render_policy=policy + ) if prepared is None: break self.write(owner, key, position, body, prepared, last=following is None) @@ -55,10 +69,27 @@ def add(self, request: AddRequest) -> AddResponse: session_id=request.session_id, ) - def write_bodies(self, request: AddRequest, key: str): + def write_bodies(self, request: AddRequest, key: str, policy: dict | None = None): + # Explicit default keeps historical audit helpers and recovery fixtures v1. + policy = policy if policy is not None else {"version": 1} + if policy == {"version": 1} and type(policy["version"]) is int: + token_budget = None + elif ( + set(policy) == {"version", "tokenizer_sha256", "max_tokens"} + and type(policy["version"]) is int + and policy["version"] == 2 + and policy["tokenizer_sha256"] == TOKENIZER_SHA256 + and type(policy["max_tokens"]) is int + and 128 <= policy["max_tokens"] <= 8192 + ): + token_budget = policy["max_tokens"] + else: + raise AdapterError(503, "Unsupported journal rendering policy; check adapter version") session = digest([self.settings.namespace, request.user_id, request.session_id]) for message_index, message in enumerate(request.messages): - for part_index, (content, observed_at) in enumerate(message_parts(message)): + for part_index, (content, observed_at) in enumerate( + message_parts(message, token_budget=token_budget) + ): marker = digest([key, message_index, part_index]) body = { "content": content, diff --git a/agent_memory_leaderboard/tests/test_ingestion.py b/agent_memory_leaderboard/tests/test_ingestion.py index fdb7378..01bd2de 100644 --- a/agent_memory_leaderboard/tests/test_ingestion.py +++ b/agent_memory_leaderboard/tests/test_ingestion.py @@ -44,7 +44,7 @@ def test_message_chunks_include_header_in_byte_limit_and_join_text_parts(): ], } ) - parts = list(message_parts(message)) + parts = list(message_parts(message, token_budget=None)) header = "[user at 1970-01-01T00:00:00.000Z]\n" assert len(parts) == 3 assert all( diff --git a/agent_memory_leaderboard/tests/test_journal_concurrency.py b/agent_memory_leaderboard/tests/test_journal_concurrency.py index 7be9150..1214272 100644 --- a/agent_memory_leaderboard/tests/test_journal_concurrency.py +++ b/agent_memory_leaderboard/tests/test_journal_concurrency.py @@ -28,7 +28,10 @@ def short_sqlite_timeout(*args, **kwargs): def hold_write(): with holder.connect("record") as db: - db.execute("INSERT INTO requests VALUES ('holder', 'fingerprint', 0)") + db.execute( + "INSERT INTO requests(key, fingerprint, complete) VALUES (?, ?, 0)", + ("holder", "fingerprint"), + ) entered.set() assert release.wait(5) @@ -114,7 +117,10 @@ def connect(*args, **kwargs): def _child_holding_transaction(directory, entered): journal = Journal(directory) with journal.connect("record") as db: - db.execute("INSERT INTO requests VALUES ('uncommitted', 'fingerprint', 0)") + db.execute( + "INSERT INTO requests(key, fingerprint, complete) VALUES (?, ?, 0)", + ("uncommitted", "fingerprint"), + ) entered.set() time.sleep(60) @@ -143,7 +149,10 @@ def test_process_exit_releases_gate_and_rolls_back_uncommitted_state(tmp_path): def test_operation_failure_releases_slot_and_preserves_rollback(tmp_path): journal = Journal(tmp_path) with pytest.raises(RuntimeError), journal.connect("record") as db: - db.execute("INSERT INTO requests VALUES ('rolled-back', 'fingerprint', 0)") + db.execute( + "INSERT INTO requests(key, fingerprint, complete) VALUES (?, ?, 0)", + ("rolled-back", "fingerprint"), + ) raise RuntimeError("synthetic failure") assert journal.begin("rolled-back", "fingerprint") is False diff --git a/agent_memory_leaderboard/tests/test_journal_metrics.py b/agent_memory_leaderboard/tests/test_journal_metrics.py index 81f589b..585664a 100644 --- a/agent_memory_leaderboard/tests/test_journal_metrics.py +++ b/agent_memory_leaderboard/tests/test_journal_metrics.py @@ -51,7 +51,8 @@ def test_default_add_summary_includes_idempotent_replay_without_payloads(tmp_pat summaries = events(caplog, "journal_add_summary") assert [e["http_status"] for e in summaries] == [200, 200, 400] assert summaries[0]["operations"]["record"]["count"] == 2 - assert summaries[1]["operations"]["prepare_write"]["count"] == 1 + assert summaries[1]["operations"]["state"]["count"] == 1 + assert "prepare_write" not in summaries[1]["operations"] assert "record" not in summaries[1]["operations"] assert len({e["add_trace"] for e in summaries}) == 3 assert all(0 <= e["journal_fraction"] <= 1.001 for e in summaries) @@ -180,7 +181,10 @@ def test_body_exception_rolls_back_without_logging_exception_text(tmp_path, capl journal = Journal(tmp_path, slow_ms=1e9) caplog.set_level(logging.INFO, logger="uvicorn.error") with pytest.raises(RuntimeError), journal.connect("begin") as db: - db.execute("INSERT INTO requests VALUES (?, ?, 0)", ("private-request", "private-body")) + db.execute( + "INSERT INTO requests(key, fingerprint, complete) VALUES (?, ?, 0)", + ("private-request", "private-body"), + ) raise RuntimeError("synthetic-private-exception") with sqlite3.connect(journal.path) as db: assert db.execute("SELECT count(*) FROM requests").fetchone() == (0,) diff --git a/agent_memory_leaderboard/tests/test_ready.py b/agent_memory_leaderboard/tests/test_ready.py index 575c3d5..f20bcc5 100644 --- a/agent_memory_leaderboard/tests/test_ready.py +++ b/agent_memory_leaderboard/tests/test_ready.py @@ -156,7 +156,10 @@ def test_readiness_does_not_queue_behind_an_active_journal_transaction(settings) with TestClient(create_app(settings, client=http)) as client: with ThreadPoolExecutor() as pool: with journal.connect("record") as db: - db.execute("INSERT INTO requests VALUES ('uncommitted', 'fingerprint', 0)") + db.execute( + "INSERT INTO requests(key, fingerprint, complete) VALUES (?, ?, 0)", + ("uncommitted", "fingerprint"), + ) probe = pool.submit(client.get, "/ready") assert probe.result(timeout=1).status_code == 200 diff --git a/agent_memory_leaderboard/tests/test_token_chunks.py b/agent_memory_leaderboard/tests/test_token_chunks.py new file mode 100644 index 0000000..079fa11 --- /dev/null +++ b/agent_memory_leaderboard/tests/test_token_chunks.py @@ -0,0 +1,237 @@ +"""Real pinned tokenizer, synthetic inputs only; no test-time network access.""" + +import hashlib +import json +import sqlite3 +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace + +import httpx +import pytest +from test_endpoints import FakeMemoria, addition + +from src.config import Settings +from src.embedding_tokens import DEFAULT_TOKEN_BUDGET, TOKENIZER_SHA256, token_count +from src.errors import AdapterError +from src.ingestion import MAX_MEMORY_BYTES, message_parts +from src.journal import Journal +from src.memoria import Memoria +from src.models import AddRequest, Message +from src.service import MemoryAdapter, digest + + +def dense_text(): + return "Synthetic event identifiers: " + " ".join( + hashlib.sha256(f"synthetic-event-{i}".encode()).hexdigest()[:32] for i in range(900) + ) + + +def test_reported_byte_valid_token_overflow_is_split_without_loss(): + message = Message(role="user", content=dense_text(), timestamp=1704067200000) + legacy = list(message_parts(message, token_budget=None)) + assert len(legacy) == 1 + assert len(legacy[0][0].encode()) < MAX_MEMORY_BYTES + assert token_count(legacy[0][0]) > 8192 + parts = list(message_parts(message)) + assert len(parts) > 1 + header = "[user at 2024-01-01T00:00:00.000Z]\n" + assert "".join(p.removeprefix(header) for p, _ in parts) == message.content + assert all(token_count(p) <= DEFAULT_TOKEN_BUDGET for p, _ in parts) + assert all(len(p.encode()) <= MAX_MEMORY_BYTES for p, _ in parts) + assert {t for _, t in parts} == {"2024-01-01T00:00:00.000Z"} + + +@pytest.mark.parametrize( + "text", + [ + " spaces\n\n", + "e\u0301é中文🙂𐐀\n", + " special ", + "a" * 333, + ], +) +def test_original_text_preserved_even_when_tokenizer_normalizes(text): + message = Message(role="assistant", content=text * 200) + header = "[assistant]\n" + parts = list(message_parts(message, token_budget=128)) + assert "".join(p[len(header) :] for p, _ in parts) == message.content + assert all(token_count(p) <= 128 and len(p.encode()) <= MAX_MEMORY_BYTES for p, _ in parts) + + +def test_short_message_unchanged_and_concurrent_tokenization_deterministic(): + message = Message(role="user", content="A short fact.", timestamp=-1) + assert list(message_parts(message)) == list(message_parts(message, token_budget=None)) + large = Message(role="assistant", content=dense_text()) + with ThreadPoolExecutor(max_workers=16) as executor: + outputs = list(executor.map(lambda _: list(message_parts(large)), range(16))) + assert all(result == outputs[0] for result in outputs) + + +@pytest.fixture +def adapter(tmp_path): + settings = Settings( + memoria_url="http://memoria.test", memoria_master_key="upstream-secret", state_dir=tmp_path + ) + upstream = FakeMemoria() + with httpx.Client(transport=httpx.MockTransport(upstream)) as http: + yield MemoryAdapter(settings, Memoria(settings, http), Journal(tmp_path)), upstream + + +def request_key(service, request): + return digest( + [service.settings.memoria_url, service.owner(request.user_id), request.request_id] + ) + + +def test_partial_retry_after_restart_keeps_persisted_budget_and_exact_bodies(adapter, monkeypatch): + service, upstream = adapter + request = AddRequest.model_validate( + addition(messages=[{"role": "user", "content": dense_text()}]) + ) + original = service.write + calls = 0 + + def crash(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise RuntimeError("synthetic interruption before second POST") + return original(*args, **kwargs) + + monkeypatch.setattr(service, "write", crash) + with pytest.raises(RuntimeError): + service.add(request) + assert len(upstream.memories) == 1 + with sqlite3.connect(service.journal.path) as db: + policy = json.loads(db.execute("SELECT render_policy FROM requests").fetchone()[0]) + assert policy == {"version": 2, "tokenizer_sha256": TOKENIZER_SHA256, "max_tokens": 7680} + expected = list(service.write_bodies(request, request_key(service, request), policy)) + changed = replace(service.settings, embedding_token_budget=256) + restarted = MemoryAdapter(changed, service.memoria, Journal(changed.state_dir)) + restarted.add(request) + assert len(upstream.memories) == len(expected) + assert sorted(m["content"] for m in upstream.memories) == sorted(b["content"] for b in expected) + before = len(upstream.calls) + restarted.add(request) + assert len(upstream.calls) == before + # A new request uses the changed setting, rather than inheriting another request's policy. + newer = request.model_copy(update={"request_id": "another-request"}) + restarted.add(newer) + with sqlite3.connect(service.journal.path) as db: + row = db.execute( + "SELECT render_policy FROM requests WHERE key=?", (request_key(service, newer),) + ).fetchone() + assert json.loads(row[0])["max_tokens"] == 256 + + +def test_migrate_old_database_pins_v1_and_completed_replay_skips_rendering(tmp_path, monkeypatch): + request = AddRequest.model_validate(addition()) + settings = Settings( + memoria_url="http://memoria.test", memoria_master_key="upstream-secret", state_dir=tmp_path + ) + key = digest( + [settings.memoria_url, digest([settings.namespace, request.user_id]), request.request_id] + ) + with sqlite3.connect(tmp_path / "requests.sqlite3") as db: + db.execute( + "CREATE TABLE requests(key TEXT PRIMARY KEY, fingerprint TEXT, complete INTEGER)" + ) + db.execute("INSERT INTO requests VALUES(?,?,1)", (key, digest(request.model_dump()))) + journal = Journal(tmp_path) + with sqlite3.connect(journal.path) as db: + assert json.loads(db.execute("SELECT render_policy FROM requests").fetchone()[0]) == { + "version": 1 + } + service = MemoryAdapter(settings, None, journal) + monkeypatch.setattr( + service, "write_bodies", lambda *_: pytest.fail("completed replay rendered") + ) + assert service.add(request).success + with pytest.raises(AdapterError) as exc: + service.add(request.model_copy(update={"session_id": "changed"})) + assert exc.value.status == 400 + + +def test_incomplete_old_request_retains_byte_boundaries(adapter): + service, _ = adapter + request = AddRequest.model_validate( + addition(messages=[{"role": "user", "content": dense_text()}]) + ) + key = request_key(service, request) + service.journal.begin(key, digest(request.model_dump())) + policy = service.journal.request_policy(key, digest(request.model_dump()), {"version": 2}) + assert policy == {"version": 1} + bodies = list(service.write_bodies(request, key, policy)) + assert len(bodies) == 1 and token_count(bodies[0]["content"]) > 8192 + + +@pytest.mark.parametrize( + "policy", + [ + {"version": 3}, + {"version": True}, + {"version": 2, "max_tokens": 1}, + {"version": 2, "max_tokens": 7680, "tokenizer_sha256": "different"}, + ], +) +def test_unknown_policy_fails_before_backend_write(adapter, policy): + service, upstream = adapter + request = AddRequest.model_validate(addition()) + key = request_key(service, request) + service.journal.begin(key, digest(request.model_dump())) + with sqlite3.connect(service.journal.path) as db: + db.execute("UPDATE requests SET render_policy=?", (json.dumps(policy),)) + with pytest.raises(AdapterError) as exc: + service.add(request) + assert exc.value.status == 503 and not upstream.calls + + +@pytest.mark.parametrize("budget", [0, 127, 8193, True, 7680.0]) +def test_invalid_token_budget_rejected(budget): + with pytest.raises(ValueError): + Settings( + memoria_url="http://memoria.test", + memoria_master_key="upstream-secret", + embedding_token_budget=budget, + ) + + +def test_add_and_visibility_queries_obey_real_token_budget(adapter): + service, upstream = adapter + checked = {"store": 0, "visibility": 0} + + def reject_oversize(request, body, _owner): + if request.url.path == "/v1/memories/idempotent": + checked["store"] += 1 + text = body["content"] + elif request.url.path == "/v1/memories/search": + checked["visibility"] += 1 + text = body["query"] + else: + return None + if token_count(text) > DEFAULT_TOKEN_BUDGET: + return httpx.Response(500) + return None + + upstream.hook = reject_oversize + request = AddRequest.model_validate( + addition(messages=[{"role": "user", "content": dense_text()}]) + ) + assert service.add(request).success + assert checked["store"] == checked["visibility"] > 1 + assert "".join(m["content"].removeprefix("[user]\n") for m in upstream.memories) == dense_text() + + +def test_tokenizer_asset_missing_or_modified_fails_closed(tmp_path, monkeypatch): + from src import embedding_tokens + + embedding_tokens.load_tokenizer.cache_clear() + monkeypatch.setenv("ADAPTER_TOKENIZER_PATH", str(tmp_path / "tokenizer.json")) + try: + with pytest.raises(RuntimeError, match="missing"): + embedding_tokens.load_tokenizer() + (tmp_path / "tokenizer.json").write_text("{}") + with pytest.raises(RuntimeError, match="checksum"): + embedding_tokens.load_tokenizer() + finally: + embedding_tokens.load_tokenizer.cache_clear() diff --git a/agent_memory_leaderboard/uv.lock b/agent_memory_leaderboard/uv.lock index 7c4b1cc..d772b31 100644 --- a/agent_memory_leaderboard/uv.lock +++ b/agent_memory_leaderboard/uv.lock @@ -11,6 +11,7 @@ dependencies = [ { name = "fastapi" }, { name = "httpx" }, { name = "pydantic" }, + { name = "tokenizers" }, { name = "uvicorn" }, ] @@ -32,6 +33,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.28,<1" }, { name = "pydantic", specifier = ">=2.10,<3" }, { name = "tiktoken", marker = "extra == 'evaluation'", specifier = ">=0.12,<1" }, + { name = "tokenizers", specifier = "==0.22.2" }, { name = "uvicorn", specifier = ">=0.34,<1" }, ] provides-extras = ["evaluation"] @@ -248,6 +250,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/b6/78aaf9141fb46742928c113f3cf6ef2259d538cb02b604b7656c1dc9883c/fastapi-0.142.2-py3-none-any.whl", hash = "sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b", size = 144409, upload-time = "2026-09-30T09:51:09.8Z" }, ] +[[package]] +name = "filelock" +version = "4.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/28/3893324bcadf8f89b1616637468d766af3f625008c0735bf704fdfa86617/filelock-4.2.4.tar.gz", hash = "sha256:c6d8663535e1650ee430507b4ffbac7b464dd64b7647ea0112a9f406499d163d", size = 652444, upload-time = "2026-10-19T01:57:51.001Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/1b/5e74289a1878e666bb4b65fe5805a04cafcac2aa5f519ef1ead67ef91c56/filelock-4.2.4-py3-none-any.whl", hash = "sha256:301241438c57936ef93f4243a94f2559022268bfcf9b375a9a72eb0f8db8358e", size = 162475, upload-time = "2026-10-19T01:57:49.419Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/77/cd/9be253869fc42e764de7f3dedd6969af7d44ff9c3375214a3442a6f3fc08/fsspec-2026.9.0.tar.gz", hash = "sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe", size = 333545, upload-time = "2026-09-18T17:50:42.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/c0/a98505f18594f1bce828bb159cec0fcf9860562f1a2c85913409fc8f3d9e/fsspec-2026.9.0-py3-none-any.whl", hash = "sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f", size = 221738, upload-time = "2026-09-18T17:50:41.341Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -257,6 +277,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "hf-xet" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/18/d91c20f815df6cc496a5d0ad8db6d274dad5c0cfcd59babfa2440a9aba44/hf_xet-1.8.0.tar.gz", hash = "sha256:10ed23b51dacf7917d64f335c25048deb470efa755e9989b218fc0531386138e", size = 1013014, upload-time = "2026-10-30T10:04:33.741Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/18/db7ebc58820d82fd5506aa405a75611e7a4398f573dc436f7387603bf63d/hf_xet-1.8.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c74e3751cbcdae9222c5170e8d788e83201a03dc5e5f4308a20f17a5b6023b56", size = 3883873, upload-time = "2026-10-30T10:04:03.24Z" }, + { url = "https://files.pythonhosted.org/packages/37/bd/6c9ed89aafbeb4a7ee0d76d6b7b9365221e3e7ee47958aeecdc2c60a0511/hf_xet-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:56735c2d4f416d7473fc986586b26a8756e067832ca4b7fdaad32a0d6f2ce740", size = 3594510, upload-time = "2026-10-30T10:04:05.364Z" }, + { url = "https://files.pythonhosted.org/packages/0e/1f/c26154b3a053b637c0984db0db9fdacbbfc257bdc2e95e9bc0782b745b0c/hf_xet-1.8.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3ecfe2934448cca55a4c42732b60f985266aecb241905945219b3d388aceb7ba", size = 4281780, upload-time = "2026-10-30T10:04:07.35Z" }, + { url = "https://files.pythonhosted.org/packages/e3/9e/d84c31f8e3acc286c9c6a4fd85f7f332e3db40636595f0de5ad776b69beb/hf_xet-1.8.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:7edeb21c5d381903ee5f4857fc746fa7b52a4b374035e8811ecfb67ee27add0c", size = 4060223, upload-time = "2026-10-30T10:04:09.029Z" }, + { url = "https://files.pythonhosted.org/packages/a4/70/c142e4ad8a679ff6ffe40964a9d017fa0211aabca9ca4d3d547c3a2fc165/hf_xet-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:07a06d7481702616726c2d8578cfb00052a5b3f21beed6a674592bc837a10330", size = 4263619, upload-time = "2026-10-30T10:04:11.085Z" }, + { url = "https://files.pythonhosted.org/packages/9d/08/92e03e3ac4eaeab046359093c447c03c8c30b577c2b1e1687dc747654f1c/hf_xet-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f42940869572dc2b05dfe06f4cc398c6291503e086ddc4eddfb7a6ab9409ad01", size = 4493381, upload-time = "2026-10-30T10:04:13.022Z" }, + { url = "https://files.pythonhosted.org/packages/61/63/a512b2906d0f92b5d325f58169d1fd8c3202a9ee61bd8405ec78c67e5315/hf_xet-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:23c262e9f5e137b18dadbce60becfeea84156b0d43119c320bffe260d32cd2fa", size = 3901344, upload-time = "2026-10-30T10:04:15.051Z" }, + { url = "https://files.pythonhosted.org/packages/2d/5e/b453f82690cb36f176c4b01e64d6d32757fabbf34dbb9bcbdf747e429a79/hf_xet-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:62b056aedb89ac809f78940baf465fe553b70c8b25ee56890fbab3039fffec1b", size = 3731681, upload-time = "2026-10-30T10:04:16.855Z" }, + { url = "https://files.pythonhosted.org/packages/97/74/d89d5235524bc14d8519e12ca8bc5204f45009ae83d63a410f52c29eb4a8/hf_xet-1.8.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e00d8f60170cd3dd55d98306e41dbaf28b27ff1bcfefbfa84279d197221be83e", size = 3875002, upload-time = "2026-10-30T10:04:18.958Z" }, + { url = "https://files.pythonhosted.org/packages/d4/f2/ef25e2085e92dcaeaa675331e443923aed2006f6fe2d984e15d8b8d40766/hf_xet-1.8.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:3e8934784425781f0f779b91044df441f3a7af4dab47375d7e3aeaecbb8eecb4", size = 3600947, upload-time = "2026-10-30T10:04:20.906Z" }, + { url = "https://files.pythonhosted.org/packages/f8/1e/1e8ecc05ef034e9a95c18ed32d2ec07905a7cd2ca611a9d88e25ee1f4850/hf_xet-1.8.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:29dab7c904cd1914a0439c6e1306c12c7497c8bd32290fbe563ebf51ca4b6d6d", size = 4292483, upload-time = "2026-10-30T10:04:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/35/c4/127a1548df019ea4c80f06e685f263e146075143335a02e78a3c41dbc4eb/hf_xet-1.8.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8800e3444770f2c630b361bc6d5c6ebb770b7830646cf632fbb79d0e5bbc9e7f", size = 4069340, upload-time = "2026-10-30T10:04:24.664Z" }, + { url = "https://files.pythonhosted.org/packages/a5/d5/281cd82736c3e181e597588589d2204d3a5c388a464d4a2a8c546995bae9/hf_xet-1.8.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a584a7b3268da62f23c3187cc7201ca01f7b610985a36b1b6119c66bccb0383c", size = 4272344, upload-time = "2026-10-30T10:04:26.519Z" }, + { url = "https://files.pythonhosted.org/packages/48/77/886f09e91a5eadb054ce03b8855323cd2a72f20db76ab4119e0ecda65eef/hf_xet-1.8.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1ebaf248c9bef8bb0c9d1d8d71db1fdf0cac14f922cc7a936a81adbf12eb5f01", size = 4500689, upload-time = "2026-10-30T10:04:28.394Z" }, + { url = "https://files.pythonhosted.org/packages/97/4d/2c76a992b370ff77d0b5219275779f8c9a7dabb8ee649deb265dfed84d0f/hf_xet-1.8.0-cp38-abi3-win_amd64.whl", hash = "sha256:92188598dedc1e1f5609bd718d4db8687a6c89915553912571e2f587892eecec", size = 3911559, upload-time = "2026-10-30T10:04:30.165Z" }, + { url = "https://files.pythonhosted.org/packages/87/91/fdb1454a43eff8011047d176ada2f0fb4926cf3d74b87357ff35793f6d98/hf_xet-1.8.0-cp38-abi3-win_arm64.whl", hash = "sha256:42b66fdc688eaa4ff3eb77dc9275e5a0f1258ecc9fdd083ddaf43bb49bec925b", size = 3738458, upload-time = "2026-10-30T10:04:31.882Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -285,6 +329,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "huggingface-hub" +version = "1.33.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/2a/484d112c0d8fc5f665d7b65137ac9cdb2953c982391598c3597968a12ee7/huggingface_hub-1.33.0.tar.gz", hash = "sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e", size = 1043548, upload-time = "2026-09-24T09:49:29.971Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/16/963096d224b80909432dc16561a615fd33d2d13beef3ce4c63fa25e40867/huggingface_hub-1.33.0-py3-none-any.whl", hash = "sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899", size = 846435, upload-time = "2026-09-24T09:49:28.059Z" }, +] + [[package]] name = "idna" version = "3.20" @@ -448,6 +512,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + [[package]] name = "regex" version = "2026.9.29" @@ -659,6 +769,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/5e/9b01afd037bfa22a0033963fa091e0f75b6fb15cd85bffb42ff86e697323/tiktoken-0.14.0-cp315-cp315t-win_amd64.whl", hash = "sha256:561e7580f84a79859af1ef6f676968e9030fcc3fe195700b15235bca64f009c9", size = 987929, upload-time = "2026-08-17T19:49:38.947Z" }, ] +[[package]] +name = "tokenizers" +version = "0.22.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, + { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, + { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, + { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, + { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, + { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, + { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, + { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, + { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, + { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, +] + +[[package]] +name = "tqdm" +version = "4.70.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/ea/b2a5bd54b28a324dae8211928b2d730b6547500342c7e6c6dea08bd0a485/tqdm-4.70.1.tar.gz", hash = "sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4", size = 171846, upload-time = "2026-09-11T07:25:16.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73", size = 80199, upload-time = "2026-09-11T07:25:14.599Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" From 5a91c295fecd0d6fcc540b502f4ac4e07baff9f8 Mon Sep 17 00:00:00 2001 From: lr90 Date: Fri, 30 Oct 2026 23:41:17 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix(adapter):=20=E5=AF=B9=E9=BD=90=E8=AF=84?= =?UTF-8?q?=E6=B5=8B=E5=88=86=E7=89=87=E7=AD=96=E7=95=A5=E5=B9=B6=E9=98=BB?= =?UTF-8?q?=E6=AD=A2=E4=B8=8D=E5=AE=89=E5=85=A8=E5=9B=9E=E6=BB=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/aml-benchmark.yml | 5 + .github/workflows/aml-ci.yml | 5 + agent_memory_leaderboard/Dockerfile | 4 +- agent_memory_leaderboard/README.md | 23 +- .../scripts/content_diagnostics_demo.py | 11 +- agent_memory_leaderboard/src/app.py | 11 + agent_memory_leaderboard/src/benchmark.py | 87 ++++-- agent_memory_leaderboard/src/config.py | 15 +- .../src/embedding_tokens.py | 24 ++ agent_memory_leaderboard/src/evidence.py | 23 +- agent_memory_leaderboard/src/ingestion.py | 7 +- agent_memory_leaderboard/src/journal.py | 32 ++- agent_memory_leaderboard/src/service.py | 45 +-- .../tests/test_benchmark.py | 3 + .../tests/test_benchmark_evaluation.py | 5 + .../tests/test_catalog_retry.py | 4 +- .../tests/test_content_diagnostics.py | 12 +- .../tests/test_evidence.py | 15 +- .../tests/test_idempotent_writes.py | 10 +- .../tests/test_ingestion.py | 2 +- .../tests/test_journal_transactions.py | 4 +- .../tests/test_longmemeval_pipeline.py | 3 + .../tests/test_longmemeval_reader.py | 13 +- .../tests/test_render_policy.py | 256 ++++++++++++++++++ .../tests/test_token_chunks.py | 10 +- .../tests/test_write_visibility.py | 4 +- 26 files changed, 532 insertions(+), 101 deletions(-) create mode 100644 agent_memory_leaderboard/tests/test_render_policy.py diff --git a/.github/workflows/aml-benchmark.yml b/.github/workflows/aml-benchmark.yml index 13360c8..d889442 100644 --- a/.github/workflows/aml-benchmark.yml +++ b/.github/workflows/aml-benchmark.yml @@ -71,6 +71,11 @@ jobs: - name: Install locked benchmark dependencies timeout-minutes: 5 run: uv sync --locked --extra evaluation + - name: Cache pinned embedding tokenizer + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: agent_memory_leaderboard/src/assets/tokenizer.json + key: bge-m3-${{ hashFiles('agent_memory_leaderboard/src/embedding_tokens.py') }} - name: Prepare pinned embedding tokenizer run: uv run --no-sync python -m src.embedding_tokens - name: Run Add/Search and selected evaluation mode diff --git a/.github/workflows/aml-ci.yml b/.github/workflows/aml-ci.yml index 9f23625..b5ed43b 100644 --- a/.github/workflows/aml-ci.yml +++ b/.github/workflows/aml-ci.yml @@ -29,6 +29,11 @@ jobs: enable-cache: true cache-dependency-glob: agent_memory_leaderboard/uv.lock - run: uv sync --locked --extra evaluation + - name: Cache pinned embedding tokenizer + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: agent_memory_leaderboard/src/assets/tokenizer.json + key: bge-m3-${{ hashFiles('agent_memory_leaderboard/src/embedding_tokens.py') }} - run: uv run --no-sync python -m src.embedding_tokens - run: uv run --no-sync ruff check src tests benchmarks scripts/benchmark_ci.py ../deploy/scripts ../deploy/tests - run: uv run --no-sync pytest -q diff --git a/agent_memory_leaderboard/Dockerfile b/agent_memory_leaderboard/Dockerfile index 3177327..9bd0834 100644 --- a/agent_memory_leaderboard/Dockerfile +++ b/agent_memory_leaderboard/Dockerfile @@ -8,8 +8,10 @@ RUN if [ "$INSTALL_EXPERIMENT_DEPS" = "true" ]; then \ pip install --no-cache-dir --require-hashes -r requirements.lock; \ fi RUN useradd --uid 1000 --create-home adapter && mkdir /app/var && chown adapter /app/var -COPY src ./src +COPY src/assets ./src/assets +COPY src/embedding_tokens.py ./src/embedding_tokens.py RUN python -m src.embedding_tokens +COPY src ./src USER adapter EXPOSE 8000 CMD ["uvicorn", "src.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8000"] diff --git a/agent_memory_leaderboard/README.md b/agent_memory_leaderboard/README.md index 3d6b460..60c2206 100644 --- a/agent_memory_leaderboard/README.md +++ b/agent_memory_leaderboard/README.md @@ -667,11 +667,22 @@ request still uses its original byte-only chunks, even if one exceeds the model limit. Never clear its journal or reinterpret existing positions to force a retry. Coordinate an explicit replay/migration plan for such requests. Keep one adapter worker/replica and stop the old version before upgrading. Do not roll -back to a byte-only binary with unfinished v2 requests: old code cannot honor -pinned token boundaries. Preserve the journal and fixed model/tokenizer for the -whole run. Ordinary Search queries are unchanged; the write-visibility search +back to a byte-only binary with unfinished v2 requests. Token-policy receipts use +an unknown-to-old-code protocol, and SQLite triggers reject v1 writes to any new +position of a v2 request. Old versions fail with 503 instead of silently skipping +content. These guards require retaining the journal schema and triggers. Preserve +the journal and fixed model/tokenizer for the whole run. Ordinary Search queries are unchanged; the write-visibility search uses each newly bounded chunk. -Local source manifests use the new default rendering version for attribution. -They must not be treated as an exact reproduction of old v1 runs or runs using -a different token budget. +The local benchmark reads each Add request's persisted policy from the authenticated, +read-only `POST /diagnostics/render-policy` endpoint (same Add body and credentials). +It records per-message policies in source manifests, including mixed v1/v2 requests +and resumed runs. Missing/invalid policies stop attribution explicitly instead of +silently using a default. To evaluate a known old byte-only adapter lacking this +endpoint, explicitly pass `--legacy-byte-rendering`; only a 404 permits that +fallback. The flag is part of the resume contract and rejects a detected v2 policy. +Pre-change benchmark checkpoints lack this policy contract and cannot be resumed by +this runner; preserve them for analysis and use the matching runner or a new run. + +The tokenizer download has its own Docker layer, independent of other application +source changes. CI caches the public asset and always verifies its checksum. diff --git a/agent_memory_leaderboard/scripts/content_diagnostics_demo.py b/agent_memory_leaderboard/scripts/content_diagnostics_demo.py index 9c5f6ec..2446c15 100644 --- a/agent_memory_leaderboard/scripts/content_diagnostics_demo.py +++ b/agent_memory_leaderboard/scripts/content_diagnostics_demo.py @@ -12,6 +12,7 @@ from src import benchmark from src.checkpoint import save from src.content_diagnostics import ContentDiagnostics +from src.embedding_tokens import DEFAULT_TOKEN_BUDGET, token_policy from src.evaluation import AnswerEvaluator, EvaluationConfig from src.ingestion import message_parts from src.models import Message @@ -62,9 +63,13 @@ def run(output_dir, scenario): def respond(request): body = json.loads(request.content) if request.content else {} if request.url.host == "adapter.invalid": + if request.url.path == "/diagnostics/render-policy": + return httpx.Response(200, json={"policy": token_policy(DEFAULT_TOKEN_BUDGET)}) if request.url.path == "/add": for message in body["messages"]: - for content, _ in message_parts(Message.model_validate(message)): + for content, _ in message_parts( + Message.model_validate(message), token_budget=DEFAULT_TOKEN_BUDGET + ): memories.append({"id": f"synthetic-{len(memories)}", "content": content}) return httpx.Response( 200, @@ -106,7 +111,9 @@ def respond(request): if scenario == "budget-exclusion": sample = json.loads(args.dataset.read_text())[0] message = next(benchmark.batches(sample["conversation"]))[2][0] - content = next(message_parts(Message.model_validate(message)))[0] + content = next( + message_parts(Message.model_validate(message), token_budget=DEFAULT_TOKEN_BUDGET) + )[0] evaluator = AnswerEvaluator(EvaluationConfig.from_env(), "locomo") _, _, tokens = evaluator.context(sample["qa"][0], [content]) os.environ["BENCHMARK_INPUT_BUDGET"] = str(tokens) diff --git a/agent_memory_leaderboard/src/app.py b/agent_memory_leaderboard/src/app.py index 49c0430..c3bb756 100644 --- a/agent_memory_leaderboard/src/app.py +++ b/agent_memory_leaderboard/src/app.py @@ -128,6 +128,17 @@ async def validation_error(request: Request, error: RequestValidationError): issues = [{"loc": e["loc"], "msg": e["msg"], "type": e["type"]} for e in error.errors()] return JSONResponse(status_code=422, content={"detail": issues}) + @app.post("/diagnostics/render-policy", dependencies=[Depends(authenticate)]) + async def render_policy(request: AddRequest): + # Authenticated, read-only lookup for local benchmark attribution. + # Return no source content or identifiers, only the persisted policy. + with request_budget(settings.request_timeout_seconds): + async with admission.slot(): + policy = await anyio.to_thread.run_sync( + adapter.render_policy, request, limiter=business_threads + ) + return {"policy": policy} + @app.post("/add", response_model=AddResponse, dependencies=[Depends(authenticate)]) async def add(request: AddRequest): with request_budget(settings.request_timeout_seconds): diff --git a/agent_memory_leaderboard/src/benchmark.py b/agent_memory_leaderboard/src/benchmark.py index 9f7f14c..172864e 100644 --- a/agent_memory_leaderboard/src/benchmark.py +++ b/agent_memory_leaderboard/src/benchmark.py @@ -17,6 +17,7 @@ from .content_diagnostics import options as diagnostic_options from .datasets import ADAPTERS, DATASETS, normalize from .diagnostics import RetryAudit, adapter_reason_code +from .embedding_tokens import policy_budget from .evaluation import AnswerEvaluator, EvaluationConfig, ModelError from .evidence import SourceManifest, prefix_recalls, recall, source_messages from .models import AddResponse, SearchResponse @@ -129,6 +130,11 @@ def run(args): "mode": mode, "evaluation": evaluator.config.contract() if evaluator else None, "kind": args.kind, + "rendering_policy_source": ( + "explicit-legacy-v1" + if getattr(args, "legacy_byte_rendering", False) + else "adapter-per-request-v1" + ), } if diagnostic_policy is not None: contract["content_diagnostics"] = diagnostic_policy @@ -232,37 +238,63 @@ def call(http, path, body, identity): user = f"{args.kind}:{report['run_id']}:{sample}" if user not in report["users"]: report["users"].append(user) - manifest = SourceManifest(conversation["conversation"]) + policies = [] + for session, index, messages in batches(conversation["conversation"]): + rid = f"{user}:{session}:{index}" + body = dict(request_id=rid, user_id=user, session_id=session, messages=messages) + if rid not in completed: + report.update(stage="add", current=rid) + persist() + if content is not None: + content.add(sample, body) + response = call(http, "/add", body, rid) + if not isinstance(response, dict) or response.get("success") is not True: + raise ValueError("Add response requires explicit success=true") + result = AddResponse.model_validate(response) + if any( + getattr(result, k) != body[k] + for k in ("request_id", "user_id", "session_id") + ): + raise ValueError("Add ID mismatch") + if content is not None: + content.add(sample, body, result.model_dump()) + report["completed_adds"].append(rid) + completed.add(rid) + persist() + print(f"Added {sample}/{session}/{index}", flush=True) + report.update(stage="render-policy", current=rid) + response = http.post( + args.adapter_url.rstrip("/") + "/diagnostics/render-policy", + headers=headers, + json=body, + ) + if response.status_code == 404 and getattr( + args, "legacy_byte_rendering", False + ): + policy = {"version": 1} + else: + response.raise_for_status() + payload = response.json() + if not isinstance(payload, dict) or "policy" not in payload: + raise ValueError( + "Adapter rendering policy unavailable; cannot attribute evidence" + ) + policy = payload["policy"] + policy_budget(policy) + if getattr(args, "legacy_byte_rendering", False) and policy != { + "version": 1 + }: + raise ValueError( + "Adapter policy differs from explicit legacy rendering" + ) + policies.extend([policy] * len(messages)) + manifest = SourceManifest(conversation["conversation"], policy=policies) manifest_path = ( args.output.parent / (args.output.stem + ".sources") / (hashlib.sha256(sample.encode()).hexdigest() + ".json") ) save(manifest_path, manifest.dump()) - for session, index, messages in batches(conversation["conversation"]): - rid = f"{user}:{session}:{index}" - if rid in completed: - continue - report.update(stage="add", current=rid) - persist() - body = dict(request_id=rid, user_id=user, session_id=session, messages=messages) - if content is not None: - content.add(sample, body) - response = call(http, "/add", body, rid) - if not isinstance(response, dict) or response.get("success") is not True: - raise ValueError("Add response requires explicit success=true") - result = AddResponse.model_validate(response) - if any( - getattr(result, k) != body[k] - for k in ("request_id", "user_id", "session_id") - ): - raise ValueError("Add ID mismatch") - if content is not None: - content.add(sample, body, result.model_dump()) - report["completed_adds"].append(rid) - completed.add(rid) - persist() - print(f"Added {sample}/{session}/{index}", flush=True) source_ids = { turn["dia_id"] for session, turns in conversation["conversation"].items() @@ -461,6 +493,11 @@ def main(default_kind="locomo", default_output=None): parser.add_argument("--all", action="store_true", help="Run every history in this dataset") parser.add_argument("--top-k", type=int, default=100) parser.add_argument("--resume", action="store_true") + parser.add_argument( + "--legacy-byte-rendering", + action="store_true", + help="Explicit byte-only attribution for old adapters without the policy endpoint", + ) parser.add_argument("--mode", choices=["retrieval", "answer-eval"], default="retrieval") parser.add_argument( "--private-dir", type=Path, help="Private resumable answer cache; do not upload" diff --git a/agent_memory_leaderboard/src/config.py b/agent_memory_leaderboard/src/config.py index b1ccec9..f326606 100644 --- a/agent_memory_leaderboard/src/config.py +++ b/agent_memory_leaderboard/src/config.py @@ -4,6 +4,8 @@ from pathlib import Path from urllib.parse import urlsplit +from .embedding_tokens import DEFAULT_TOKEN_BUDGET, MAX_TOKEN_BUDGET, MIN_TOKEN_BUDGET + @dataclass(frozen=True) class Settings: @@ -25,14 +27,17 @@ class Settings: journal_slow_ms: float = 100 journal_queue_timeout_seconds: float = 30 time_normalization_enabled: bool = False - embedding_token_budget: int = 7680 + embedding_token_budget: int = DEFAULT_TOKEN_BUDGET def __post_init__(self) -> None: if ( type(self.embedding_token_budget) is not int - or not 128 <= self.embedding_token_budget <= 8192 + or not MIN_TOKEN_BUDGET <= self.embedding_token_budget <= MAX_TOKEN_BUDGET ): - raise ValueError("ADAPTER_EMBEDDING_TOKEN_BUDGET must be an integer in [128, 8192]") + raise ValueError( + f"ADAPTER_EMBEDDING_TOKEN_BUDGET must be an integer in " + f"[{MIN_TOKEN_BUDGET}, {MAX_TOKEN_BUDGET}]" + ) if type(self.time_normalization_enabled) is not bool: raise ValueError("MEMORY_TIME_NORMALIZATION_ENABLED must be a boolean") url = urlsplit(self.memoria_url) @@ -98,7 +103,9 @@ def from_env(cls) -> "Settings": if journal_logging not in {"true", "false", "1", "0"}: raise ValueError("ADAPTER_JOURNAL_LOG_ALL_OPERATIONS must be true/false or 1/0") return cls( - embedding_token_budget=int(os.getenv("ADAPTER_EMBEDDING_TOKEN_BUDGET", "7680")), + embedding_token_budget=int( + os.getenv("ADAPTER_EMBEDDING_TOKEN_BUDGET", str(DEFAULT_TOKEN_BUDGET)) + ), time_normalization_enabled=time_normalization in {"true", "1"}, memoria_url=os.environ["MEMORIA_URL"].rstrip("/"), memoria_master_key=os.environ["MEMORIA_MASTER_KEY"], diff --git a/agent_memory_leaderboard/src/embedding_tokens.py b/agent_memory_leaderboard/src/embedding_tokens.py index d14512e..214e70e 100644 --- a/agent_memory_leaderboard/src/embedding_tokens.py +++ b/agent_memory_leaderboard/src/embedding_tokens.py @@ -13,6 +13,30 @@ TOKENIZER_SHA256 = "21106b6d7dab2952c1d496fb21d5dc9db75c28ed361a05f5020bbba27810dd08" TOKENIZER_URL = f"https://huggingface.co/BAAI/bge-m3/resolve/{TOKENIZER_REVISION}/tokenizer.json" DEFAULT_TOKEN_BUDGET = 7680 +MIN_TOKEN_BUDGET = 128 +MAX_TOKEN_BUDGET = 8192 + + +def token_policy(budget: int) -> dict: + policy = {"version": 2, "tokenizer_sha256": TOKENIZER_SHA256, "max_tokens": budget} + policy_budget(policy) + return policy + + +def policy_budget(policy: dict) -> int | None: + if policy == {"version": 1} and type(policy["version"]) is int: + return None + if ( + isinstance(policy, dict) + and set(policy) == {"version", "tokenizer_sha256", "max_tokens"} + and type(policy["version"]) is int + and policy["version"] == 2 + and policy["tokenizer_sha256"] == TOKENIZER_SHA256 + and type(policy["max_tokens"]) is int + and MIN_TOKEN_BUDGET <= policy["max_tokens"] <= MAX_TOKEN_BUDGET + ): + return policy["max_tokens"] + raise ValueError("Unsupported rendering policy") def tokenizer_path() -> Path: diff --git a/agent_memory_leaderboard/src/evidence.py b/agent_memory_leaderboard/src/evidence.py index 9844353..416b0de 100644 --- a/agent_memory_leaderboard/src/evidence.py +++ b/agent_memory_leaderboard/src/evidence.py @@ -3,7 +3,8 @@ import hashlib import re -from .ingestion import RENDER_VERSION, message_parts +from .embedding_tokens import policy_budget +from .ingestion import message_parts from .models import Message @@ -33,12 +34,22 @@ def source_messages(conversation): class SourceManifest: - def __init__(self, conversation): + def __init__(self, conversation, *, policy: dict | list[dict]): self.hashes = {} self.entries = [] self.reader_rows = {} - for session, index, source, body in source_messages(conversation): - parts = [content for content, _ in message_parts(Message.model_validate(body))] + messages = list(source_messages(conversation)) + policies = [policy] * len(messages) if isinstance(policy, dict) else policy + if len(policies) != len(messages): + raise ValueError("Source manifest rendering policy count mismatch") + self.policies = policies + for (session, index, source, body), rendering in zip(messages, policies, strict=True): + parts = [ + content + for content, _ in message_parts( + Message.model_validate(body), token_budget=policy_budget(rendering) + ) + ] # Include exact wire content for adapters returning the original message. hashes = sorted({content_hash(body["content"]), *(content_hash(p) for p in parts)}) for value in hashes: @@ -72,8 +83,8 @@ def __init__(self, conversation): def dump(self): return { - "version": 1, - "render_version": RENDER_VERSION, + "version": 2, + "render_policies": self.policies, "method": "exact-content-sha256", "entries": self.entries, } diff --git a/agent_memory_leaderboard/src/ingestion.py b/agent_memory_leaderboard/src/ingestion.py index 8c84d56..4d2831a 100644 --- a/agent_memory_leaderboard/src/ingestion.py +++ b/agent_memory_leaderboard/src/ingestion.py @@ -2,10 +2,9 @@ from datetime import UTC, datetime, timedelta -from .embedding_tokens import DEFAULT_TOKEN_BUDGET, token_count +from .embedding_tokens import token_count from .models import text_content -RENDER_VERSION = 2 MAX_MEMORY_BYTES = 32768 @@ -41,7 +40,7 @@ def split_tokens(text: str, header: str, budget: int): hi = middle if lo == 0: raise ValueError("Token budget cannot fit the message prefix and a character") - # Prefer a nearby paragraph/word boundary, without dropping whitespace. + # Choose the latest nearby paragraph/word boundary, without dropping whitespace. floor = lo * 3 // 4 boundary = max(remaining.rfind(mark, floor, lo) + 1 for mark in ("\n", " ", "。")) if boundary > floor and token_count(header + remaining[:boundary]) <= budget: @@ -50,7 +49,7 @@ def split_tokens(text: str, header: str, budget: int): offset += lo -def message_parts(message, *, token_budget: int | None = DEFAULT_TOKEN_BUDGET): +def message_parts(message, *, token_budget: int | None): observed_at = None header = f"[{message.role}]\n" if message.timestamp is not None: diff --git a/agent_memory_leaderboard/src/journal.py b/agent_memory_leaderboard/src/journal.py index 87d378a..e27ce7b 100644 --- a/agent_memory_leaderboard/src/journal.py +++ b/agent_memory_leaderboard/src/journal.py @@ -16,6 +16,7 @@ LEGACY_PROTOCOL = "legacy" IDEMPOTENT_PROTOCOL = "idempotent_v1" +TOKEN_PROTOCOL = "idempotent_tokens_v2" class TimedConnection: @@ -77,6 +78,15 @@ def __init__( columns = {row[1] for row in db.execute("PRAGMA table_info(writes)")} if "protocol" not in columns: db.execute("ALTER TABLE writes ADD COLUMN protocol TEXT NOT NULL DEFAULT 'legacy'") + # Old binaries ignore render_policy. Guard absent positions as well as + # existing v2 receipts: their v1 INSERT/UPSERT must fail atomically. + for operation in ("INSERT", "UPDATE"): + db.execute(f"""CREATE TRIGGER IF NOT EXISTS protect_token_writes_{operation.lower()} + BEFORE {operation} ON writes + WHEN EXISTS (SELECT 1 FROM requests WHERE key = NEW.request_key + AND json_extract(render_policy, '$.version') = 2) + AND NEW.protocol != '{TOKEN_PROTOCOL}' + BEGIN SELECT RAISE(ABORT, 'Unsupported rendering protocol'); END""") def _acquire_connection_slot(self): queue_budget = budget_remaining(self.queue_timeout_seconds) @@ -235,8 +245,12 @@ def begin(self, key: str, fingerprint: str) -> bool: with self.connect("begin") as db: return self._begin(db, key, fingerprint) - def request_policy(self, key: str, fingerprint: str, default: dict) -> dict | None: - """Read under lock(key); the first prepare_write persists the chosen policy. + def request_policy( + self, key: str, fingerprint: str, default: dict | None, *, include_completed=False + ) -> dict | None: + """The Add caller holds lock(key); diagnostics only reads immutable policies. + + The first prepare_write persists the chosen policy atomically with intent. Existing rows migrate to byte-only v1. Never reinterpret old positions or rewrite old receipts when the default budget changes. None is a @@ -247,6 +261,8 @@ def request_policy(self, key: str, fingerprint: str, default: dict) -> dict | No "SELECT fingerprint, complete, render_policy FROM requests WHERE key = ?", (key,) ).fetchone() if row is None: + if default is None: + raise AdapterError(503, "Request rendering policy is not recorded") return default if row[0] != fingerprint: raise AdapterError( @@ -254,7 +270,7 @@ def request_policy(self, key: str, fingerprint: str, default: dict) -> dict | No "request_id was already used with a different payload; " "restore the original payload", ) - if row[1]: + if row[1] and not include_completed: return None try: policy = json.loads(row[2]) @@ -301,14 +317,20 @@ def prepare_write( if self._begin(db, key, fingerprint, render_policy): return None state, memory_id, protocol = self._state(db, key, position) - if protocol not in {LEGACY_PROTOCOL, IDEMPOTENT_PROTOCOL}: + if protocol not in {LEGACY_PROTOCOL, IDEMPOTENT_PROTOCOL, TOKEN_PROTOCOL}: raise AdapterError(503, "Unsupported journal write protocol; check adapter version") if state not in {"pending", "inflight", "stored"}: raise AdapterError(503, "Unsupported journal write state; check adapter version") if state == "pending": # Pending legacy writes are known not to have been sent. Persist # the atomic protocol before HTTP, including on the first chunk. - state, memory_id, protocol = "inflight", None, IDEMPOTENT_PROTOCOL + policy = json.loads( + db.execute( + "SELECT render_policy FROM requests WHERE key = ?", (key,) + ).fetchone()[0] + ) + protocol = TOKEN_PROTOCOL if policy.get("version") == 2 else IDEMPOTENT_PROTOCOL + state, memory_id = "inflight", None self._record(db, key, position, state, memory_id, protocol) return state, memory_id, protocol diff --git a/agent_memory_leaderboard/src/service.py b/agent_memory_leaderboard/src/service.py index d2886a5..59c5f9f 100644 --- a/agent_memory_leaderboard/src/service.py +++ b/agent_memory_leaderboard/src/service.py @@ -5,7 +5,7 @@ from .budget import catalog_retry_budget, request_budget from .budget import remaining as budget_remaining from .config import Settings -from .embedding_tokens import TOKENIZER_SHA256 +from .embedding_tokens import policy_budget, token_policy from .errors import AdapterError from .ingestion import message_parts from .journal import LEGACY_PROTOCOL, Journal @@ -44,11 +44,7 @@ def add(self, request: AddRequest) -> AddResponse: policy = self.journal.request_policy( key, fingerprint, - { - "version": 2, - "tokenizer_sha256": TOKENIZER_SHA256, - "max_tokens": self.settings.embedding_token_budget, - }, + token_policy(self.settings.embedding_token_budget), ) # One-item lookahead identifies the last chunk without buffering # the entire request, so its stored marker and completion commit together. @@ -69,22 +65,27 @@ def add(self, request: AddRequest) -> AddResponse: session_id=request.session_id, ) - def write_bodies(self, request: AddRequest, key: str, policy: dict | None = None): - # Explicit default keeps historical audit helpers and recovery fixtures v1. - policy = policy if policy is not None else {"version": 1} - if policy == {"version": 1} and type(policy["version"]) is int: - token_budget = None - elif ( - set(policy) == {"version", "tokenizer_sha256", "max_tokens"} - and type(policy["version"]) is int - and policy["version"] == 2 - and policy["tokenizer_sha256"] == TOKENIZER_SHA256 - and type(policy["max_tokens"]) is int - and 128 <= policy["max_tokens"] <= 8192 - ): - token_budget = policy["max_tokens"] - else: - raise AdapterError(503, "Unsupported journal rendering policy; check adapter version") + def render_policy(self, request: AddRequest) -> dict: + owner = self.owner(request.user_id) + key = digest([self.settings.memoria_url.rstrip("/"), owner, request.request_id]) + policy = self.journal.request_policy( + key, digest(request.model_dump()), None, include_completed=True + ) + try: + policy_budget(policy) + except ValueError: + raise AdapterError( + 503, "Unsupported journal rendering policy; check adapter version" + ) from None + return policy + + def write_bodies(self, request: AddRequest, key: str, policy: dict): + try: + token_budget = policy_budget(policy) + except ValueError: + raise AdapterError( + 503, "Unsupported journal rendering policy; check adapter version" + ) from None session = digest([self.settings.namespace, request.user_id, request.session_id]) for message_index, message in enumerate(request.messages): for part_index, (content, observed_at) in enumerate( diff --git a/agent_memory_leaderboard/tests/test_benchmark.py b/agent_memory_leaderboard/tests/test_benchmark.py index 9d91060..9598c54 100644 --- a/agent_memory_leaderboard/tests/test_benchmark.py +++ b/agent_memory_leaderboard/tests/test_benchmark.py @@ -13,6 +13,7 @@ summarize, ) from src.datasets import normalize +from src.embedding_tokens import DEFAULT_TOKEN_BUDGET, token_policy def test_ingestion_keeps_source_order_and_does_not_include_annotations(): @@ -144,6 +145,8 @@ def transport(request): body = json.loads(request.content) if request.content else {} calls.append((request.url.path, body)) assert "GOLD MUST STAY LOCAL" not in str(body) + if request.url.path == "/diagnostics/render-policy": + return httpx.Response(200, json={"policy": token_policy(DEFAULT_TOKEN_BUDGET)}) if request.url.path == "/add": if rejected: return httpx.Response(403, json={"detail": { diff --git a/agent_memory_leaderboard/tests/test_benchmark_evaluation.py b/agent_memory_leaderboard/tests/test_benchmark_evaluation.py index 36c9513..77809c3 100644 --- a/agent_memory_leaderboard/tests/test_benchmark_evaluation.py +++ b/agent_memory_leaderboard/tests/test_benchmark_evaluation.py @@ -7,6 +7,7 @@ from scripts import benchmark_ci as ci from src import benchmark, evaluation +from src.embedding_tokens import DEFAULT_TOKEN_BUDGET, token_policy @pytest.mark.parametrize("failure", ["answer", "judge", "length", "429", "503", "transport"]) @@ -51,6 +52,8 @@ def handler(request): if host == "adapter.test": calls[path] += 1 assert "PRIVATE_GOLD" not in str(body) + if request.url.path == "/diagnostics/render-policy": + return httpx.Response(200, json={"policy": token_policy(DEFAULT_TOKEN_BUDGET)}) if path == "/add": return httpx.Response( 200, @@ -171,6 +174,8 @@ def handler(request): body = json.loads(request.content) if request.content else {} if request.url.host == "answer.test": return httpx.Response(401, text="PRIVATE credential detail") + if request.url.path == "/diagnostics/render-policy": + return httpx.Response(200, json={"policy": token_policy(DEFAULT_TOKEN_BUDGET)}) if request.url.path == "/add": return httpx.Response( 200, diff --git a/agent_memory_leaderboard/tests/test_catalog_retry.py b/agent_memory_leaderboard/tests/test_catalog_retry.py index ef5216b..21712f3 100644 --- a/agent_memory_leaderboard/tests/test_catalog_retry.py +++ b/agent_memory_leaderboard/tests/test_catalog_retry.py @@ -10,7 +10,7 @@ from src import scheduling as scheduling_module from src.budget import request_budget from src.errors import AdapterError -from src.journal import IDEMPOTENT_PROTOCOL, Journal +from src.journal import TOKEN_PROTOCOL, Journal from src.memoria import IDEMPOTENT_PATH, Memoria from src.models import AddRequest from src.service import digest @@ -125,7 +125,7 @@ def unavailable(request, body, owner): owner = digest([settings.namespace, model.user_id]) key = digest([settings.memoria_url, owner, model.request_id]) assert Journal(settings.state_dir).state(key, 0) == ( - "inflight", None, IDEMPOTENT_PROTOCOL + "inflight", None, TOKEN_PROTOCOL ) assert len(upstream.writes) == 6 assert sum(clock.sleeps) == 5 diff --git a/agent_memory_leaderboard/tests/test_content_diagnostics.py b/agent_memory_leaderboard/tests/test_content_diagnostics.py index c30bdb2..b731643 100644 --- a/agent_memory_leaderboard/tests/test_content_diagnostics.py +++ b/agent_memory_leaderboard/tests/test_content_diagnostics.py @@ -17,6 +17,7 @@ read_record, signals, ) +from src.embedding_tokens import DEFAULT_TOKEN_BUDGET, token_policy def setup_args(tmp_path, *, enabled=True, mode="retrieval"): @@ -88,6 +89,8 @@ def respond(request): calls[(request.url.host, request.url.path)] += 1 if request.url.host == "adapter.test": assert "reference_answers" not in str(body) + if request.url.path == "/diagnostics/render-policy": + return httpx.Response(200, json={"policy": token_policy(DEFAULT_TOKEN_BUDGET)}) if request.url.path == "/add": return httpx.Response( 200, @@ -471,4 +474,11 @@ def test_pending_resume_discloses_missing_search_and_rejects_cache_mismatch( assert q["model"]["answer_output"]["from_signature_verified_cache"] else: assert final["error"]["type"] == "ValueError" - assert sum(n for (_, path), n in calls.items() if path != "/health") == 0 + assert ( + sum( + n + for (_, path), n in calls.items() + if path not in {"/health", "/diagnostics/render-policy"} + ) + == 0 + ) diff --git a/agent_memory_leaderboard/tests/test_evidence.py b/agent_memory_leaderboard/tests/test_evidence.py index 7e35c45..7a07b37 100644 --- a/agent_memory_leaderboard/tests/test_evidence.py +++ b/agent_memory_leaderboard/tests/test_evidence.py @@ -1,5 +1,6 @@ import json +from src.embedding_tokens import DEFAULT_TOKEN_BUDGET, token_policy from src.evidence import SourceManifest, prefix_recalls, recall, source_messages from src.ingestion import message_parts from src.models import Message @@ -17,9 +18,9 @@ def conversation(texts): def test_manifest_matches_original_and_adapter_rendered_content_without_labels(): data = conversation(["private fact"]) - manifest = SourceManifest(data) + manifest = SourceManifest(data, policy=token_policy(DEFAULT_TOKEN_BUDGET)) body = next(source_messages(data))[-1] - rendered = list(message_parts(Message.model_validate(body)))[0][0] + rendered = list(message_parts(Message.model_validate(body), token_budget=7680))[0][0] assert manifest.match(body["content"]) == ["D1:1"] assert manifest.match(rendered) == ["D1:1"] assert "private fact" not in json.dumps(manifest.dump()) @@ -28,7 +29,7 @@ def test_manifest_matches_original_and_adapter_rendered_content_without_labels() def test_duplicate_content_is_ambiguous_and_never_credited(): data = conversation(["same fact", "same fact"]) - manifest = SourceManifest(data) + manifest = SourceManifest(data, policy=token_policy(DEFAULT_TOKEN_BUDGET)) content = next(source_messages(data))[-1]["content"] ranked, audit = manifest.audit([content, "unknown"]) assert ranked == [[], []] @@ -38,15 +39,17 @@ def test_duplicate_content_is_ambiguous_and_never_credited(): def test_long_utf8_message_matches_each_adapter_chunk(): data = conversation(["记忆" * 16000 + "ending"]) - manifest = SourceManifest(data) + manifest = SourceManifest(data, policy=token_policy(DEFAULT_TOKEN_BUDGET)) body = next(source_messages(data))[-1] - parts = list(message_parts(Message.model_validate(body))) + parts = list(message_parts(Message.model_validate(body), token_budget=7680)) assert len(parts) >= 3 assert all(manifest.match(text) == ["D1:1"] for text, _ in parts) def test_redacted_content_stays_unmapped_instead_of_inventing_provenance(): - manifest = SourceManifest(conversation(["phone 12345"])) + manifest = SourceManifest( + conversation(["phone 12345"]), policy=token_policy(DEFAULT_TOKEN_BUDGET) + ) assert manifest.audit(["[user]\n[date] A: phone [phone]"])[1]["unmapped_records"] == 1 diff --git a/agent_memory_leaderboard/tests/test_idempotent_writes.py b/agent_memory_leaderboard/tests/test_idempotent_writes.py index ce62721..0ebf2e3 100644 --- a/agent_memory_leaderboard/tests/test_idempotent_writes.py +++ b/agent_memory_leaderboard/tests/test_idempotent_writes.py @@ -13,7 +13,7 @@ from src.config import Settings from src.errors import AdapterError -from src.journal import IDEMPOTENT_PROTOCOL, LEGACY_PROTOCOL, Journal +from src.journal import IDEMPOTENT_PROTOCOL, LEGACY_PROTOCOL, TOKEN_PROTOCOL, Journal from src.memoria import IDEMPOTENT_PATH, Memoria from src.models import AddRequest from src.service import MemoryAdapter, digest @@ -38,7 +38,9 @@ def legacy_journal(settings, upstream, payload, entries, *, complete=False): request = AddRequest.model_validate(payload) owner = digest([settings.namespace, request.user_id]) key = digest([settings.memoria_url, owner, request.request_id]) - bodies = list(MemoryAdapter(settings, None, None).write_bodies(request, key)) + bodies = list( + MemoryAdapter(settings, None, None).write_bodies(request, key, policy={"version": 1}) + ) with sqlite3.connect(settings.state_dir / "requests.sqlite3") as db: db.executescript(""" CREATE TABLE requests(key TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, @@ -119,7 +121,7 @@ def test_process_exit_after_inflight_before_http_recovers(settings, upstream): assert not (settings.state_dir / "unexpected-post").exists() with sqlite3.connect(settings.state_dir / "requests.sqlite3") as db: assert db.execute("SELECT state, protocol FROM writes").fetchall() == [ - ("inflight", IDEMPOTENT_PROTOCOL) + ("inflight", TOKEN_PROTOCOL) ] with running(settings, upstream) as client: assert client.post("/add", json=addition()).status_code == 200 @@ -309,7 +311,7 @@ def test_backend_rejects_changed_body_for_same_atomic_key(settings, upstream): request = AddRequest.model_validate(addition()) adapter = MemoryAdapter(settings, memoria, None) owner = adapter.owner(request.user_id) - body = next(adapter.write_bodies(request, "synthetic-key")) + body = next(adapter.write_bodies(request, "synthetic-key", policy={"version": 1})) marker = body["extra_metadata"]["aml_write"] memoria.store_idempotent(owner, body, marker) with pytest.raises(AdapterError) as raised: diff --git a/agent_memory_leaderboard/tests/test_ingestion.py b/agent_memory_leaderboard/tests/test_ingestion.py index 01bd2de..c1997ce 100644 --- a/agent_memory_leaderboard/tests/test_ingestion.py +++ b/agent_memory_leaderboard/tests/test_ingestion.py @@ -30,7 +30,7 @@ def test_split_empty_and_invalid_limit(): def test_message_timestamp_uses_source_milliseconds(timestamp, expected): message = Message(role="assistant", content="fact", timestamp=timestamp) header = f"[assistant at {expected}]\n" if expected else "[assistant]\n" - assert list(message_parts(message)) == [(header + "fact", expected)] + assert list(message_parts(message, token_budget=7680)) == [(header + "fact", expected)] def test_message_chunks_include_header_in_byte_limit_and_join_text_parts(): diff --git a/agent_memory_leaderboard/tests/test_journal_transactions.py b/agent_memory_leaderboard/tests/test_journal_transactions.py index c3a6d6a..09ede7e 100644 --- a/agent_memory_leaderboard/tests/test_journal_transactions.py +++ b/agent_memory_leaderboard/tests/test_journal_transactions.py @@ -8,7 +8,7 @@ from src.config import Settings from src.errors import AdapterError -from src.journal import IDEMPOTENT_PROTOCOL, Journal +from src.journal import IDEMPOTENT_PROTOCOL, TOKEN_PROTOCOL, Journal from src.memoria import Memoria from src.models import AddRequest from src.service import MemoryAdapter, digest @@ -85,7 +85,7 @@ def commit(self): assert not upstream.writes else: assert requests == [(0,)] - assert writes[0][0] == "inflight" and writes[0][2] == IDEMPOTENT_PROTOCOL + assert writes[0][0] == "inflight" and writes[0][2] == TOKEN_PROTOCOL assert (writes[0][1] is not None) == (failed_commit == 3) # Reopen the journal as on restart. A lost ACK can replay the same receipt; # a durable known ID must recover without issuing another write. diff --git a/agent_memory_leaderboard/tests/test_longmemeval_pipeline.py b/agent_memory_leaderboard/tests/test_longmemeval_pipeline.py index 8565942..6093209 100644 --- a/agent_memory_leaderboard/tests/test_longmemeval_pipeline.py +++ b/agent_memory_leaderboard/tests/test_longmemeval_pipeline.py @@ -5,6 +5,7 @@ import httpx from src import benchmark, evaluation +from src.embedding_tokens import DEFAULT_TOKEN_BUDGET, token_policy def test_longmemeval_resume_reuses_add_search_and_answer( @@ -48,6 +49,8 @@ def handler(request): calls[(host, route)] += 1 if host == "adapter.test": assert "PRIVATE_GOLD" not in str(body) and "has_answer" not in str(body) + if route == "/diagnostics/render-policy": + return httpx.Response(200, json={"policy": token_policy(DEFAULT_TOKEN_BUDGET)}) if route == "/add": stored.extend(body["messages"]) return httpx.Response( diff --git a/agent_memory_leaderboard/tests/test_longmemeval_reader.py b/agent_memory_leaderboard/tests/test_longmemeval_reader.py index 64893a7..4a79d2c 100644 --- a/agent_memory_leaderboard/tests/test_longmemeval_reader.py +++ b/agent_memory_leaderboard/tests/test_longmemeval_reader.py @@ -4,6 +4,7 @@ import httpx import pytest +from src.embedding_tokens import DEFAULT_TOKEN_BUDGET, token_policy from src.evaluation import AnswerEvaluator, EvaluationConfig, ModelError from src.evidence import SourceManifest, source_messages from src.ingestion import message_parts @@ -30,7 +31,7 @@ def returned(conversation): return [ part for _, _, _, body in source_messages(conversation) - for part, _ in message_parts(Message.model_validate(body)) + for part, _ in message_parts(Message.model_validate(body), token_budget=7680) ] @@ -54,7 +55,7 @@ def config(mode="con", **options): ) def test_reader_matches_pinned_native_prompt_bytes(conversation, fake_encoding, mode, expected): # Goldens are generated from upstream prepare_prompt on this synthetic fixture. - manifest = SourceManifest(conversation) + manifest = SourceManifest(conversation, policy=token_policy(DEFAULT_TOKEN_BUDGET)) contents = list(reversed(returned(conversation))) evaluator = AnswerEvaluator(config(mode), "longmemeval") messages, info = context( @@ -75,7 +76,7 @@ def test_reader_matches_pinned_native_prompt_bytes(conversation, fake_encoding, def test_reader_never_adds_unretrieved_turns_or_answer_labels(conversation, fake_encoding): contents = returned(conversation)[:1] - manifest = SourceManifest(conversation) + manifest = SourceManifest(conversation, policy=token_policy(DEFAULT_TOKEN_BUDGET)) evaluator = AnswerEvaluator(config(), "longmemeval") messages, _ = context( {"question": "q", "question_date": "date", "answer": "PRIVATE_GOLD"}, @@ -96,7 +97,9 @@ def test_reader_never_adds_unretrieved_turns_or_answer_labels(conversation, fake def test_fragment_metadata_does_not_expand_to_whole_source(conversation): conversation["session_1"][0]["text"] = "x" * 40000 + "UNRETRIEVED_SUFFIX" contents = returned(conversation) - row = SourceManifest(conversation).reader_records(contents[:1])[0] + row = SourceManifest(conversation, policy=token_policy(DEFAULT_TOKEN_BUDGET)).reader_records( + contents[:1] + )[0] assert row["role"] == "user" assert "UNRETRIEVED_SUFFIX" not in row["content"] assert len(row["content"]) < 40000 @@ -105,7 +108,7 @@ def test_fragment_metadata_does_not_expand_to_whole_source(conversation): def test_ambiguous_and_unknown_records_are_not_assigned_source_metadata(conversation): conversation["session_2_date_time"] = conversation["session_1_date_time"] conversation["session_2"][0]["text"] = "Box is blue." - manifest = SourceManifest(conversation) + manifest = SourceManifest(conversation, policy=token_policy(DEFAULT_TOKEN_BUDGET)) assert manifest.reader_records([returned(conversation)[0], "unknown actual memory"]) == [ None, None, diff --git a/agent_memory_leaderboard/tests/test_render_policy.py b/agent_memory_leaderboard/tests/test_render_policy.py new file mode 100644 index 0000000..f2a006b --- /dev/null +++ b/agent_memory_leaderboard/tests/test_render_policy.py @@ -0,0 +1,256 @@ +"""Synthetic regressions for attribution and downgrade safety.""" + +import hashlib +import json +from dataclasses import replace +from types import SimpleNamespace + +import httpx +import pytest +from test_endpoints import FakeMemoria, addition, running + +from src import benchmark +from src.config import Settings +from src.embedding_tokens import token_policy +from src.errors import AdapterError +from src.evidence import SourceManifest, source_messages +from src.ingestion import message_parts +from src.journal import IDEMPOTENT_PROTOCOL, TOKEN_PROTOCOL, Journal +from src.models import AddRequest, Message +from src.service import MemoryAdapter, digest + + +def dense(): + return " ".join(hashlib.sha256(str(i).encode()).hexdigest() for i in range(500)) + + +def conversation(): + return { + "session_1_date_time": "2025-01-01", + "session_1": [{"dia_id": "D1:1", "speaker": "A", "text": dense()}], + "session_2_date_time": "2025-01-02", + "session_2": [{"dia_id": "D2:1", "speaker": "A", "text": dense()}], + } + + +def test_manifest_uses_explicit_mixed_policies_and_reader_metadata(): + data = conversation() + policies = [{"version": 1}, token_policy(4096)] + manifest = SourceManifest(data, policy=policies) + for (_, _, source, body), policy in zip(source_messages(data), policies, strict=True): + parts = [ + p + for p, _ in message_parts( + Message.model_validate(body), token_budget=policy.get("max_tokens") + ) + ] + mapped, audit = manifest.audit(parts) + assert all(ids == [source] for ids in mapped) + assert audit["unmapped_records"] == 0 + assert all( + row is not None and row["date"] != "unknown" for row in manifest.reader_records(parts) + ) + assert manifest.dump()["render_policies"] == policies + assert manifest.dump()["version"] == 2 + with pytest.raises(ValueError, match="count mismatch"): + SourceManifest(data, policy=[token_policy(4096)]) + with pytest.raises(TypeError): + SourceManifest(data) + with pytest.raises(TypeError): + list(message_parts(Message(role="user", content="fact"))) + + +def test_policy_endpoint_auth_fingerprint_persistence_and_no_backend_calls(tmp_path): + settings = Settings( + memoria_url="http://memoria.test", + memoria_master_key="upstream-secret", + state_dir=tmp_path, + api_key="synthetic-key", + embedding_token_budget=4096, + ) + upstream = FakeMemoria() + payload = addition() + auth = {"Authorization": "Bearer synthetic-key"} + with running(settings, upstream) as client: + assert client.post("/diagnostics/render-policy", json=payload).status_code == 401 + assert ( + client.post("/diagnostics/render-policy", json=payload, headers=auth).status_code == 503 + ) + assert client.post("/add", json=payload, headers=auth).status_code == 200 + before = len(upstream.calls) + with running(replace(settings, embedding_token_budget=256), upstream) as client: + result = client.post("/diagnostics/render-policy", json=payload, headers=auth) + assert result.status_code == 200 and result.json() == {"policy": token_policy(4096)} + assert ( + client.post( + "/diagnostics/render-policy", + json={**payload, "session_id": "changed"}, + headers=auth, + ).status_code + == 400 + ) + assert len(upstream.calls) == before + + +@pytest.mark.parametrize("state", ["inflight", "stored"]) +def test_old_protocol_reader_rejects_existing_token_positions(tmp_path, state): + journal = Journal(tmp_path) + journal.prepare_write("key", "fingerprint", 0, render_policy=token_policy(7680)) + journal.record("key", 0, state, "synthetic-memory", protocol=TOKEN_PROTOCOL) + # The old binary checks this whitelist before its stored/complete fast path. + with journal.connect("prepare_write") as db: + _, _, protocol = db.execute( + "SELECT state, memory_id, protocol FROM writes WHERE request_key=? AND position=?", + ("key", 0), + ).fetchone() + assert protocol not in {"legacy", IDEMPOTENT_PROTOCOL} + assert db.execute("SELECT complete FROM requests").fetchone()[0] == 0 + + +@pytest.mark.parametrize("position", [0, 1, 7]) +def test_sql_guard_blocks_old_protocol_including_unwritten_positions(tmp_path, position): + journal = Journal(tmp_path) + journal.prepare_write("key", "fingerprint", 0, render_policy=token_policy(7680)) + # This is the old binary's actual UPSERT helper; SQLite rejects both insertion + # of missing positions and replacement of already-recorded token receipts. + with pytest.raises(AdapterError) as error: + journal.record("key", position, "inflight", protocol=IDEMPOTENT_PROTOCOL) + assert error.value.status == 503 + assert journal.state("key", 0)[2] == TOKEN_PROTOCOL + if position: + assert journal.state("key", position)[0] == "pending" + # Current code can continue, and old v1 requests remain writable. + journal.prepare_write("key", "fingerprint", 1, render_policy=token_policy(7680)) + assert journal.state("key", 1)[2] == TOKEN_PROTOCOL + journal.prepare_write("old-key", "old-fingerprint", 0, render_policy={"version": 1}) + assert journal.state("old-key", 0)[2] == IDEMPOTENT_PROTOCOL + + +def test_runner_mixed_old_and_custom_budget_attribution_and_resume(tmp_path, monkeypatch): + source = tmp_path / "input.json" + source.write_text( + json.dumps( + [ + { + "sample_id": "sample", + "conversation": conversation(), + "qa": [ + { + "question": "identifiers", + "category": 1, + "answer": "synthetic", + "evidence": ["D1:1", "D2:1"], + } + ], + } + ] + ) + ) + settings = Settings( + memoria_url="http://memoria.test", + memoria_master_key="upstream-secret", + state_dir=tmp_path / "state", + embedding_token_budget=4096, + ) + upstream = FakeMemoria() + args = SimpleNamespace( + dataset=source, + kind="locomo", + conversations=1, + top_k=100, + adapter_url="http://adapter.test", + output=tmp_path / "run.json", + resume=False, + ) + original_client = httpx.Client + policy_calls = [] + with running(settings, upstream) as client: + + def route(request): + body = json.loads(request.content) if request.content else {} + if request.url.path == "/add" and body["session_id"] == "session_1": + adapter = MemoryAdapter(settings, None, Journal(settings.state_dir)) + key = digest( + [settings.memoria_url, adapter.owner(body["user_id"]), body["request_id"]] + ) + adapter.journal.begin(key, digest(AddRequest.model_validate(body).model_dump())) + if request.url.path == "/diagnostics/render-policy": + policy_calls.append(body["request_id"]) + response = client.request( + request.method, + request.url.path, + content=request.content, + headers={"Content-Type": "application/json"}, + ) + return httpx.Response(response.status_code, content=response.content) + + monkeypatch.setattr( + benchmark.httpx, + "Client", + lambda **kw: original_client(transport=httpx.MockTransport(route), **kw), + ) + assert benchmark.run(args) == 0 + report = json.loads(args.output.read_text()) + question = next(iter(report["questions"].values())) + assert question["evidence_recall"] == 1 + assert question["source_attribution"]["unmapped_records"] == 0 + manifest = json.loads(next((tmp_path / "run.sources").glob("*.json")).read_text()) + assert manifest["render_policies"] == [{"version": 1}, token_policy(4096)] + before = len(upstream.writes) + args.resume = True + assert benchmark.run(args) == 0 + assert len(upstream.writes) == before and len(policy_calls) == 4 + + +@pytest.mark.parametrize( + "legacy,status,policy,success", + [ + (False, 404, None, False), + (True, 404, None, True), + (False, 200, {"version": 99}, False), + (True, 200, token_policy(4096), False), + (True, 403, None, False), + ], +) +def test_runner_missing_policy_fails_closed_unless_explicit_old_adapter( + tmp_path, + monkeypatch, + legacy, + status, + policy, + success, +): + source = tmp_path / "input.json" + source.write_text( + json.dumps([{"sample_id": "sample", "conversation": conversation(), "qa": []}]) + ) + + def route(request): + if request.url.path == "/diagnostics/render-policy": + return httpx.Response(status, json={"policy": policy}) + if request.url.path == "/add": + body = json.loads(request.content) + return httpx.Response( + 200, + json={ + "success": True, + **{k: body[k] for k in ("request_id", "user_id", "session_id")}, + }, + ) + return httpx.Response(200, json={"status": "ok"}) + + original = httpx.Client + monkeypatch.setattr( + benchmark.httpx, "Client", lambda **kw: original(transport=httpx.MockTransport(route), **kw) + ) + args = SimpleNamespace( + dataset=source, + kind="locomo", + conversations=1, + top_k=100, + adapter_url="http://adapter.test", + output=tmp_path / "run.json", + resume=False, + legacy_byte_rendering=legacy, + ) + assert benchmark.run(args) == (0 if success else 1) diff --git a/agent_memory_leaderboard/tests/test_token_chunks.py b/agent_memory_leaderboard/tests/test_token_chunks.py index 079fa11..2dde652 100644 --- a/agent_memory_leaderboard/tests/test_token_chunks.py +++ b/agent_memory_leaderboard/tests/test_token_chunks.py @@ -32,7 +32,7 @@ def test_reported_byte_valid_token_overflow_is_split_without_loss(): assert len(legacy) == 1 assert len(legacy[0][0].encode()) < MAX_MEMORY_BYTES assert token_count(legacy[0][0]) > 8192 - parts = list(message_parts(message)) + parts = list(message_parts(message, token_budget=7680)) assert len(parts) > 1 header = "[user at 2024-01-01T00:00:00.000Z]\n" assert "".join(p.removeprefix(header) for p, _ in parts) == message.content @@ -60,10 +60,14 @@ def test_original_text_preserved_even_when_tokenizer_normalizes(text): def test_short_message_unchanged_and_concurrent_tokenization_deterministic(): message = Message(role="user", content="A short fact.", timestamp=-1) - assert list(message_parts(message)) == list(message_parts(message, token_budget=None)) + assert list(message_parts(message, token_budget=7680)) == list( + message_parts(message, token_budget=None) + ) large = Message(role="assistant", content=dense_text()) with ThreadPoolExecutor(max_workers=16) as executor: - outputs = list(executor.map(lambda _: list(message_parts(large)), range(16))) + outputs = list( + executor.map(lambda _: list(message_parts(large, token_budget=7680)), range(16)) + ) assert all(result == outputs[0] for result in outputs) diff --git a/agent_memory_leaderboard/tests/test_write_visibility.py b/agent_memory_leaderboard/tests/test_write_visibility.py index 8a5bae9..c200a5c 100644 --- a/agent_memory_leaderboard/tests/test_write_visibility.py +++ b/agent_memory_leaderboard/tests/test_write_visibility.py @@ -9,7 +9,7 @@ from src import memoria as memoria_module from src.config import Settings from src.errors import AdapterError -from src.journal import IDEMPOTENT_PROTOCOL, LEGACY_PROTOCOL, Journal +from src.journal import IDEMPOTENT_PROTOCOL, LEGACY_PROTOCOL, TOKEN_PROTOCOL, Journal from src.memoria import Memoria from src.models import AddRequest from src.scheduling import BackendScheduler @@ -135,7 +135,7 @@ def hidden(request, body, owner): [boundary.settings.memoria_url.rstrip("/"), adapter.owner(model.user_id), model.request_id] ) state, memory_id, protocol = journal.state(key, 0) - assert state == "inflight" and memory_id and protocol == IDEMPOTENT_PROTOCOL + assert state == "inflight" and memory_id and protocol == TOKEN_PROTOCOL assert journal.begin(key, digest(model.model_dump())) is False assert len(upstream.writes) == 1 upstream.hook = None From 4a6b85c946f8358827a586237f7ccc74320c71e2 Mon Sep 17 00:00:00 2001 From: lr90 Date: Fri, 30 Oct 2026 23:48:11 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix(benchmark):=20=E4=B8=BA=E5=88=86?= =?UTF-8?q?=E7=89=87=E7=AD=96=E7=95=A5=E6=9F=A5=E8=AF=A2=E8=A1=A5=E9=BD=90?= =?UTF-8?q?=E9=87=8D=E8=AF=95=E4=B8=8E=E5=AE=A1=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent_memory_leaderboard/README.md | 12 +- agent_memory_leaderboard/src/benchmark.py | 17 +- .../tests/test_render_policy.py | 160 ++++++++++++++++++ 3 files changed, 177 insertions(+), 12 deletions(-) diff --git a/agent_memory_leaderboard/README.md b/agent_memory_leaderboard/README.md index 60c2206..0167278 100644 --- a/agent_memory_leaderboard/README.md +++ b/agent_memory_leaderboard/README.md @@ -670,7 +670,12 @@ adapter worker/replica and stop the old version before upgrading. Do not roll back to a byte-only binary with unfinished v2 requests. Token-policy receipts use an unknown-to-old-code protocol, and SQLite triggers reject v1 writes to any new position of a v2 request. Old versions fail with 503 instead of silently skipping -content. These guards require retaining the journal schema and triggers. Preserve +content. If an old binary reaches the SQLite trigger, it raises an IntegrityError +with `Unsupported rendering protocol`. The pre-upgrade journal catches SQLite +errors and returns generic HTTP 503 `Idempotency journal unavailable`, rather than +the explicit version-check message from its protocol whitelist. Check the deployed +binary/journal versions first; restore the compatible binary, not delete the journal +or trigger. These guards require retaining the journal schema and triggers. Preserve the journal and fixed model/tokenizer for the whole run. Ordinary Search queries are unchanged; the write-visibility search uses each newly bounded chunk. @@ -678,7 +683,10 @@ The local benchmark reads each Add request's persisted policy from the authentic read-only `POST /diagnostics/render-policy` endpoint (same Add body and credentials). It records per-message policies in source manifests, including mixed v1/v2 requests and resumed runs. Missing/invalid policies stop attribution explicitly instead of -silently using a default. To evaluate a known old byte-only adapter lacking this +silently using a default. Policy lookups use the same six-attempt bounded retry, +Retry-After handling and body-free request audit as Add/Search. A failed lookup +keeps completed Add checkpoints, so resume does not repeat successful writes. +To evaluate a known old byte-only adapter lacking this endpoint, explicitly pass `--legacy-byte-rendering`; only a 404 permits that fallback. The flag is part of the resume contract and rejects a detected v2 policy. Pre-change benchmark checkpoints lack this policy contract and cannot be resumed by diff --git a/agent_memory_leaderboard/src/benchmark.py b/agent_memory_leaderboard/src/benchmark.py index 172864e..47d7821 100644 --- a/agent_memory_leaderboard/src/benchmark.py +++ b/agent_memory_leaderboard/src/benchmark.py @@ -263,18 +263,15 @@ def call(http, path, body, identity): persist() print(f"Added {sample}/{session}/{index}", flush=True) report.update(stage="render-policy", current=rid) - response = http.post( - args.adapter_url.rstrip("/") + "/diagnostics/render-policy", - headers=headers, - json=body, - ) - if response.status_code == 404 and getattr( - args, "legacy_byte_rendering", False - ): + try: + payload = call(http, "/diagnostics/render-policy", body, rid) + except httpx.HTTPStatusError as exc: + if exc.response.status_code != 404 or not getattr( + args, "legacy_byte_rendering", False + ): + raise policy = {"version": 1} else: - response.raise_for_status() - payload = response.json() if not isinstance(payload, dict) or "policy" not in payload: raise ValueError( "Adapter rendering policy unavailable; cannot attribute evidence" diff --git a/agent_memory_leaderboard/tests/test_render_policy.py b/agent_memory_leaderboard/tests/test_render_policy.py index f2a006b..78288be 100644 --- a/agent_memory_leaderboard/tests/test_render_policy.py +++ b/agent_memory_leaderboard/tests/test_render_policy.py @@ -253,4 +253,164 @@ def route(request): resume=False, legacy_byte_rendering=legacy, ) + monkeypatch.setattr("src.smoke.time.sleep", lambda _: pytest.fail("unexpected retry")) assert benchmark.run(args) == (0 if success else 1) + events = [ + json.loads(line) + for line in args.output.with_suffix(".requests.jsonl").read_text().splitlines() + if json.loads(line)["route"] == "/diagnostics/render-policy" + ] + assert len(events) == (2 if success else 1) + assert all(event["status"] == status and not event["will_retry"] for event in events) + + +def policy_retry_args(tmp_path): + source = tmp_path / "input.json" + source.write_text( + json.dumps( + [ + { + "sample_id": "sample", + "conversation": { + "session_1_date_time": "2025-01-01", + "session_1": [{"dia_id": "D1:1", "speaker": "A", "text": "synthetic fact"}], + }, + "qa": [{"question": "fact?", "category": 1, "evidence": ["D1:1"]}], + } + ] + ) + ) + return SimpleNamespace( + dataset=source, + kind="locomo", + conversations=1, + top_k=100, + adapter_url="http://adapter.test", + output=tmp_path / "run.json", + resume=False, + ) + + +@pytest.mark.parametrize("failure", [408, 429, 503, "transport"]) +def test_policy_transient_failure_retries_identical_body_and_audits_then_searches( + tmp_path, + monkeypatch, + failure, +): + args = policy_retry_args(tmp_path) + adds, policies, searches, delays = [], [], [], [] + monkeypatch.setenv("ADAPTER_API_KEY", "synthetic-policy-key") + monkeypatch.setattr("src.smoke.time.sleep", delays.append) + + def route(request): + body = json.loads(request.content) if request.content else {} + if request.url.path == "/add": + adds.append(body) + return httpx.Response( + 200, + json={ + "success": True, + **{k: body[k] for k in ("request_id", "user_id", "session_id")}, + }, + ) + if request.url.path == "/diagnostics/render-policy": + assert request.headers["Authorization"] == "Bearer synthetic-policy-key" + policies.append(body) + if len(policies) == 1: + if failure == "transport": + raise httpx.ReadTimeout("synthetic upstream detail", request=request) + return httpx.Response( + failure, + headers={"Retry-After": "4"}, + json={"detail": "synthetic upstream detail"}, + ) + return httpx.Response(200, json={"policy": token_policy(4096)}) + if request.url.path == "/search": + searches.append(body) + return httpx.Response( + 200, + json={ + "data": [ + {"id": "synthetic", "content": "[user]\n[2025-01-01] A: synthetic fact"} + ] + }, + ) + return httpx.Response(200, json={"status": "ok"}) + + original = httpx.Client + monkeypatch.setattr( + benchmark.httpx, "Client", lambda **kw: original(transport=httpx.MockTransport(route), **kw) + ) + assert benchmark.run(args) == 0 + assert len(adds) == 1 and len(searches) == 1 + assert policies == [adds[0], adds[0]] + assert len(delays) == 1 + if failure in {429, 503}: + assert 4 <= delays[0] <= 5 + audit_text = args.output.with_suffix(".requests.jsonl").read_text() + events = [ + json.loads(line) + for line in audit_text.splitlines() + if json.loads(line)["route"] == "/diagnostics/render-policy" + ] + assert [event["attempt"] for event in events] == [1, 2] + assert [event["will_retry"] for event in events] == [True, False] + assert [event["status"] for event in events] == [failure, 200] + assert len({event["request_sha256"] for event in events}) == 1 + for value in ("synthetic-policy-key", "synthetic fact", "synthetic upstream detail"): + assert value not in audit_text + report = json.loads(args.output.read_text()) + assert report["summary"]["mean_evidence_recall"] == 1 + assert any( + row["route"] == "/diagnostics/render-policy" and row["retried_attempts"] == 1 + for row in report["request_diagnostics"] + ) + + +def test_policy_retry_exhaustion_preserves_add_checkpoint_and_resume_skips_add( + tmp_path, + monkeypatch, +): + args = policy_retry_args(tmp_path) + calls, delays = [], [] + unavailable = True + monkeypatch.setattr("src.smoke.time.sleep", delays.append) + + def route(request): + calls.append(request.url.path) + if request.url.path == "/diagnostics/render-policy": + return httpx.Response(503 if unavailable else 200, json={"policy": token_policy(4096)}) + if request.url.path == "/add": + body = json.loads(request.content) + return httpx.Response( + 200, + json={ + "success": True, + **{k: body[k] for k in ("request_id", "user_id", "session_id")}, + }, + ) + if request.url.path == "/search": + return httpx.Response(200, json={"data": []}) + return httpx.Response(200, json={"status": "ok"}) + + original = httpx.Client + monkeypatch.setattr( + benchmark.httpx, "Client", lambda **kw: original(transport=httpx.MockTransport(route), **kw) + ) + assert benchmark.run(args) == 1 + report = json.loads(args.output.read_text()) + assert report["stage"] == "render-policy" and report["error"]["http_status"] == 503 + assert len(report["completed_adds"]) == 1 and not report["questions"] + assert calls.count("/diagnostics/render-policy") == 6 and len(delays) == 5 + assert calls.count("/add") == 1 and "/search" not in calls + unavailable = False + args.resume = True + assert benchmark.run(args) == 0 + assert calls.count("/add") == 1 and calls.count("/search") == 1 + assert calls.count("/diagnostics/render-policy") == 7 + events = [ + json.loads(line) + for line in args.output.with_suffix(".requests.jsonl").read_text().splitlines() + if json.loads(line)["route"] == "/diagnostics/render-policy" + ] + assert [event["will_retry"] for event in events] == [True] * 5 + [False, False]