修复长文本 embedding 超限，并固定请求分片策略

.github/workflows/aml-benchmark.yml

@@ -71,6 +71,13 @@ jobs:
       - name: Install locked benchmark dependencies
         timeout-minutes: 5
         run: uv sync --locked --extra evaluation
+      - name: Cache pinned embedding tokenizer
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: agent_memory_leaderboard/src/assets/tokenizer.json
+          key: bge-m3-${{ hashFiles('agent_memory_leaderboard/src/embedding_tokens.py') }}
+      - name: Prepare pinned embedding tokenizer
+        run: uv run --no-sync python -m src.embedding_tokens
       - name: Run Add/Search and selected evaluation mode
         id: execute
         timeout-minutes: 120

.github/workflows/aml-ci.yml

@@ -29,6 +29,12 @@ jobs:
           enable-cache: true
           cache-dependency-glob: agent_memory_leaderboard/uv.lock
       - run: uv sync --locked --extra evaluation
+      - name: Cache pinned embedding tokenizer
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: agent_memory_leaderboard/src/assets/tokenizer.json
+          key: bge-m3-${{ hashFiles('agent_memory_leaderboard/src/embedding_tokens.py') }}
+      - run: uv run --no-sync python -m src.embedding_tokens
       - run: uv run --no-sync ruff check src tests benchmarks scripts/benchmark_ci.py ../deploy/scripts ../deploy/tests
       - run: uv run --no-sync pytest -q
       - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4

agent_memory_leaderboard/.gitignore

@@ -22,3 +22,6 @@ state/*.json
 *.sqlite3-journal
 *.sqlite3-wal
 *.sqlite3-shm
+
+# Pinned public tokenizer downloaded and verified during setup/build.
+src/assets/tokenizer.json

agent_memory_leaderboard/Dockerfile

@@ -8,6 +8,9 @@ RUN if [ "$INSTALL_EXPERIMENT_DEPS" = "true" ]; then \
       pip install --no-cache-dir --require-hashes -r requirements.lock; \
     fi
 RUN useradd --uid 1000 --create-home adapter && mkdir /app/var && chown adapter /app/var
+COPY src/assets ./src/assets
+COPY src/embedding_tokens.py ./src/embedding_tokens.py
+RUN python -m src.embedding_tokens
 COPY src ./src
 USER adapter
 EXPOSE 8000

agent_memory_leaderboard/README.md

@@ -41,6 +41,7 @@ personal API key cannot provide isolation for multiple leaderboard users.
 
 ```sh
 uv sync --locked
+uv run python -m src.embedding_tokens
 export MEMORIA_URL=http://127.0.0.1:8100
 export MEMORIA_MASTER_KEY='your-memoria-master-key'
 export ADAPTER_API_KEY='a-separate-leaderboard-access-key'
@@ -626,3 +627,70 @@ The self-run benchmark client can generate bounded, private local per-question
 Add/Search/Answer/Eval reports for independently sourced synthetic or permitted
 public data. This is disabled by default and adds no official-request body logs.
 See [the offline demonstration, source declaration and limits](benchmarks/EVALUATION.md#opt-in-local-content-diagnostics).
+
+### Token-aware ingestion and retry compatibility
+
+New Add requests use both the existing 32 KiB UTF-8 bound and a BGE-M3 token
+budget. `ADAPTER_EMBEDDING_TOKEN_BUDGET` defaults to **7680** (allowed range
+128–8192); the count includes the transmitted role/time prefix and special
+tokens. Original Unicode is sliced without tokenizer decoding, truncation or
+summarization. Short messages that fit both limits remain unchanged. Splitting
+increases write/embedding/visibility-check work for long messages.
+
+The tokenizer is pinned to `BAAI/bge-m3` revision
+`5617a9f61b028005a4858fdac845db406aefb181`, with SHA-256
+`21106b6d7dab2952c1d496fb21d5dc9db75c28ed361a05f5020bbba27810dd08`.
+The upstream [BGE-M3 model/tokenizer](https://huggingface.co/BAAI/bge-m3) is
+MIT-licensed. The tokenizer runtime is locked with the Python dependencies.
+This configuration targets BGE-M3's 8192-token input limit; it is not a generic
+limit for other embedding models. Changing the backend model requires checking
+its tokenizer and adding a compatible, versioned rendering policy first.
+
+Run `uv run python -m src.embedding_tokens` after installing dependencies and
+before local startup/tests. This downloads only the public tokenizer, verifies
+its checksum and stores it in ignored `src/assets/tokenizer.json`. The image
+build does this automatically. `ADAPTER_TOKENIZER_PATH` may point to a preloaded
+copy of the **same** asset. Runtime requests never download assets; missing or
+mismatched assets fail application startup. For offline builds, preload the
+verified asset in the build context. An existing verified copy is reused.
+
+Each request's journal row pins its rendering version, tokenizer digest and
+budget in the same durable transaction as its first write intent. Changing the
+default budget only affects new requests. Existing journals migrate old rows to
+byte-only version 1, retaining their chunk boundaries, positions and idempotency
+keys. Completed retries return without re-rendering or backend calls. Unknown
+policies fail before sending a write. The number of durable journal commits
+remains three per new chunk; policy lookup adds one read per Add.
+
+**This does not automatically repair old oversized requests.** An incomplete v1
+request still uses its original byte-only chunks, even if one exceeds the model
+limit. Never clear its journal or reinterpret existing positions to force a
+retry. Coordinate an explicit replay/migration plan for such requests. Keep one
+adapter worker/replica and stop the old version before upgrading. Do not roll
+back to a byte-only binary with unfinished v2 requests. Token-policy receipts use
+an unknown-to-old-code protocol, and SQLite triggers reject v1 writes to any new
+position of a v2 request. Old versions fail with 503 instead of silently skipping
+content. If an old binary reaches the SQLite trigger, it raises an IntegrityError
+with `Unsupported rendering protocol`. The pre-upgrade journal catches SQLite
+errors and returns generic HTTP 503 `Idempotency journal unavailable`, rather than
+the explicit version-check message from its protocol whitelist. Check the deployed
+binary/journal versions first; restore the compatible binary, not delete the journal
+or trigger. These guards require retaining the journal schema and triggers. Preserve
+the journal and fixed model/tokenizer for the whole run. Ordinary Search queries are unchanged; the write-visibility search
+uses each newly bounded chunk.
+
+The local benchmark reads each Add request's persisted policy from the authenticated,
+read-only `POST /diagnostics/render-policy` endpoint (same Add body and credentials).
+It records per-message policies in source manifests, including mixed v1/v2 requests
+and resumed runs. Missing/invalid policies stop attribution explicitly instead of
+silently using a default. Policy lookups use the same six-attempt bounded retry,
+Retry-After handling and body-free request audit as Add/Search. A failed lookup
+keeps completed Add checkpoints, so resume does not repeat successful writes.
+To evaluate a known old byte-only adapter lacking this
+endpoint, explicitly pass `--legacy-byte-rendering`; only a 404 permits that
+fallback. The flag is part of the resume contract and rejects a detected v2 policy.
+Pre-change benchmark checkpoints lack this policy contract and cannot be resumed by
+this runner; preserve them for analysis and use the matching runner or a new run.
+
+The tokenizer download has its own Docker layer, independent of other application
+source changes. CI caches the public asset and always verifies its checksum.

agent_memory_leaderboard/pyproject.toml

@@ -9,6 +9,7 @@ dependencies = [
     "httpx>=0.28,<1",
     "pydantic>=2.10,<3",
     "uvicorn>=0.34,<1",
+    "tokenizers==0.22.2",
 ]
 
 [project.optional-dependencies]

agent_memory_leaderboard/requirements-experiment.lock

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv export --locked --no-dev --extra evaluation --format requirements-txt --output-file requirements-experiment.lock
+#    uv export --locked --no-dev --extra evaluation --no-emit-project --format requirements-txt --output-file requirements-experiment.lock
 annotated-doc==0.0.5 \
     --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \
     --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb
@@ -151,25 +151,64 @@ charset-normalizer==3.5.2 \
 click==8.5.0 \
     --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \
     --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34
-    # via uvicorn
+    # via
+    #   huggingface-hub
+    #   uvicorn
+colorama==0.4.6 ; sys_platform == 'win32' \
+    --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
+    --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
+    # via tqdm
 fastapi==0.142.2 \
     --hash=sha256:06366626f2e70576367714d9ab2fe8472e6c8456dba69b399f9f797ab5e92570 \
     --hash=sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b
     # via agent-memory-leaderboard
+filelock==4.2.4 \
+    --hash=sha256:301241438c57936ef93f4243a94f2559022268bfcf9b375a9a72eb0f8db8358e \
+    --hash=sha256:c6d8663535e1650ee430507b4ffbac7b464dd64b7647ea0112a9f406499d163d
+    # via huggingface-hub
+fsspec==2026.9.0 \
+    --hash=sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe \
+    --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f
+    # via huggingface-hub
 h11==0.16.0 \
     --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \
     --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
     # via
     #   httpcore
     #   uvicorn
+hf-xet==1.8.0 ; platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \
+    --hash=sha256:07a06d7481702616726c2d8578cfb00052a5b3f21beed6a674592bc837a10330 \
+    --hash=sha256:10ed23b51dacf7917d64f335c25048deb470efa755e9989b218fc0531386138e \
+    --hash=sha256:1ebaf248c9bef8bb0c9d1d8d71db1fdf0cac14f922cc7a936a81adbf12eb5f01 \
+    --hash=sha256:23c262e9f5e137b18dadbce60becfeea84156b0d43119c320bffe260d32cd2fa \
+    --hash=sha256:29dab7c904cd1914a0439c6e1306c12c7497c8bd32290fbe563ebf51ca4b6d6d \
+    --hash=sha256:3e8934784425781f0f779b91044df441f3a7af4dab47375d7e3aeaecbb8eecb4 \
+    --hash=sha256:3ecfe2934448cca55a4c42732b60f985266aecb241905945219b3d388aceb7ba \
+    --hash=sha256:42b66fdc688eaa4ff3eb77dc9275e5a0f1258ecc9fdd083ddaf43bb49bec925b \
+    --hash=sha256:56735c2d4f416d7473fc986586b26a8756e067832ca4b7fdaad32a0d6f2ce740 \
+    --hash=sha256:62b056aedb89ac809f78940baf465fe553b70c8b25ee56890fbab3039fffec1b \
+    --hash=sha256:7edeb21c5d381903ee5f4857fc746fa7b52a4b374035e8811ecfb67ee27add0c \
+    --hash=sha256:8800e3444770f2c630b361bc6d5c6ebb770b7830646cf632fbb79d0e5bbc9e7f \
+    --hash=sha256:92188598dedc1e1f5609bd718d4db8687a6c89915553912571e2f587892eecec \
+    --hash=sha256:a584a7b3268da62f23c3187cc7201ca01f7b610985a36b1b6119c66bccb0383c \
+    --hash=sha256:c74e3751cbcdae9222c5170e8d788e83201a03dc5e5f4308a20f17a5b6023b56 \
+    --hash=sha256:e00d8f60170cd3dd55d98306e41dbaf28b27ff1bcfefbfa84279d197221be83e \
+    --hash=sha256:f42940869572dc2b05dfe06f4cc398c6291503e086ddc4eddfb7a6ab9409ad01
+    # via huggingface-hub
 httpcore==1.0.9 \
     --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \
     --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8
     # via httpx
 httpx==0.28.1 \
     --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
     --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
-    # via agent-memory-leaderboard
+    # via
+    #   agent-memory-leaderboard
+    #   huggingface-hub
+huggingface-hub==1.33.0 \
+    --hash=sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899 \
+    --hash=sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e
+    # via tokenizers
 idna==3.20 \
     --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \
     --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c
@@ -181,6 +220,10 @@ opentelemetry-api==1.45.0 \
     --hash=sha256:711ede81773c8025c2c03dac0450bc89f3d30aea6eabcc815c570d4e35a963f7 \
     --hash=sha256:80e068aba7cd56c8b58512d6a36f8d25cb1dfaa0c0a4cc1c938ccf9f362d9cb3
     # via fastapi
+packaging==26.3 \
+    --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \
+    --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c
+    # via huggingface-hub
 pydantic==2.13.5 \
     --hash=sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73 \
     --hash=sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08
@@ -254,6 +297,47 @@ pydantic-core==2.46.5 \
     --hash=sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0 \
     --hash=sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9
     # via pydantic
+pyyaml==6.0.3 \
+    --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \
+    --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \
+    --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \
+    --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \
+    --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \
+    --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \
+    --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \
+    --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \
+    --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \
+    --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \
+    --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \
+    --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \
+    --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \
+    --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \
+    --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \
+    --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \
+    --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \
+    --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \
+    --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \
+    --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \
+    --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \
+    --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \
+    --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \
+    --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \
+    --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \
+    --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \
+    --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \
+    --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \
+    --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \
+    --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \
+    --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \
+    --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \
+    --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \
+    --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \
+    --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \
+    --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \
+    --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \
+    --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \
+    --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0
+    # via huggingface-hub
 regex==2026.9.29 \
     --hash=sha256:01000ddf0e3ffef97f2413ceb514f6313040106b6d18a03ee00a4fe35c1eb1db \
     --hash=sha256:0166844493626c5015c6088ee15c9ca2fd060ca15b7641d1657da6a58432ae33 \
@@ -406,12 +490,35 @@ tiktoken==0.14.0 \
     --hash=sha256:f2af4a336ea56d6c14f27741a0e1d8294a35dd0b038bcf990d232ebb54eb994b \
     --hash=sha256:f702e0aeeb6506e57687e881c59e844ebe8f0a6a097ddafe20e3ab25f387be4e
     # via agent-memory-leaderboard
+tokenizers==0.22.2 \
+    --hash=sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e \
+    --hash=sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001 \
+    --hash=sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7 \
+    --hash=sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd \
+    --hash=sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4 \
+    --hash=sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67 \
+    --hash=sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a \
+    --hash=sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5 \
+    --hash=sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917 \
+    --hash=sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c \
+    --hash=sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a \
+    --hash=sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc \
+    --hash=sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92 \
+    --hash=sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5 \
+    --hash=sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48 \
+    --hash=sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b
+    # via agent-memory-leaderboard
+tqdm==4.70.1 \
+    --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 \
+    --hash=sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4
+    # via huggingface-hub
 typing-extensions==4.16.0 \
     --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \
     --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5
     # via
     #   anyio
     #   fastapi
+    #   huggingface-hub
     #   opentelemetry-api
     #   pydantic
     #   pydantic-core