test(server): add tool-state isolation smoke harness

.github/workflows/ci.yml

@@ -15,6 +15,14 @@ on:
 # Also turn on "Require approval for all outside collaborators" in repo Actions
 # settings before making this repo public.
 jobs:
+  benchmark-tools:
+    name: benchmark tooling (unit tests)
+    if: github.event_name == 'push' || (github.event.pull_request.head.repo.full_name == github.repository && (github.base_ref == 'main' || github.base_ref == 'dev'))
+    runs-on: [self-hosted, macOS, ARM64, m4pro]
+    steps:
+      - uses: actions/checkout@v4
+      - run: python3 -m unittest discover -s benchmarks/tests -v
+
   base-convert:
     name: base-convert (build + test)
     if: github.event_name == 'push' || (github.event.pull_request.head.repo.full_name == github.repository && (github.base_ref == 'main' || github.base_ref == 'dev'))

.github/workflows/serve-smoke.yml

@@ -1,6 +1,6 @@
 name: Server smoke
 
-# Manually-triggered: start baseRT_serve from an engine release and assert it
+# Manually-triggered: start basert-serve from an engine release and assert it
 # answers an OpenAI /v1/chat/completions request. Needs an engine release and a
 # .base model URL.
 
@@ -10,6 +10,11 @@ on:
       model_url:
         description: "URL to a .base model"
         required: true
+      run_tool_state_smoke:
+        description: "Run sequential/concurrent tool-state regression checks"
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   serve:
@@ -20,18 +25,20 @@ jobs:
         run: |
           mkdir -p build
           gh release download --repo ${{ github.repository }} \
-            --pattern 'baseRT-engine-macos-arm64*.tar.gz' -D build
-          tar -xzf build/baseRT-engine-macos-arm64*.tar.gz -C build
+            --pattern 'basert-engine-macos-arm64*.tar.gz' -D build
+          tar -xzf build/basert-engine-macos-arm64*.tar.gz -C build
         env:
           GH_TOKEN: ${{ github.token }}
       - name: Fetch model
         run: |
           mkdir -p models
-          curl -fsSL "${{ inputs.model_url }}" -o models/model.base
-      - name: Start server + assert a completion
+          curl -fsSL "$MODEL_URL" -o models/model.base
+        env:
+          MODEL_URL: ${{ inputs.model_url }}
+      - name: Start server + run smoke checks
         run: |
           KEY="ci-$RANDOM"
-          ./build/baseRT_serve --model models/model.base --api-key "$KEY" --port 8080 \
+          ./build/basert-serve models/model.base --api-key "$KEY" --port 8080 \
             > serve.log 2>&1 &
           SV=$!
           trap 'kill $SV 2>/dev/null || true' EXIT
@@ -44,3 +51,11 @@ jobs:
             http://127.0.0.1:8080/v1/chat/completions)
           echo "$RESP"
           echo "$RESP" | python3 -c "import sys,json; c=json.load(sys.stdin)['choices'][0]['message']['content']; assert c, 'empty completion'; print('OK:', repr(c[:60]))"
+          if [[ "$RUN_TOOL_STATE_SMOKE" == "true" ]]; then
+            python3 benchmarks/scripts/tool_state_smoke.py \
+              --base-url http://127.0.0.1:8080/v1 \
+              --model model.base \
+              --api-key "$KEY"
+          fi
+        env:
+          RUN_TOOL_STATE_SMOKE: ${{ inputs.run_tool_state_smoke }}

benchmarks/README.md

@@ -23,6 +23,36 @@ Results are written to `benchmarks/results/<arch>_baseRT.csv` with columns
 throughput at prompt length N; `tgN` rows are decode (token-generation)
 throughput.
 
+## Tool-state regression smoke
+
+`tool_state_smoke.py` checks whether repeated tool schemas preserve isolated
+arguments between requests. It sends distinct `lookup_key` sentinels in 10
+sequential requests, then concurrent batches of 2 and 4 by default. Failures are
+consistent with cross-request state leakage, but this diagnostic does not prove
+an engine root cause by itself. It is opt-in for tool-capable models; plain chat
+models are expected to fail it:
+
+```sh
+python3 benchmarks/scripts/tool_state_smoke.py \
+  --base-url http://127.0.0.1:8080/v1 \
+  --model model.base \
+  --sequential 10 --concurrency 2 4 \
+  --timeout 300 --max-tokens 2048
+```
+
+Set `BASERT_API_KEY` in the environment when the server requires a bearer
+token. The harness uses only the Python standard library. Each mismatch is
+printed as a `FAIL` JSON record, followed by a `SUMMARY` JSON record, and any
+mismatch makes the process exit nonzero.
+
+The manual **Server smoke** workflow can run this check after its basic chat
+completion by enabling `run_tool_state_smoke`. Pure unit tests use a local fake
+HTTP server and do not require an engine or model:
+
+```sh
+python3 -m unittest benchmarks.tests.test_tool_state_smoke -v
+```
+
 ## Example results
 
 `results/m4-pro_baseRT.csv` and `results/m3-base_baseRT.csv` hold reference