From b8c8d95ceecd1501073d5515579023d0cc1cd18b Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Thu, 9 Jul 2026 04:53:21 +0800 Subject: [PATCH] feat!: remove dollar metering and the $20 budget cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runs are bounded by the agent step limit only (per-rule 35, whole-repo 300); mini-swe-agent's default cost check is disabled in one place (cost_limit=0). Token usage (4-bucket usage_totals, total_tokens_k, bugs/Ktok) stays. - cost.py → usage.py: drop Price/pricing; keep token accounting - runner/scheduler: drop per-rule budget, spend tracking, skipped_budget - run_submission: drop BUDGET_USD/PER_RULE_BUDGET/SAFETY_MARGIN/PRICE_*; schema_version 2.0 (no budget_cap/total_cost_usd/prices; rows lose "cost") - verify_submission/backend_score: drop cost re-metering and $-efficiency; tokens recomputed from usage_totals - site: drop $20 copy, Spent column, bugs/$, budget_cap ranking filter - docs/env/skill: drop price+budget knobs Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/skills/run-benchmark/SKILL.md | 30 ++-- .../references/env-and-troubleshoot.md | 11 +- .github/scripts/check_aggregate.py | 6 +- .github/workflows/score-from-r2.yml | 2 +- CONTRIBUTING.md | 50 +++--- Makefile | 8 +- README.md | 12 +- benchmark/backend_score.py | 14 +- benchmark/config.yaml | 3 +- benchmark/config_repo.yaml | 15 +- benchmark/preflight.py | 21 +-- benchmark/results.schema.json | 32 +--- benchmark/run_mini.py | 141 ++++++--------- benchmark/run_submission.py | 161 +++++------------- benchmark/runner.py | 27 +-- benchmark/scheduler.py | 66 +------ benchmark/submission.schema.json | 36 +--- benchmark/submit.py | 4 +- benchmark/tests/test_backend_score.py | 24 ++- benchmark/tests/test_preflight.py | 10 +- benchmark/tests/test_run_submission.py | 110 +++++------- benchmark/tests/test_scheduler.py | 122 +++---------- benchmark/tests/test_submit.py | 12 +- .../tests/{test_cost.py => test_usage.py} | 51 +----- benchmark/tests/test_verify_submission.py | 50 +++--- benchmark/tests/test_whole_repo.py | 33 ++-- benchmark/{cost.py => usage.py} | 75 +------- benchmark/verify_submission.py | 57 +++---- site/README.md | 4 +- site/index.html | 46 +++-- submission.env.example | 19 +-- submissions/README.md | 1 - 32 files changed, 360 insertions(+), 893 deletions(-) rename benchmark/tests/{test_cost.py => test_usage.py} (61%) rename benchmark/{cost.py => usage.py} (52%) diff --git a/.claude/skills/run-benchmark/SKILL.md b/.claude/skills/run-benchmark/SKILL.md index da0ba0b..92a81c2 100644 --- a/.claude/skills/run-benchmark/SKILL.md +++ b/.claude/skills/run-benchmark/SKILL.md @@ -3,7 +3,7 @@ name: run-benchmark description: >- Run this repo's problem-reductions bug-finding benchmark end-to-end and produce out/submission.json: detect a container engine, build the runner image, configure - submission.env, preflight, and run the budgeted agent. Works on macOS and Linux + submission.env, preflight, and run the step-limited agent. Works on macOS and Linux (Docker or rootless Podman). Use when asked to run, test, reproduce, or smoke-test the benchmark, or to generate a submission for a model. NOT for `make test-unit` / pytest. --- @@ -30,9 +30,7 @@ Step 6 — don't ask for intake secrets unless they're submitting. Confirm the user has: -- A **model API key** and its **price per 1M tokens** (input + output) — needed for *either* - goal, since running the agent calls the model. Both *required*; a real run hard-fails - without `PRICE_IN`/`PRICE_OUT` (there is no built-in price table). +- A **model API key** — needed for *either* goal, since running the agent calls the model. - **Only if submitting**: `PRB_SUBMIT_URL` + `PRB_API_KEY`, from the maintainer. - **git** and a **container engine** (checked next). @@ -78,16 +76,13 @@ If `submission.env` is absent: `cp submission.env.example submission.env`. It's ```ini MODEL_NAME=openai/gpt-5.4 # any LiteLLM-routable name (anthropic/… openai/… openrouter/… gemini/…) API_KEY=sk-... # generic; or a provider var (OPENAI_API_KEY / ANTHROPIC_API_KEY / …) -PRICE_IN=3.0 # USD / 1M input tokens — REQUIRED -PRICE_OUT=15.0 # USD / 1M output tokens — REQUIRED ``` -For a cheap smoke run (don't spend the full $20) add `MAX_RULES=1`. A **ranked** submission -must keep `BUDGET_USD=20` and omit `MAX_RULES`. +For a cheap smoke run add `MAX_RULES=1`. A **ranked** submission must omit `MAX_RULES`. **Agent mode** (`AGENT_MODE`, default `per-rule`): `per-rule` runs one isolated agent session -per rule with the budget split evenly; `whole-repo` runs ONE session over the whole library -and lets the agent enumerate and triage the rules itself under a single budget. Both produce +per rule (35 steps each); `whole-repo` runs ONE session over the whole library (300 steps) +and lets the agent enumerate and triage the rules itself. Both produce the same `out/submission.json` and are scored identically — set `AGENT_MODE=whole-repo` to try it. (`MAX_RULES` only applies to `per-rule`.) In `whole-repo`, the agent also writes each certificate to `TRAJECTORY_DIR/certs.txt` (default `/out`) as it finds it, and the trajectory @@ -99,13 +94,12 @@ overwrite each other. **Confirm the experiment parameters with the user — don't silently default them.** These shape the result and the spend, so state the resolved set and get an explicit OK before -running: **mode** (`AGENT_MODE`), **budget** (a full ranked run at `BUDGET_USD=20`, or a -cheap smoke run via `MAX_RULES=1` / a smaller budget), and — only if they care — -`PER_RULE_BUDGET` and `MAX_TOKENS`. Ranked runs require `BUDGET_USD=20` and no `MAX_RULES`. +running: **mode** (`AGENT_MODE`), full ranked run vs cheap smoke run (`MAX_RULES=1`), and — +only if they care — `MAX_TOKENS`. Ranked runs must omit `MAX_RULES`. ## Step 4 — Preflight (one tiny real API call, ~a fraction of a cent) -Always run this before the full run; it validates key/endpoint/price + pred/rules through the +Always run this before the full run; it validates key/endpoint + pred/rules through the exact batch code path and fails fast. - **docker**: `make preflight` @@ -113,12 +107,12 @@ exact batch code path and fails fast. It prints three checks (`pred binary`, `library rules`, `model call`). If any **FAIL**, read the detail and fix it — the `model call` line carries the real error (auth / endpoint / model -name / pricing). Decode table in `references/env-and-troubleshoot.md`. Do not proceed on a FAIL. +name). Decode table in `references/env-and-troubleshoot.md`. Do not proceed on a FAIL. ## Step 5 — Full run → out/submission.json **Gate**: a full run spends real money and takes a while. Restate the resolved parameters -(model, mode, budget, any smoke caps) and get an explicit OK before you launch it. +(model, mode, any smoke caps) and get an explicit OK before you launch it. - **docker**: `make run` - **podman/raw**: use the `RUN_FLAGS` from Step 1, e.g. @@ -131,11 +125,11 @@ name / pricing). Decode table in `references/env-and-troubleshoot.md`. Do not pr user may need `sudo`/`chown` to move it.) The run needs network only to reach the model API. When it finishes, confirm -`out/submission.json` exists and report the result (bugs found, spend). +`out/submission.json` exists and report the result (bugs found, tokens used). ## Step 6 — Hand back, or submit -`out/submission.json` now exists; report the result (bugs found, spend). Then branch on the +`out/submission.json` now exists; report the result (bugs found, tokens used). Then branch on the goal from Step 0: - **Run/test locally** → done. `out/submission.json` is the deliverable; no secrets, no upload. diff --git a/.claude/skills/run-benchmark/references/env-and-troubleshoot.md b/.claude/skills/run-benchmark/references/env-and-troubleshoot.md index 744286a..81e3dea 100644 --- a/.claude/skills/run-benchmark/references/env-and-troubleshoot.md +++ b/.claude/skills/run-benchmark/references/env-and-troubleshoot.md @@ -13,20 +13,15 @@ these as env vars (CLI flags would override, but the skill uses the env-file). |---|---|---| | `MODEL_NAME` | LiteLLM-routable model name (`anthropic/…`, `openai/…`, `openrouter/…`, `gemini/…`, or `openai/` + `API_BASE`) | run hard-errors: "`--model (or env MODEL_NAME) is required`" | | API key | `API_KEY` (generic) **or** a provider var (`OPENAI_API_KEY`/`ANTHROPIC_API_KEY`/`OPENROUTER_API_KEY`/`GEMINI_API_KEY`) — provider vars pass straight through to LiteLLM | not checked in Python; surfaces at the `model call` preflight as an auth error | -| `PRICE_IN`, `PRICE_OUT` | USD / 1M input & output tokens; spend = tokens × price (the $20 cap basis) | **must be given together**; a real run hard-errors if absent — there is deliberately no built-in price table | ### Optional (defaults shown; uncomment only to change) | Var | Default | Use | |---|---|---| -| `PRICE_CACHE_READ` / `PRICE_CACHE_WRITE` | 0 | prompt-caching models | | `API_BASE` | — | OpenAI-compatible endpoint (OpenRouter/gateway/vLLM/Azure) | | `MODEL_KWARGS` | — | JSON object of extra litellm kwargs (`custom_llm_provider`, `api_version`, `extra_headers`…). Invalid JSON / non-object errors at startup | -| `BUDGET_USD` | 20 | must be **20 to be ranked** (not enforced by the runner; unrankable otherwise) | -| `PER_RULE_BUDGET` | 0.5 | per-rule cost cap | -| `SAFETY_MARGIN` | 1.0 | USD held back so the budget-crossing call stays under cap | | `MAX_TOKENS` | 8192 | per-call output ceiling | | `MAX_RULES` | all | cap rules attempted — **smoke runs only**; omit for a ranked run (per-rule only) | -| `AGENT_MODE` | `per-rule` | `per-rule` (isolated session/rule, budget split evenly) or `whole-repo` (ONE session, the agent triages the rules itself) | +| `AGENT_MODE` | `per-rule` | `per-rule` (isolated session per rule, 35 steps each) or `whole-repo` (ONE session, 300 steps, the agent triages the rules itself) | | `TRAJECTORY_DIR` | `OUTPUT`'s dir (`/out`) | where **whole-repo** persists the trajectory + the durable incremental cert log (`certs.txt`); the agent writes each certificate here the moment it finds it, so an early-stop/crash still leaves the found bugs on disk | | `AGENT_CONFIG` / `AGENT_STRATEGY_FILE` | bundled | bring-your-own prompt; the files must be **mounted** into the container (`-v "$PWD/cfg:/cfg"`) and the path given as a container path | | `SUBMITTED_BY` | — | your handle, recorded in the envelope | @@ -40,8 +35,6 @@ MODEL_NAME=openai/my-model API_BASE=https://my-gateway.example/v1 API_KEY=... MODEL_KWARGS={"custom_llm_provider":"openai"} -PRICE_IN=1.5 -PRICE_OUT=6.0 ``` ## Preflight failure decoding @@ -53,7 +46,7 @@ checks and exits non-zero if any fail. It never raises — it prints `PASS`/`FAI |---|---|---| | **pred binary** | pred missing or version ≠ pinned | should always pass inside the image; a FAIL = broken/overridden image or a wrong `EXPECTED_PRED_VERSION`. Rebuild at the right `PR_REF` | | **library rules** | no `.rs` rules under `REPO_DIR/src/rules` | source tree not copied / `REPO_DIR` overridden. Rebuild the image; don't set `REPO_DIR` | -| **model call** | the real error (spends ~$0.0001) — its detail names the exception type | **auth error** → bad/missing key; **connection error** → wrong `API_BASE`/endpoint; **routing/model-not-found** → wrong `MODEL_NAME`; **pricing** → `PRICE_IN`/`PRICE_OUT`. Fix that line in `submission.env` and rerun preflight | +| **model call** | the real error (spends ~$0.0001) — its detail names the exception type | **auth error** → bad/missing key; **connection error** → wrong `API_BASE`/endpoint; **routing/model-not-found** → wrong `MODEL_NAME`. Fix that line in `submission.env` and rerun preflight | Only proceed to the full run when preflight prints `Preflight PASSED`. diff --git a/.github/scripts/check_aggregate.py b/.github/scripts/check_aggregate.py index df7d5ba..5decbf0 100644 --- a/.github/scripts/check_aggregate.py +++ b/.github/scripts/check_aggregate.py @@ -15,9 +15,9 @@ from pathlib import Path ALLOWED_KEYS = { - "model", "library_commit", "budget_cap", "bugs_found", "rules_tested", - "total_cost_usd", "total_tokens_k", "efficiency_bugs_per_ktok", - "efficiency_bugs_per_dollar", "submitted_by", "placeholder", + "model", "library_commit", "bugs_found", "rules_tested", + "total_tokens_k", "usage_totals", "efficiency_bugs_per_ktok", + "submitted_by", "placeholder", # per-submission entry files (site/results/.json) also carry provenance tags "timestamp", "submission_id", } diff --git a/.github/workflows/score-from-r2.yml b/.github/workflows/score-from-r2.yml index 2611ed0..6b9dc04 100644 --- a/.github/workflows/score-from-r2.yml +++ b/.github/workflows/score-from-r2.yml @@ -135,7 +135,7 @@ jobs: echo "updated PR #$EXISTING ($slug)" else gh pr create --base "${{ github.ref_name }}" --head "$BR" --title "$TITLE" \ - --body "One submission's public entry (\`$dest\`) — aggregate only (counts / cost / tokens / efficiency), no certificates or rule identities (guarded by \`check_aggregate.py\`). The full certificate stays private in R2. Merging rebuilds \`site/results.json\` from all entries and deploys via publish-on-merge. This PR is scoped to a single submission; merge/close/revert it independently." + --body "One submission's public entry (\`$dest\`) — aggregate only (counts / tokens / efficiency), no certificates or rule identities (guarded by \`check_aggregate.py\`). The full certificate stays private in R2. Merging rebuilds \`site/results.json\` from all entries and deploys via publish-on-merge. This PR is scoped to a single submission; merge/close/revert it independently." fi opened=$((opened+1)) done diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 59018cc..3df6f7e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,7 @@ # Submitting a model run -The benchmark gives every model the **same $20 API budget** and asks: how many distinct -reduction-rule bugs can it find? +The benchmark gives every model the **same step-limited agent session** and asks: how many +distinct reduction-rule bugs can it find? ``` make run ─▶ submission.json ─▶ python -m benchmark.submit ──▶ private store (R2) @@ -18,8 +18,8 @@ recomputed by `pred`. ## 1. Produce a `submission.json` (dockerized runner) The runner image bundles the `pred` binary, the agent stack (mini-swe-agent + LiteLLM), -and the problem-reductions source pinned at `v0.6.0`. LiteLLM enforces the budget across -whatever provider key you supply. +and the problem-reductions source pinned at `v0.6.0`. Any LiteLLM-routable provider key +works. The **target library version is not hardcoded** — it tracks the benchmark. The single knob is the `PR_REF` build arg (a tag or commit of problem-reductions); the image bakes the @@ -39,23 +39,21 @@ All run config goes in **one env-file** so you don't juggle a dozen `-e` flags. template, fill the two required lines, and run: ```bash -cp submission.env.example submission.env # then edit the REQUIRED lines (model, key, price) +cp submission.env.example submission.env # then edit the REQUIRED lines (model, key) mkdir -p out docker run --rm --env-file submission.env -v "$PWD/out:/out" \ problem-reductions-runner:v0.6.0 # → ./out/submission.json (or just: make run) ``` -The **required lines are model + API key + price** (`MODEL_NAME`, a key, `PRICE_IN`/`PRICE_OUT`). -Everything else in the template has a sane default — uncomment only what you need. The knobs, -by tier: +The **required lines are model + API key** (`MODEL_NAME`, a key). Everything else in the +template has a sane default — uncomment only what you need. The knobs, by tier: | Tier | Vars | When | |---|---|---| -| **Required** | `MODEL_NAME`, one API key (`ANTHROPIC_API_KEY` / `OPENAI_API_KEY` / generic `API_KEY`), `PRICE_IN`, `PRICE_OUT` | always — see the price note below | -| **Pricing (caching)** | `PRICE_CACHE_READ`, `PRICE_CACHE_WRITE` | prompt-caching models | +| **Required** | `MODEL_NAME`, one API key (`ANTHROPIC_API_KEY` / `OPENAI_API_KEY` / generic `API_KEY`) | always | | **Non-standard provider** | `API_BASE`, `API_KEY`, `MODEL_KWARGS` (JSON of extra litellm kwargs: `api_version`, `custom_llm_provider`, `extra_headers`, …) | OpenRouter / gateway / local vLLM / Azure | -| **Budget (defaults = ranked config)** | `BUDGET_USD=20`, `PER_RULE_BUDGET=0.5`, `SAFETY_MARGIN=1`, `MAX_TOKENS=8192`, `MAX_RULES` | quick test runs / tuning only | +| **Limits (defaults = ranked config)** | `MAX_TOKENS=8192`, `MAX_RULES` | quick test runs / tuning only | | **Custom prompt** | `AGENT_CONFIG`, `AGENT_STRATEGY_FILE` (mount the files too) | bring your own bug-hunting prompt | | **Version pins** | `EXPECTED_PRED_VERSION` (empty disables), `EXPECTED_PRED_COMMIT` | debugging only — baked from the image build | @@ -64,14 +62,10 @@ by tier: provider. (`REPO_DIR` / `OUTPUT` are container-internal and already defaulted; you don't set them.) -> Why you pass the price (and why it's required): you pay your own bill at your own rate, so -> you set it. The runner computes spend from raw token counts × your price rather than -> trusting the gateway's dollar figure (which can be stale or wrong — LiteLLM $0-pricing -> incidents, Anthropic prompt-cache mis-pricing ~10×), so `$20` is a real cap. There is -> deliberately **no built-in price table**: a wrong default would silently mis-meter the -> budget, so a real run fails fast without `PRICE_IN`/`PRICE_OUT`. The backend re-verifies -> bugs regardless and ranks on **bugs/Ktok** (token counts are auditable); self-reported -> dollars are advisory only. +> Runs are bounded by the agent step limit (35 steps per rule; 300 steps for a whole-repo +> session), not by a dollar budget — you pay your own bill. Raw token counts are recorded +> and travel in the submission (`usage_totals`); ranking is by **confirmed bugs**, with +> **bugs/Ktok** as the efficiency tie-break. For example, a non-standard endpoint in `submission.env`: @@ -80,8 +74,6 @@ MODEL_NAME=openai/my-model API_BASE=https://my-gateway.example/v1 API_KEY=... MODEL_KWARGS={"custom_llm_provider":"openai"} -PRICE_IN=1.5 -PRICE_OUT=6.0 ``` Equivalently with raw `-e` flags (the env-file just bundles these): @@ -92,7 +84,6 @@ docker run --rm \ -e API_BASE=https://my-gateway.example/v1 \ -e API_KEY=$MY_KEY \ -e MODEL_KWARGS='{"custom_llm_provider":"openai","extra_headers":{"X-Org":"acme"}}' \ - -e PRICE_IN=1.5 -e PRICE_OUT=6.0 -e BUDGET_USD=20 \ -v "$PWD/out:/out" \ problem-reductions-runner:v0.6.0 ``` @@ -108,8 +99,8 @@ docker run --rm --env-file submission.env \ # For a full prompt rewrite instead, mount a config.yaml and set AGENT_CONFIG=/cfg/config.yaml. ``` -**Before the full run, validate your config** with one tiny real call (a fraction of a cent) -so a bad key / wrong endpoint / missing price surfaces now, not 20 rules in: +**Before the full run, validate your config** with one tiny real call so a bad key / wrong +endpoint surfaces now, not 20 rules in: ```bash make preflight # docker run --env-file submission.env --preflight @@ -207,14 +198,13 @@ would hide. The round-trip judging itself is explained in the [README](README.md - Counterexamples are **deterministically re-checkable** — we don't even need a hidden answer key; a bug either violates the rule under `pred` or it doesn't. - Distinct-rule de-duplication caps the count at one per rule. -- The $20 budget is enforced inside the runner by recomputing spend from raw token usage × - your declared price (not the gateway's self-reported dollars), held back by a safety - margin and bounded per call by `MAX_TOKENS`; the backend cross-checks reported spend. +- Sessions are bounded by the agent step limit and per call by `MAX_TOKENS`; token totals + travel as raw 4-bucket counts (`usage_totals`) the backend recomputes `total_tokens_k` from. ## Status: validated against a live model The runner pipeline is unit-tested end-to-end with `FakeRunner` + the certificate fixtures **and** has been exercised against a live model API (a DeepSeek OpenAI-compatible endpoint -via `MODEL_NAME=openai/` + `API_BASE`): preflight passes, and a real budgeted run -drives the agent across a rule and emits a schema-valid `submission.json`. PR scoring and -GitHub Pages publishing are live; full `$20` runs are the remaining step. +via `MODEL_NAME=openai/` + `API_BASE`): preflight passes, and a real run drives the +agent across a rule and emits a schema-valid `submission.json`. PR scoring and GitHub Pages +publishing are live; full official runs are the remaining step. diff --git a/Makefile b/Makefile index 510e578..bc0b9ab 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ # preflight Validate submission.env with one tiny real call before a full run # run Run the benchmark via Docker → out/submission.json (does NOT upload) # -# Model + key + price for the real run live in submission.env (any provider — see +# Model + key for the real run live in submission.env (any provider — see # submission.env.example); preflight/submission read it via --env-file. REPO_DIR is only # for the local-clone targets (audit). @@ -47,14 +47,14 @@ runner-build: --build-arg PR_REF=$(PR_REF) -t $(IMAGE) . ## Preflight: validate submission.env with one tiny real API call + pred/rules checks, -## BEFORE committing to a full $20 run. Spends a fraction of a cent. (The no-API wiring of +## BEFORE committing to a full run. Makes one tiny real API call. (The no-API wiring of ## the runner itself is covered by the pytest suite, not a make target.) preflight: @if [ ! -f "$(ENV_FILE)" ]; then \ echo "No $(ENV_FILE) — copy submission.env.example and fill it in first"; exit 1; fi docker run --rm --env-file "$(ENV_FILE)" $(IMAGE) --preflight -## Run the budgeted bug-finding agent via Docker → writes ./out/submission.json. +## Run the bug-finding agent via Docker → writes ./out/submission.json. ## This RUNS the benchmark locally; it does NOT submit — submitting is a separate step ## (open a GitHub PR adding the file, see CONTRIBUTING.md). Config lives in submission.env ## (copy submission.env.example); run `make preflight` first to validate it. @@ -118,4 +118,4 @@ help: @echo "" @echo "Variables:" @echo " REPO_DIR=$(REPO_DIR)" - @echo " ENV_FILE=$(ENV_FILE) (model/key/price for preflight + submission)" + @echo " ENV_FILE=$(ENV_FILE) (model/key for preflight + submission)" diff --git a/README.md b/README.md index d53a6b6..9d139cb 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ solve(a) == solve(reduce(a)) A mismatch is a bug. The AI finds these by constructing **counterexample certificates** — a JSON object naming the source instance `a` and the rule; the backend re-derives the bundle and round-trips it with `pred`, so the AI's claim is never trusted directly. The mismatch is reported with a derived label (`optimum_not_preserved`, `feasibility_not_preserved`, or `spurious_solution`); an optional `target_config` witness can additionally expose extraction bugs on a specific target solution (`unsound_extraction` / `suboptimal_extraction`). **Primary metric: bugs found** — the number of *distinct rules* with at least one confirmed bug, on a pinned library commit. One rule = one bug, no matter how many counterexamples (or violation types) target it. This count is fully verifiable and cannot be inflated by resubmitting certificates. -**Secondary metrics: bugs/Ktok and bugs/$** — token- and cost-efficiency. These have a self-reported denominator (tokens/cost), so they rank ties and serve as reference, not as the headline. +**Secondary metric: bugs/Ktok** — token efficiency. It has a self-reported denominator (tokens), so it ranks ties and serves as reference, not as the headline. Provenance is intentionally *not* scored: on a fixed commit, a `pred`-confirmed certificate is a bug regardless of who or what produced it. @@ -33,12 +33,11 @@ Implement the `AgentRunner` interface in `benchmark/runner.py`: from benchmark.runner import AgentRunner class MyRunner(AgentRunner): - def run(self, ctx, model: str, rule_name: str, per_rule_budget: float) -> dict: + def run(self, ctx, model: str, rule_name: str) -> dict: # Run the model, return a certificate if a bug is found return { "rule": rule_name, "result": "bug_found", # or "no_certificate" | "rejected" | "error:..." - "cost": 0.05, # USD spent "tokens_k": 12.3, # tokens used (thousands) "certificate": {...}, # required when result == "bug_found" } @@ -63,7 +62,7 @@ make test-unit make verify-calibration # Configure your run, then validate it with one tiny real call before the full batch -cp submission.env.example submission.env # fill in MODEL_NAME, key, PRICE_IN/OUT +cp submission.env.example submission.env # fill in MODEL_NAME + key make preflight # Run the benchmark via Docker → ./out/submission.json (this does NOT upload it) @@ -87,8 +86,5 @@ Key `make` targets: |--------|---------|-------------| | `bugs_found` | distinct rules with a confirmed bug | **Primary ranking** — fully verifiable, cannot be inflated | | `bugs/Ktok` | bugs ÷ tokens(K) | Tiebreak / efficiency reference — self-reported denominator | -| `bugs/$` | bugs ÷ USD spent | Tiebreak / cost-efficiency reference — self-reported denominator | -Rank by `bugs_found`. Among models that find the same number of bugs, `bugs/Ktok` breaks the tie (use `bugs/$` when optimizing for budget). The efficiency metrics divide by tokens/cost, which the submitter self-reports — treat them as informative, not authoritative. - -Models that don't publish pricing can still compete on `bugs/Ktok`. +Rank by `bugs_found`. Among models that find the same number of bugs, `bugs/Ktok` breaks the tie. The efficiency metric divides by tokens, which the submitter self-reports — treat it as informative, not authoritative. diff --git a/benchmark/backend_score.py b/benchmark/backend_score.py index 7c27dbe..4a7af15 100644 --- a/benchmark/backend_score.py +++ b/benchmark/backend_score.py @@ -64,9 +64,9 @@ def board_slug(scored: dict, stem: str) -> str: def board_entry(scored: dict, stem: str) -> dict: """The public per-submission leaderboard entry, tagged with its time + id.""" - sub_view = {"model": scored["model"], "budget_cap": scored.get("budget_cap", 20), - "submitted_by": scored.get("submitted_by")} - entry = leaderboard_entry(sub_view, scored) + # leaderboard_entry reads only ``submitted_by`` from its first arg; the scored file + # carries it when present, so it doubles as the submission view. + entry = leaderboard_entry(scored, scored) entry["timestamp"] = _submission_ts(scored, stem) entry["submission_id"] = _submission_id(stem) return entry @@ -220,11 +220,9 @@ def aggregate_leaderboard(results_dir: Path) -> list[dict]: # leaderboard — skip them here so an end-to-end test can't pollute production. if scored.get("test"): continue - # Reconstruct a minimal submission view for the entry (budget_cap lives in the - # scored file only if we put it there; default to RANKED 20 for scored results). - sub_view = {"model": scored["model"], "budget_cap": scored.get("budget_cap", 20), - "submitted_by": scored.get("submitted_by")} - entries.append(leaderboard_entry(sub_view, scored)) + # leaderboard_entry reads only ``submitted_by`` from its first arg, which the + # scored file carries when present — it doubles as the submission view. + entries.append(leaderboard_entry(scored, scored)) ranked = _dedup_best(entries) (results_dir / "leaderboard.json").write_text(json.dumps(ranked, indent=2), encoding="utf-8") return ranked diff --git a/benchmark/config.yaml b/benchmark/config.yaml index bf86502..a1c1ace 100644 --- a/benchmark/config.yaml +++ b/benchmark/config.yaml @@ -67,13 +67,12 @@ agent: Library repo: {{repo_dir}} (commit {{commit_hash}}) Rule source: {{rule_file}} - Use pred commands to test the round-trip correctness. You have a limited budget — be efficient and don't over-explore. + Use pred commands to test the round-trip correctness. You have a limited number of steps — be efficient and don't over-explore. If you find a bug, emit the CERTIFICATE_START...CERTIFICATE_END block. If no bug found, run: echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT step_limit: 35 - cost_limit: 0.5 environment: timeout: 120 diff --git a/benchmark/config_repo.yaml b/benchmark/config_repo.yaml index a5b0546..39110c8 100644 --- a/benchmark/config_repo.yaml +++ b/benchmark/config_repo.yaml @@ -1,8 +1,8 @@ agent: system_template: | You are an expert at finding bugs in problem reduction rules using the `pred` CLI. You - are given the WHOLE library and one budget; you decide which rules to probe and in what - order, and report every bug you find. + are given the WHOLE library and one session with a limited number of steps; you decide + which rules to probe and in what order, and report every bug you find. Each rule implements two functions: - reduce_to(): transforms source problem A into target problem B @@ -36,7 +36,7 @@ agent: How to work the whole repo: 1. `pred list --rules --json` to see every rule. Rule sources are under {{repo_dir}}/src/rules/.rs — read the ones you probe. - 2. Prioritize by SUSPICION — spend your budget where a bug is most likely first. + 2. Prioritize by SUSPICION — spend your steps where a bug is most likely first. Good signals: rules touching weights/costs, offsets or constant shifts, min↔max or feasibility flips, empty/degenerate inputs, or extraction code that drops information. Don't sweep alphabetically; triage. @@ -60,7 +60,7 @@ agent: CERTIFICATE_END THEN, immediately, append the SAME block to the durable log so the bug survives even if - the run is cut off (budget/crash) — one command, append only, never overwrite: + the run is cut off (step limit/crash) — one command, append only, never overwrite: cat >> {{certs_file}} <<'CERT' CERTIFICATE_START { ...the identical certificate JSON... } @@ -72,7 +72,7 @@ agent: Only `rule`, `source`, and the target type (from `bundle`) are required; `target_config` is optional. The verifier re-derives everything from `source` with pred and never trusts your claim — a wrong or non-minimal certificate is simply rejected, so report only what - you have actually reproduced. Keep going until you run out of budget or have covered the + you have actually reproduced. Keep going until you run out of steps or have covered the rules that look worth checking, then run: echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT instance_template: | @@ -86,12 +86,11 @@ agent: round-trip correctness of each with pred. Emit a CERTIFICATE_START...CERTIFICATE_END block for every bug — many are expected; do not stop after the first — AND append the same block to {{certs_file}} as you go, so no bug is lost if the run stops early. You - have a limited budget — be efficient and spend it on the most suspicious rules first. - When done, run: echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT + have a limited number of steps — be efficient and spend them on the most suspicious + rules first. When done, run: echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT step_limit: 300 - cost_limit: 20.0 environment: timeout: 120 diff --git a/benchmark/preflight.py b/benchmark/preflight.py index a06df1f..dbcc352 100644 --- a/benchmark/preflight.py +++ b/benchmark/preflight.py @@ -1,23 +1,23 @@ """ Preflight check for a real submission run. -Uses the SAME config you'd give the full batch (model, key, api_base, model_kwargs, price, +Uses the SAME config you'd give the full batch (model, key, api_base, model_kwargs, pred version) but does the minimum real work needed to prove the run won't error out: 1. verify the `pred` binary + version, 2. confirm the library's reduction rules are present, - 3. make ONE tiny real model call (~$0.0001) to validate credentials / endpoint / pricing. + 3. make ONE tiny real model call to validate credentials / endpoint. -Exits 0 only if every check passes — so you can launch the $20 batch with confidence +Exits 0 only if every check passes — so you can launch the full batch with confidence instead of discovering a typo'd key or wrong base URL 20 rules in. This is a user-facing -preflight (it spends a fraction of a cent of real money); the no-API wiring of the runner -itself is covered by the pytest suite (tests/test_run_submission.py), not here. +preflight (it makes one tiny real API call); the no-API wiring of the runner itself is +covered by the pytest suite (tests/test_run_submission.py), not here. """ from __future__ import annotations -from benchmark.cost import Price, usage_from_response from benchmark.env_setup import verify_pred_version from benchmark.run_mini import DEFAULT_MAX_TOKENS, _build_model, list_rules +from benchmark.usage import usage_from_response PROBE_PROMPT = "Reply with exactly: OK" @@ -32,7 +32,6 @@ def run_checks( api_key: str | None = None, model_kwargs: dict | None = None, max_tokens: int = DEFAULT_MAX_TOKENS, - price: Price | None = None, ) -> list[Check]: """Run the three preflight checks and return their results (never raises).""" results: list[Check] = [] @@ -55,21 +54,19 @@ def run_checks( results.append(("library rules", False, str(e))) # 3. one real model call through the exact batch model config (validates key, endpoint, - # model name, model_kwargs, and that pricing computes). + # model name, model_kwargs). try: - model = _build_model(model_name, api_base, max_tokens, price, + model = _build_model(model_name, api_base, max_tokens, model_kwargs=model_kwargs, api_key=api_key) msgs = [{"role": "user", "content": PROBE_PROMPT}] # Call the raw completion, NOT model.query(): query() also parses the reply into an # agent bash-action and raises FormatError on a trivial probe. We only need to prove - # the API round-trips and that pricing computes from the returned usage. + # the API round-trips and that usage reports back. prep = (model._prepare_messages_for_api(msgs) if hasattr(model, "_prepare_messages_for_api") else msgs) response = model._query(prep) u = usage_from_response(getattr(response, "usage", None)) detail = f"API reachable; {u.total_tokens} tokens this call" - if price is not None: - detail += f", ≈ ${price.cost(u):.6f}" results.append(("model call", True, detail)) except Exception as e: results.append(("model call", False, f"{type(e).__name__}: {e}")) diff --git a/benchmark/results.schema.json b/benchmark/results.schema.json index d0f6cc3..f3dc1c1 100644 --- a/benchmark/results.schema.json +++ b/benchmark/results.schema.json @@ -4,7 +4,7 @@ "title": "BenchmarkResults", "description": "Output of one benchmark run: model, library version, summary metrics, and per-rule details.", "type": "object", - "required": ["model", "library_commit", "bugs_found", "total_cost_usd", "total_tokens_k", "efficiency_bugs_per_ktok", "efficiency_bugs_per_dollar", "rules_tested", "results"], + "required": ["model", "library_commit", "bugs_found", "total_tokens_k", "efficiency_bugs_per_ktok", "rules_tested", "results"], "properties": { "model": { "type": "string", @@ -23,11 +23,6 @@ "type": "boolean", "description": "Carried through from the submission: scored + stored privately but excluded from the public leaderboard." }, - "total_cost_usd": { - "type": "number", - "minimum": 0, - "description": "Total spend in USD — re-metered by the backend as usage_totals × prices (falls back to the self-reported figure only for legacy submissions without those)." - }, "total_tokens_k": { "type": "number", "minimum": 0, @@ -35,7 +30,7 @@ }, "usage_totals": { "type": ["object", "null"], - "description": "Aggregate token usage in the four billed buckets (input/output/cache_read/cache_write) the cost was metered from. The reproducible primitive — lets any reader recompute cost under other prices.", + "description": "Aggregate token usage in the four provider-reported buckets (input/output/cache_read/cache_write) — the reproducible primitive behind total_tokens_k.", "properties": { "input": {"type": "integer", "minimum": 0}, "output": {"type": "integer", "minimum": 0}, @@ -43,25 +38,10 @@ "cache_write": {"type": "integer", "minimum": 0} } }, - "prices": { - "type": ["object", "null"], - "description": "Declared per-token price snapshot (USD per 1M tokens) the cost was metered from; dated by the submission's created_at. null for legacy/FAKE submissions.", - "properties": { - "input": {"type": "number", "minimum": 0}, - "output": {"type": "number", "minimum": 0}, - "cache_read": {"type": "number", "minimum": 0}, - "cache_write": {"type": "number", "minimum": 0} - } - }, "efficiency_bugs_per_ktok": { "type": "number", "minimum": 0, - "description": "Primary fair metric: bugs per 1000 tokens" - }, - "efficiency_bugs_per_dollar": { - "type": "number", - "minimum": 0, - "description": "Practical metric: bugs per dollar spent" + "description": "Efficiency metric: bugs per 1000 tokens" }, "rules_tested": { "type": "integer", @@ -73,7 +53,7 @@ "description": "Per-rule results", "items": { "type": "object", - "required": ["rule", "result", "cost", "tokens_k"], + "required": ["rule", "result", "tokens_k"], "properties": { "rule": { "type": "string", @@ -84,10 +64,6 @@ "enum": ["bug_found", "no_certificate", "rejected", "no_bug", "error"], "description": "Outcome for this rule" }, - "cost": { - "type": "number", - "minimum": 0 - }, "tokens_k": { "type": "number", "minimum": 0 diff --git a/benchmark/run_mini.py b/benchmark/run_mini.py index 48d53ef..8d4e313 100644 --- a/benchmark/run_mini.py +++ b/benchmark/run_mini.py @@ -12,19 +12,16 @@ import yaml -from benchmark.cost import Price, extract_usage, usage_from_response from benchmark.env_context import EnvContext from benchmark.env_setup import setup_env +from benchmark.usage import extract_usage, usage_as_dict from benchmark.verify import count_bugs, verify CONFIG_FILE = Path(__file__).parent / "config.yaml" REPO_CONFIG_FILE = Path(__file__).parent / "config_repo.yaml" SKIP_RULES = {"mod", "traits", "graph_helpers", "analysis", "cost", "registry", "graph"} -# Rough average cost per 1K tokens (used as fallback when model doesn't report usage) -AVG_COST_PER_KTOK = 6.0 -# Per-call output-token ceiling — bounds the single call that may cross the budget line to -# well under $1 even at premium prices. The submitter can override via --max-tokens. +# Per-call output-token ceiling. The submitter can override via --max-tokens. DEFAULT_MAX_TOKENS = 8192 @@ -94,15 +91,12 @@ def save_trajectory(messages: list, path: Path) -> None: f.write(json.dumps({"role": msg.get("role", ""), "content": _message_text(msg)}) + "\n") -def _session_cost(agent, price): - """Authoritative session spend from the trajectory. ``agent.cost`` is already our - token×price figure (see _build_model); cross-check against a fresh recompute and take - the max — never under-count for the budget guard. Returns (cost, tokens_k, usage).""" +def _session_usage(agent): + """Session token usage summed from the trajectory. Returns (tokens_k, usage).""" usage = extract_usage(agent.messages) - cost = max(agent.cost, price.cost(usage) if price is not None else 0.0) total_tokens = usage.total_tokens or extract_total_tokens(agent.messages) - tokens_k = round(total_tokens / 1000, 2) if total_tokens else round(cost / AVG_COST_PER_KTOK * 1000, 2) - return cost, tokens_k, usage + tokens_k = round(total_tokens / 1000, 2) + return tokens_k, usage def _trajectory(agent) -> list[dict]: @@ -114,13 +108,11 @@ def _trajectory(agent) -> list[dict]: return [{"role": m.get("role", ""), "content": _message_text(m)} for m in agent.messages] -def _build_model(model_name: str, api_base: str | None, max_tokens: int, price: Price | None, +def _build_model(model_name: str, api_base: str | None, max_tokens: int, model_kwargs: dict | None = None, api_key: str | None = None, observation_template: str | None = None, format_error_template: str | None = None): - """A LitellmModel whose cost is OUR token×price figure, so mini-swe-agent's own - per-step ``cost_limit`` enforces the per-rule budget with the authoritative number and - never raises on an unpriceable model. + """A LitellmModel configured for this benchmark's runs. Everything that configures the API call flows through ``model_kwargs`` (forwarded to litellm.completion) — these are NOT top-level config fields in mini-swe-agent v2 and @@ -136,18 +128,9 @@ def _build_model(model_name: str, api_base: str | None, max_tokens: int, price: ``model:`` section carries the truncating templates; the caller threads them through.""" from minisweagent.models.litellm_model import LitellmModel - class PricedLitellmModel(LitellmModel): - def _calculate_cost(self, response): - if price is not None: - try: - return {"cost": price.cost(usage_from_response(getattr(response, "usage", None)))} - except Exception: - pass - return super()._calculate_cost(response) - mk: dict = dict(model_kwargs or {}) # arbitrary passthrough for non-standard providers if max_tokens: - mk["max_tokens"] = max_tokens # per-call ceiling → bounds the budget-crossing call + mk["max_tokens"] = max_tokens # per-call output ceiling if api_base: mk["api_base"] = api_base if api_key: @@ -158,17 +141,33 @@ def _calculate_cost(self, response): cfg["observation_template"] = observation_template if format_error_template is not None: cfg["format_error_template"] = format_error_template - return PricedLitellmModel(model_name=model_name, model_kwargs=mk, **cfg) + return LitellmModel(model_name=model_name, model_kwargs=mk, **cfg) + + +def _load_agent_config(config_path: str | Path | None, default_file: Path, + strategy: str | None) -> tuple[dict, dict, str]: + """Resolve the prompt config + strategy hints shared by both session runners. + + Returns (agent_cfg, model_cfg, strategy). ``cost_limit`` is force-disabled HERE — one + place — so mini-swe-agent's default ($3) can never bound a run, including under a + user-supplied AGENT_CONFIG; the config's step_limit is the only run bound.""" + cfg_file = Path(config_path) if config_path else default_file + config = yaml.safe_load(cfg_file.read_text(encoding="utf-8")) + if strategy is None: + strat_file = os.environ.get("AGENT_STRATEGY_FILE") + strategy = Path(strat_file).read_text(encoding="utf-8") if strat_file else "" + agent_cfg = config.get("agent", {}) + agent_cfg["cost_limit"] = 0 + model_cfg = config.get("model", {}) or {} # observation/format templates live here + return agent_cfg, model_cfg, strategy def run_one( model_name: str, ctx: EnvContext, rule_name: str, - cost_limit: float, api_base: str | None = None, trajectory_dir: Path | None = None, - price: Price | None = None, max_tokens: int = DEFAULT_MAX_TOKENS, config_path: str | Path | None = None, strategy: str | None = None, @@ -187,17 +186,10 @@ def run_one( safe_name = rule_name.replace("-", "_") rule_file = ctx.repo_path / "src" / "rules" / f"{rule_name}.rs" - cfg_file = Path(config_path) if config_path else CONFIG_FILE - config = yaml.safe_load(cfg_file.read_text(encoding="utf-8")) - if strategy is None: - strat_file = os.environ.get("AGENT_STRATEGY_FILE") - strategy = Path(strat_file).read_text(encoding="utf-8") if strat_file else "" - agent_cfg = config.get("agent", {}) - agent_cfg["cost_limit"] = cost_limit - model_cfg = config.get("model", {}) or {} # observation/format templates live here + agent_cfg, model_cfg, strategy = _load_agent_config(config_path, CONFIG_FILE, strategy) agent = DefaultAgent( - _build_model(model_name, api_base, max_tokens, price, + _build_model(model_name, api_base, max_tokens, model_kwargs=model_kwargs, api_key=api_key, observation_template=model_cfg.get("observation_template"), format_error_template=model_cfg.get("format_error_template")), @@ -210,17 +202,14 @@ def run_one( "safe_name": safe_name, "rule_file": str(rule_file), "commit_hash": ctx.commit_hash[:7], - "cost_limit": cost_limit, - "strategy": strategy or "", + "strategy": strategy, } cert = None try: agent.run(task=rule_name) - cost, tokens_k, usage = _session_cost(agent, price) - usage_row = {"input": usage.input_tokens, "output": usage.output_tokens, - "cache_read": usage.cache_read_tokens, "cache_write": usage.cache_write_tokens, - "accounted_cost_usd": round(agent.cost, 6)} + tokens_k, usage = _session_usage(agent) + usage_row = usage_as_dict(usage) trajectory = _trajectory(agent) cert = parse_certificate(agent.messages) if trajectory_dir is not None: @@ -230,7 +219,6 @@ def run_one( return { "rule": rule_name, "result": f"error: {e}", - "cost": getattr(agent, "cost", 0.0), "tokens_k": 0, } @@ -238,7 +226,6 @@ def run_one( return { "rule": rule_name, "result": "no_certificate", - "cost": cost, "tokens_k": tokens_k, "steps": agent.n_calls, "usage": usage_row, @@ -254,7 +241,6 @@ def run_one( return { "rule": rule_name, "result": "rejected", - "cost": cost, "tokens_k": tokens_k, "steps": agent.n_calls, "reject_reason": verdict.reason, @@ -268,7 +254,6 @@ def run_one( return { "rule": rule_name, "result": "bug_found", - "cost": cost, "tokens_k": tokens_k, "steps": agent.n_calls, "certificate": cert, @@ -281,24 +266,21 @@ def run_one( def run_repo_session( model_name: str, ctx: EnvContext, - cost_limit: float, *, api_base: str | None = None, trajectory_dir: Path | None = None, certs_path: Path | None = None, - price: Price | None = None, max_tokens: int = DEFAULT_MAX_TOKENS, - step_limit: int | None = None, config_path: str | Path | None = None, strategy: str | None = None, model_kwargs: dict | None = None, api_key: str | None = None, ) -> dict: - """One WHOLE-REPO bug-hunting session: the agent gets the entire library + pred and the - full budget as its ``cost_limit``, chooses which rules to probe, and emits a certificate - per bug. Returns ``{"rows": [...], "cost": float, "tokens_k": float}`` — one result row - per distinct emitted certificate, each re-verified with pred and carrying the shared - session trajectory for provenance. Contrast with ``run_one`` (one isolated rule/session). + """One WHOLE-REPO bug-hunting session: the agent gets the entire library + pred, chooses + which rules to probe within the config's ``step_limit``, and emits a certificate per bug. + Returns ``{"rows": [...], "tokens_k": float, ...}`` — one result row per distinct emitted + certificate, each re-verified with pred and carrying the shared session trajectory for + provenance. Contrast with ``run_one`` (one isolated rule/session). Durability: the agent is prompted to append each certificate to ``certs_path`` the moment it finds it (the {{certs_file}} slot), so bugs survive an early stop; certificates are @@ -309,16 +291,7 @@ def run_repo_session( from minisweagent.agents.default import DefaultAgent from minisweagent.environments.local import LocalEnvironment - cfg_file = Path(config_path) if config_path else REPO_CONFIG_FILE - config = yaml.safe_load(cfg_file.read_text(encoding="utf-8")) - if strategy is None: - strat_file = os.environ.get("AGENT_STRATEGY_FILE") - strategy = Path(strat_file).read_text(encoding="utf-8") if strat_file else "" - agent_cfg = config.get("agent", {}) - agent_cfg["cost_limit"] = cost_limit - if step_limit is not None: - agent_cfg["step_limit"] = step_limit - model_cfg = config.get("model", {}) or {} # observation/format templates live here + agent_cfg, model_cfg, strategy = _load_agent_config(config_path, REPO_CONFIG_FILE, strategy) safe_model = model_name.replace("/", "_").replace(":", "_") # Per-step trajectory persistence (mini-swe-agent saves config.output_path after every @@ -336,7 +309,7 @@ def run_repo_session( certs_path.write_text("", encoding="utf-8") agent = DefaultAgent( - _build_model(model_name, api_base, max_tokens, price, + _build_model(model_name, api_base, max_tokens, model_kwargs=model_kwargs, api_key=api_key, observation_template=model_cfg.get("observation_template"), format_error_template=model_cfg.get("format_error_template")), @@ -346,7 +319,6 @@ def run_repo_session( agent.extra_template_vars = { "repo_dir": str(ctx.repo_path), "commit_hash": ctx.commit_hash[:7], - "cost_limit": cost_limit, "strategy": strategy, "certs_file": str(certs_path) if certs_path is not None else "/tmp/certs.txt", } @@ -361,7 +333,7 @@ def run_repo_session( except Exception as e: # noqa: BLE001 — any failure still salvages partial results run_error = f"{type(e).__name__}: {e}" - cost, tokens_k, usage = _session_cost(agent, price) + tokens_k, usage = _session_usage(agent) trajectory = _trajectory(agent) if trajectory_dir is not None: save_trajectory(agent.messages, Path(trajectory_dir) / f"{safe_model}_whole-repo.jsonl") @@ -376,9 +348,9 @@ def run_repo_session( # The whole session is ONE trajectory shared by every bug — return it once for the # envelope (build_submission) instead of copying it onto each row. ``usage`` is the # session-level 4-bucket token total (per-rule rows carry their own; whole-repo rows - # don't), so the envelope can re-price it — see build_submission. ``error`` is set when - # the session died on a fatal error (the partial results are still valid). - return {"rows": rows, "cost": cost, "tokens_k": tokens_k, "trajectory": trajectory, + # don't). ``error`` is set when the session died on a fatal error (the partial results + # are still valid). + return {"rows": rows, "tokens_k": tokens_k, "trajectory": trajectory, "usage": usage, "error": run_error} @@ -392,8 +364,7 @@ def _rows_from_certificates(certs: list[dict]) -> list[dict]: row = { "rule": cert.get("rule"), "result": "bug_found" if verdict.accepted else "rejected", - "cost": 0.0, # session cost/tokens live on the submission envelope, not per row - "tokens_k": 0.0, + "tokens_k": 0.0, # session tokens live on the submission envelope, not per row "certificate": cert, } if verdict.accepted: @@ -408,8 +379,6 @@ def main() -> None: parser = argparse.ArgumentParser(description="pred-based bug-finding benchmark") parser.add_argument("--model", default="anthropic/claude-sonnet-4-6", help="LiteLLM model name") parser.add_argument("--api-base", default=None, help="Custom API base URL") - parser.add_argument("--budget", type=float, default=20.0, help="Total USD budget") - parser.add_argument("--per-rule", type=float, default=0.5, help="Per-rule cost limit ($)") parser.add_argument("--rules", nargs="*", help="Specific rule names (default: all)") parser.add_argument("--output", default="results/results_mini.json") parser.add_argument("--trajectory-dir", default=None, help="Directory to save per-rule JSONL trajectories") @@ -419,24 +388,18 @@ def main() -> None: ctx = setup_env(args.repo_dir) rules = args.rules if args.rules else list_rules(str(ctx.repo_path)) - results, total_cost, total_tokens_k = [], 0.0, 0.0 + results, total_tokens_k = [], 0.0 for rule_name in rules: - remaining = args.budget - total_cost - if remaining <= 0: - print("Budget exhausted.") - break - limit = min(args.per_rule, remaining) - print(f" {rule_name} (limit ${limit:.2f})...", end=" ", flush=True) - - r = run_one(args.model, ctx, rule_name, limit, api_base=args.api_base, + print(f" {rule_name}...", end=" ", flush=True) + + r = run_one(args.model, ctx, rule_name, api_base=args.api_base, trajectory_dir=Path(args.trajectory_dir) if args.trajectory_dir else None) results.append(r) - total_cost += r.get("cost", 0) total_tokens_k += r.get("tokens_k", 0) status = "BUG FOUND" if r["result"] == "bug_found" else r["result"] - print(f"{status} (${r.get('cost', 0):.4f}, {r.get('tokens_k', 0):.1f}K tok)") + print(f"{status} ({r.get('tokens_k', 0):.1f}K tok)") bugs_found = count_bugs(results) # one rule = one bug efficiency_per_ktok = round(bugs_found / total_tokens_k, 4) if total_tokens_k else 0 @@ -444,10 +407,8 @@ def main() -> None: "model": args.model, "library_commit": ctx.commit_hash, "bugs_found": bugs_found, - "total_cost_usd": round(total_cost, 6), "total_tokens_k": round(total_tokens_k, 2), "efficiency_bugs_per_ktok": efficiency_per_ktok, - "efficiency_bugs_per_dollar": round(bugs_found / total_cost, 4) if total_cost else 0, "rules_tested": len(results), "results": results, } @@ -455,7 +416,7 @@ def main() -> None: out = Path(args.output) out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(summary, indent=2), encoding="utf-8") - print(f"\n{bugs_found} bugs | ${total_cost:.4f} | {efficiency_per_ktok:.4f} bugs/Ktok") + print(f"\n{bugs_found} bugs | {total_tokens_k:.1f}K tok | {efficiency_per_ktok:.4f} bugs/Ktok") print(f"Results → {args.output}") diff --git a/benchmark/run_submission.py b/benchmark/run_submission.py index 6e1e6a1..8856d49 100644 --- a/benchmark/run_submission.py +++ b/benchmark/run_submission.py @@ -2,14 +2,14 @@ """ Submission runner — the dockerized entry point. -Given a model (LiteLLM name), a provider API key (via the standard env var, e.g. -ANTHROPIC_API_KEY / OPENAI_API_KEY), and a fixed USD budget (default $20), run the -bug-hunting agent across the reduction rules and emit a single, rankable -``submission.json`` recording the bugs (rule counterexamples) it claims. - -Budget is enforced by the shared Scheduler (per-rule LiteLLM ``cost_limit`` + a hard -total cap). Self-reported bug counts are advisory only — the backend re-verifies every -certificate with ``pred`` before anything reaches the leaderboard (see +Given a model (LiteLLM name) and a provider API key (via the standard env var, e.g. +ANTHROPIC_API_KEY / OPENAI_API_KEY), run the bug-hunting agent across the reduction rules +and emit a single, rankable ``submission.json`` recording the bugs (rule counterexamples) +it claims. + +Runs are bounded by the agent step limit in the config (per-rule: 35 steps/rule; +whole-repo: 300 steps/session). Self-reported bug counts are advisory only — the backend +re-verifies every certificate with ``pred`` before anything reaches the leaderboard (see benchmark/verify_submission.py). Docker usage (all config — any provider — in submission.env, key never baked in): @@ -19,9 +19,8 @@ # → ./out/submission.json (template: submission.env.example) Env vars (CLI flags override): MODEL_NAME, the matching API key (generic API_KEY or a -provider var like OPENAI_API_KEY/ANTHROPIC_API_KEY), PRICE_IN/PRICE_OUT (required), -API_BASE, MODEL_KWARGS, BUDGET_USD, PER_RULE_BUDGET, MAX_RULES, AGENT_CONFIG, -AGENT_STRATEGY_FILE; FAKE (1 → no API/pred, used by tests). +provider var like OPENAI_API_KEY/ANTHROPIC_API_KEY), API_BASE, MODEL_KWARGS, MAX_RULES, +AGENT_CONFIG, AGENT_STRATEGY_FILE; FAKE (1 → no API/pred, used by tests). """ import argparse import json @@ -30,57 +29,48 @@ import tempfile from pathlib import Path -from benchmark.cost import Usage, price_as_dict, usage_as_dict, usage_from_dict from benchmark.env_context import EnvContext from benchmark.env_setup import find_pred_binary, pinned_commit, verify_pred_version from benchmark.run_mini import list_rules from benchmark.runner import FakeRunner, MiniSweRunner from benchmark.scheduler import Scheduler +from benchmark.usage import Usage, usage_as_dict, usage_from_dict from benchmark.verify import count_bugs -SCHEMA_VERSION = "1.0" -RUNNER_VERSION = "0.6.0" +SCHEMA_VERSION = "2.0" +RUNNER_VERSION = "0.7.0" def build_submission( model: str, rows: list[dict], *, - budget_cap: float, library_commit: str, runner_version: str = RUNNER_VERSION, created_at: str | None = None, submitted_by: str | None = None, - total_cost_usd: float | None = None, total_tokens_k: float | None = None, trajectory: list[dict] | None = None, pred_version: str = "", - price=None, usage_totals=None, agent_mode: str | None = None, run_error: str | None = None, ) -> dict: """Assemble the submission envelope from the runner's result rows. - ``rules_tested`` is the number of DISTINCT rules with a result (skipped_budget rows - don't count as "reached"). For per-rule that is the rules attempted; for whole-repo it - is the distinct rules the agent emitted a certificate for — a floor, since rules the - agent probed but found clean aren't represented as rows. ``bugs_found`` is distinct - rules with a confirmed bug. - - Cost is metered from tokens, not hand-reported: when ``price`` (the submitter's declared - per-token rate) is given, ``total_cost_usd`` is DERIVED as ``price × token_usage`` — the - same authoritative figure the backend recomputes (benchmark/verify_submission.py), so the - self-reported total is never trusted. The 4-bucket token total is either passed in - (``usage_totals`` — whole-repo, one session) or summed from each row's ``usage`` block - (per-rule). It rides on the envelope as ``usage_totals`` (the reproducible primitive) next - to the ``prices`` snapshot, so any reader can recompute spend under other prices. Without a - ``price`` (e.g. FAKE mode) it falls back to explicit session totals or the row-sum. + ``rules_tested`` is the number of DISTINCT rules with a result. For per-rule that is + the rules attempted; for whole-repo it is the distinct rules the agent emitted a + certificate for — a floor, since rules the agent probed but found clean aren't + represented as rows. ``bugs_found`` is distinct rules with a confirmed bug. + + The 4-bucket token total is either passed in (``usage_totals`` — whole-repo, one + session) or summed from each row's ``usage`` block (per-rule). It rides on the envelope + as ``usage_totals`` (the reproducible primitive); ``total_tokens_k`` is derived from it + when it carries counts, else the explicit session total / row-sum is used (FAKE mode). """ - attempted = [r for r in rows if r.get("result") != "skipped_budget"] bugs = count_bugs(rows) - # 4-bucket token usage: the reproducible primitive the backend re-prices. + # 4-bucket token usage: the reproducible primitive. if usage_totals is None: usage_totals = Usage() for r in rows: @@ -88,28 +78,21 @@ def build_submission( elif isinstance(usage_totals, dict): usage_totals = usage_from_dict(usage_totals) - if price is not None: - cost = price.cost(usage_totals) + if usage_totals.total_tokens: tokens_k = usage_totals.total_tokens / 1000 else: - cost = total_cost_usd if total_cost_usd is not None else sum(r.get("cost", 0.0) for r in rows) tokens_k = total_tokens_k if total_tokens_k is not None else sum(r.get("tokens_k", 0.0) for r in rows) envelope = { "schema_version": SCHEMA_VERSION, "model": model, "library_commit": library_commit, - "budget_cap": budget_cap, "bugs_found": bugs, - "total_cost_usd": round(cost, 6), "total_tokens_k": round(tokens_k, 2), "efficiency_bugs_per_ktok": round(bugs / tokens_k, 4) if tokens_k else 0, - "efficiency_bugs_per_dollar": round(bugs / cost, 4) if cost else 0, - "rules_tested": len({r.get("rule") for r in attempted}), - # Aggregate token totals + the declared price snapshot — the backend re-meters cost - # from these (zero-trust); dated by created_at, recomputable under any price table. + "rules_tested": len({r.get("rule") for r in rows}), + # Aggregate token totals — reproducible primitive behind total_tokens_k. "usage_totals": usage_as_dict(usage_totals), - "prices": price_as_dict(price) if price is not None else None, "results": rows, # Version/provenance stamp so a produced file self-identifies its run (no more # "everything overwrites one submission.json"). agent_mode + created_at + the @@ -135,20 +118,15 @@ def run( model: str, repo_dir: str, *, - budget: float = 20.0, - per_rule_budget: float = 0.5, fake: bool = False, fake_result: str = "no_certificate", - fake_cost: float = 0.01, max_rules: int | None = None, library_commit: str | None = None, api_base: str | None = None, output: Path | None = None, created_at: str | None = None, submitted_by: str | None = None, - price=None, max_tokens: int | None = None, - safety_margin: float = 1.0, config_path: str | Path | None = None, strategy: str | None = None, model_kwargs: dict | None = None, @@ -156,7 +134,7 @@ def run( mode: str = "per-rule", trajectory_dir: str | Path | None = None, ) -> dict: - """Run the full budgeted session for one model and return the submission dict. + """Run the full session for one model and return the submission dict. ``trajectory_dir`` is where the whole-repo agent's trajectory + the durable incremental cert log are persisted (default: the output file's directory). Beside the stable @@ -164,12 +142,9 @@ def run( so successive runs don't all overwrite one submission.json. ``mode`` selects the runner: ``per-rule`` (default) schedules one isolated agent session - per rule under a shared budget; ``whole-repo`` runs ONE session over the whole library — - the agent enumerates and triages the rules itself and emits a certificate per bug. - - ``price`` is the submitter's per-token rate (benchmark.cost.Price); with it, spend is - recomputed from token usage so the budget is a hard cap. ``safety_margin`` is held back - from the budget so the boundary-crossing call still lands under it. + per rule; ``whole-repo`` runs ONE session over the whole library — the agent enumerates + and triages the rules itself and emits a certificate per bug. Each session is bounded by + the config's agent step limit. In ``fake`` mode no API key or pred binary is needed (FakeRunner) — used by tests and for smoke-running the container wiring; ``fake`` always uses the per-rule path. @@ -183,7 +158,7 @@ def run( from types import SimpleNamespace ctx = SimpleNamespace(repo_path=repo, pred_binary=Path("pred"), commit_hash=commit, pred_version="") - runner = FakeRunner(cost_per_rule=fake_cost, result=fake_result) + runner = FakeRunner(result=fake_result) else: pred_binary = find_pred_binary() pred_ver = verify_pred_version(pred_binary) # fail fast if pred != pinned version @@ -191,7 +166,7 @@ def run( pred_version=pred_ver) # Only the per-rule path uses this, but the constructor just stores kwargs (no I/O), # so building it unconditionally keeps `runner` assigned in exactly one place. - runner = MiniSweRunner(api_base=api_base, price=price, max_tokens=max_tokens, + runner = MiniSweRunner(api_base=api_base, max_tokens=max_tokens, config_path=config_path, strategy=strategy, model_kwargs=model_kwargs, api_key=api_key) @@ -206,16 +181,16 @@ def run( from benchmark.run_mini import run_repo_session out_dir = traj_dir session = run_repo_session( - model, ctx, cost_limit=max(budget - safety_margin, 0.0), - api_base=api_base, price=price, max_tokens=max_tokens, + model, ctx, + api_base=api_base, max_tokens=max_tokens, trajectory_dir=out_dir, certs_path=(out_dir / "certs.txt") if out_dir is not None else None, config_path=config_path, strategy=strategy, model_kwargs=model_kwargs, api_key=api_key, ) - rows, total_cost, total_tokens = session["rows"], session["cost"], session["tokens_k"] + rows, total_tokens = session["rows"], session["tokens_k"] session_trajectory = session["trajectory"] - session_usage = session.get("usage") # session-level 4-bucket total to re-price + session_usage = session.get("usage") # session-level 4-bucket total run_error = session.get("error") # set if the session died on a fatal error if run_error: print(f"WARNING: session ended on error — salvaged partial results: {run_error}") @@ -229,21 +204,16 @@ def run( runner=runner, models=[model], rules=rules, - total_budget=budget, - per_rule_budget=per_rule_budget, results_dir=Path(tmp) / "results", checkpoint_path=Path(tmp) / "checkpoint.json", ctx=ctx, resume=False, parallelism=1, - safety_margin=safety_margin, ) completed = scheduler.run_all() - spent = scheduler._spent.get(model) - # per-rule totals: cost is the scheduler's tracked spend; tokens sum from the rows. # Per-rule rows carry their own trajectories AND their own 4-bucket ``usage``, so the # envelope aggregates usage from the rows (session_usage=None). - rows, total_cost, total_tokens = completed[model], spent, None + rows, total_tokens = completed[model], None session_trajectory = None session_usage = None # per-rule already isolates each rule's failure into an "error:" row (run_one), so the @@ -251,11 +221,11 @@ def run( run_error = None sub = build_submission( - model, rows, budget_cap=budget, library_commit=commit, + model, rows, library_commit=commit, created_at=created_at, submitted_by=submitted_by, - total_cost_usd=total_cost, total_tokens_k=total_tokens, trajectory=session_trajectory, + total_tokens_k=total_tokens, trajectory=session_trajectory, pred_version=getattr(ctx, "pred_version", ""), - price=price, usage_totals=session_usage, + usage_totals=session_usage, agent_mode=mode, run_error=run_error, ) @@ -288,18 +258,9 @@ def _env(name: str, default: str | None = None) -> str | None: return os.environ.get(name, default) -def _float_env(name: str) -> float | None: - v = os.environ.get(name) - return float(v) if v else None - - def main() -> None: - parser = argparse.ArgumentParser(description="Budgeted bug-finding runner → submission.json") + parser = argparse.ArgumentParser(description="Bug-finding runner → submission.json") parser.add_argument("--model", default=_env("MODEL_NAME"), help="LiteLLM model name (env MODEL_NAME)") - parser.add_argument("--budget", type=float, default=float(_env("BUDGET_USD", "20") or 20), - help="Total USD budget (env BUDGET_USD, default 20)") - parser.add_argument("--per-rule", type=float, default=float(_env("PER_RULE_BUDGET", "0.5") or 0.5), - help="Per-rule cost cap (env PER_RULE_BUDGET)") parser.add_argument("--repo-dir", default=_env("REPO_DIR", "/app/pr-src"), help="problem-reductions source tree (env REPO_DIR)") parser.add_argument("--output", default=_env("OUTPUT", "/out/submission.json"), @@ -319,22 +280,8 @@ def main() -> None: parser.add_argument("--max-rules", type=lambda v: int(v) if v else None, default=_env("MAX_RULES"), help="Cap rules attempted (smoke runs)") parser.add_argument("--submitted-by", default=_env("SUBMITTED_BY")) - # Submitter-supplied price (USD / 1M tokens). You pay the bill, so you set the rate; we - # recompute spend from token usage instead of trusting the gateway. Omit to use a built-in - # default for known models (see benchmark/cost.py), or the gateway figure if unknown. - parser.add_argument("--price-in", type=float, default=_float_env("PRICE_IN"), - help="USD per 1M input tokens (env PRICE_IN)") - parser.add_argument("--price-out", type=float, default=_float_env("PRICE_OUT"), - help="USD per 1M output tokens (env PRICE_OUT)") - parser.add_argument("--price-cache-read", type=float, default=_float_env("PRICE_CACHE_READ"), - help="USD per 1M cache-read tokens (env PRICE_CACHE_READ)") - parser.add_argument("--price-cache-write", type=float, default=_float_env("PRICE_CACHE_WRITE"), - help="USD per 1M cache-write tokens (env PRICE_CACHE_WRITE)") parser.add_argument("--max-tokens", type=lambda v: int(v) if v else None, default=_env("MAX_TOKENS"), help="Per-call output-token ceiling") - parser.add_argument("--safety-margin", type=float, - default=float(_env("SAFETY_MARGIN", "1.0") or 1.0), - help="USD held back from the budget as overshoot headroom (default 1)") parser.add_argument("--expected-pred-version", default=_env("EXPECTED_PRED_VERSION"), help="Require this pred version (default: pinned; empty string disables)") parser.add_argument("--expected-pred-commit", default=_env("EXPECTED_PRED_COMMIT"), @@ -381,49 +328,29 @@ def main() -> None: if not isinstance(model_kwargs, dict): parser.error("--model-kwargs must be a JSON object") - # Price is always submitter-supplied — there is no built-in table (a stale default would - # silently mis-meter the $20 cap). A real run REQUIRES --price-in and --price-out. - from benchmark.cost import Price - price = None - if args.price_in is not None and args.price_out is not None: - price = Price(args.price_in, args.price_out, - args.price_cache_read or 0.0, args.price_cache_write or 0.0) - elif args.price_in is not None or args.price_out is not None: - parser.error("--price-in and --price-out must be given together") - if price is None and not args.fake: - parser.error("--price-in and --price-out (env PRICE_IN/PRICE_OUT) are required: " - "spend is metered as token_usage × your price, so you must declare it " - "(USD / 1M tokens). There is no built-in price table.") - if args.preflight: from benchmark.preflight import format_report, run_checks from benchmark.run_mini import DEFAULT_MAX_TOKENS print(f"Preflight for {args.model} (one tiny real call + pred/rules checks)...") results = run_checks(args.model, repo_dir=args.repo_dir, api_base=args.api_base, api_key=args.api_key, model_kwargs=model_kwargs, - max_tokens=args.max_tokens or DEFAULT_MAX_TOKENS, price=price) + max_tokens=args.max_tokens or DEFAULT_MAX_TOKENS) raise SystemExit(0 if format_report(results) else 1) import datetime created_at = datetime.datetime.now(datetime.timezone.utc).isoformat() - detail = f"per-rule ${args.per_rule:.2f}" if args.mode == "per-rule" else "whole-repo" - print(f"Running {args.model} at ${args.budget:.0f} budget " - f"({detail}){' [FAKE]' if args.fake else ''}...") + print(f"Running {args.model} ({args.mode}){' [FAKE]' if args.fake else ''}...") sub = run( args.model, args.repo_dir, - budget=args.budget, - per_rule_budget=args.per_rule, fake=args.fake, max_rules=args.max_rules, api_base=args.api_base, output=Path(args.output), created_at=created_at, submitted_by=args.submitted_by, - price=price, max_tokens=args.max_tokens, - safety_margin=args.safety_margin, config_path=args.config, strategy=strategy, model_kwargs=model_kwargs, @@ -431,7 +358,7 @@ def main() -> None: mode=args.mode, trajectory_dir=args.trajectory_dir, ) - print(f"\n{sub['bugs_found']} claimed bugs | ${sub['total_cost_usd']:.4f} | " + print(f"\n{sub['bugs_found']} claimed bugs | {sub['total_tokens_k']:.1f}K tok | " f"{sub['rules_tested']} rules attempted") archive = _versioned_name(Path(args.output), args.model, created_at) print(f"Submission → {args.output} (archive: {archive})") diff --git a/benchmark/runner.py b/benchmark/runner.py index a41e675..a70525e 100644 --- a/benchmark/runner.py +++ b/benchmark/runner.py @@ -9,13 +9,12 @@ class AgentRunner(ABC): @abstractmethod - def run(self, ctx, model: str, rule_name: str, per_rule_budget: float) -> dict: + def run(self, ctx, model: str, rule_name: str) -> dict: """Run one bug-hunting session. Returns a dict with at minimum: rule str — rule name - result str — "bug_found" | "no_certificate" | "rejected" | "error:..." | "skipped_budget" - cost float — USD spent + result str — "bug_found" | "no_certificate" | "rejected" | "error:..." tokens_k float — tokens used (in thousands) """ @@ -23,43 +22,35 @@ def run(self, ctx, model: str, rule_name: str, per_rule_budget: float) -> dict: class FakeRunner(AgentRunner): """Returns a canned result — no API calls, no pred calls. For testing only.""" - def __init__(self, cost_per_rule: float = 0.01, result: str = "no_certificate"): - self.cost_per_rule = cost_per_rule + def __init__(self, result: str = "no_certificate"): self._result = result self.call_log: list[tuple[str, str]] = [] # (model, rule) for each real call - def run(self, ctx, model: str, rule_name: str, per_rule_budget: float) -> dict: + def run(self, ctx, model: str, rule_name: str) -> dict: self.call_log.append((model, rule_name)) return { "rule": rule_name, "result": self._result, - "cost": self.cost_per_rule, "tokens_k": 0.5, } class MiniSweRunner(AgentRunner): - """Wraps benchmark.run_mini.run_one() — the default real agent. + """Wraps benchmark.run_mini.run_one() — the default real agent.""" - ``price`` (submitter-supplied per-token rate) makes cost accounting authoritative; - when None, run_one falls back to the gateway's figure plus the step/token backstops. - """ - - def __init__(self, api_base: str | None = None, price=None, max_tokens: int | None = None, + def __init__(self, api_base: str | None = None, max_tokens: int | None = None, config_path=None, strategy: str | None = None, model_kwargs: dict | None = None, api_key: str | None = None): self.api_base = api_base - self.price = price self.max_tokens = max_tokens self.config_path = config_path # hand-editable prompt override (None → bundled config) self.strategy = strategy # extra hints injected into the prompt's {{strategy}} slot self.model_kwargs = model_kwargs # arbitrary litellm passthrough for non-standard providers - self.api_key = api_key # generic key (no provider-specific env var needed) + self.api_key = api_key # generic key (no provider-specific env var name needed) - def run(self, ctx, model: str, rule_name: str, per_rule_budget: float) -> dict: + def run(self, ctx, model: str, rule_name: str) -> dict: from benchmark.run_mini import DEFAULT_MAX_TOKENS, run_one # lazy — keep scheduler mini-swe-free - return run_one(model, ctx, rule_name, per_rule_budget, api_base=self.api_base, - price=self.price, + return run_one(model, ctx, rule_name, api_base=self.api_base, max_tokens=self.max_tokens if self.max_tokens is not None else DEFAULT_MAX_TOKENS, config_path=self.config_path, strategy=self.strategy, model_kwargs=self.model_kwargs, api_key=self.api_key) diff --git a/benchmark/scheduler.py b/benchmark/scheduler.py index 7c20627..cccec7b 100644 --- a/benchmark/scheduler.py +++ b/benchmark/scheduler.py @@ -1,6 +1,6 @@ """ -Multi-model scheduler: runs N sessions across M models fairly, in parallel, -with checkpoint/resume and hard budget caps. +Multi-model scheduler: runs N sessions across M models in parallel, +with checkpoint/resume. """ import json import threading @@ -11,18 +11,9 @@ from benchmark.verify import count_bugs -class BudgetExhausted(Exception): - pass - - -def _per_model_cap(total_budget: float, n_models: int) -> float: - """Each model gets an equal share of the total budget.""" - return round(total_budget / max(n_models, 1), 6) - - class Scheduler: """ - Runs sessions (model × rule) fairly under a shared budget. + Runs sessions (model × rule) with checkpoint/resume. checkpoint_path: JSON file where completed sessions are persisted. resume: if True, load the checkpoint and skip finished sessions. @@ -34,22 +25,15 @@ def __init__( runner: AgentRunner, models: list[str], rules: list[str], - total_budget: float, - per_rule_budget: float, results_dir: Path, checkpoint_path: Path, ctx, resume: bool = False, parallelism: int = 1, - safety_margin: float = 0.0, ): self.runner = runner self.models = models self.rules = rules - # Stop with margin: never plan to spend the last `safety_margin` dollars, so the - # call that crosses the line still lands under the true cap (cost_limit=19 not 20). - self.total_budget = max(total_budget - safety_margin, 0.0) - self.per_rule_budget = per_rule_budget self.results_dir = Path(results_dir) self.checkpoint_path = Path(checkpoint_path) self.ctx = ctx @@ -57,9 +41,7 @@ def __init__( self.parallelism = parallelism self._lock = threading.Lock() - self._total_spent: float = 0.0 self._completed: dict[str, list[dict]] = {m: [] for m in models} - self._spent: dict[str, float] = {m: 0.0 for m in models} if resume and self.checkpoint_path.exists(): self._load_checkpoint() @@ -71,8 +53,6 @@ def _load_checkpoint(self) -> None: for model, rows in data.get("completed", {}).items(): if model in self._completed: self._completed[model] = rows - self._spent[model] = sum(r.get("cost", 0.0) for r in rows) - self._total_spent = sum(self._spent.values()) def _save_checkpoint(self) -> None: self.checkpoint_path.parent.mkdir(parents=True, exist_ok=True) @@ -81,15 +61,6 @@ def _save_checkpoint(self) -> None: encoding="utf-8", ) - # ── budget helpers ──────────────────────────────────────────────────────── - - def _model_cap(self) -> float: - return _per_model_cap(self.total_budget, len(self.models)) - - def _budget_ok(self, model: str) -> bool: - cap = self._model_cap() - return self._total_spent < self.total_budget and self._spent[model] < cap - # ── completed-rule lookup ───────────────────────────────────────────────── def _done_rules(self, model: str) -> set[str]: @@ -106,7 +77,7 @@ def run_all(self) -> dict[str, list[dict]]: for rule in self.rules: if rule in done: continue - f = pool.submit(self._run_one, model, rule) + f = pool.submit(self.runner.run, self.ctx, model, rule) futures[f] = (model, rule) for f in as_completed(futures): @@ -122,38 +93,11 @@ def run_all(self) -> dict[str, list[dict]]: return dict(self._completed) - def _run_one(self, model: str, rule: str) -> dict: - with self._lock: - if not self._budget_ok(model): - return {"rule": rule, "result": "skipped_budget", "cost": 0.0, "tokens_k": 0.0} - # compute effective limit and pre-reserve budget to prevent overspend on burst - remaining_model = self._model_cap() - self._spent[model] - remaining_total = self.total_budget - self._total_spent - effective_limit = min(self.per_rule_budget, remaining_model, remaining_total) - # optimistically reserve the maximum we could spend - self._spent[model] += effective_limit - self._total_spent += effective_limit - - result = self.runner.run(self.ctx, model, rule, effective_limit) - - # correct the reservation to the actual spend, capped at effective_limit - # (a well-behaved runner stays within its limit; if it overspends we still - # only charge what we budgeted — the hard cap is enforced by the runner's - # own cost_limit, e.g. LiteLLM's LimitsExceeded) - actual = min(result.get("cost", 0.0), effective_limit) - with self._lock: - delta = actual - effective_limit - self._spent[model] += delta - self._total_spent += delta - - return result - # ── results output ──────────────────────────────────────────────────────── def _write_results(self, model: str) -> None: rows = self._completed[model] bugs = count_bugs(rows) # one rule = one bug - cost = self._spent[model] tokens_k = sum(r.get("tokens_k", 0.0) for r in rows) safe_model = model.replace("/", "_").replace(":", "_") out = self.results_dir / f"{safe_model}.json" @@ -162,10 +106,8 @@ def _write_results(self, model: str) -> None: "model": model, "library_commit": getattr(self.ctx, "commit_hash", "unknown"), "bugs_found": bugs, - "total_cost_usd": round(cost, 6), "total_tokens_k": round(tokens_k, 2), "efficiency_bugs_per_ktok": round(bugs / tokens_k, 4) if tokens_k else 0, - "efficiency_bugs_per_dollar": round(bugs / cost, 4) if cost else 0, "rules_tested": len(rows), "results": rows, }, indent=2), diff --git a/benchmark/submission.schema.json b/benchmark/submission.schema.json index aed26cb..dffff54 100644 --- a/benchmark/submission.schema.json +++ b/benchmark/submission.schema.json @@ -4,7 +4,7 @@ "title": "BenchmarkSubmission", "description": "A self-describing, rankable submission emitted by the dockerized runner and uploaded to the Space. Superset of results.schema.json: the results[] items stay byte-compatible with the scheduler's per-rule output, with an envelope added so the backend can rank and re-verify it.", "type": "object", - "required": ["schema_version", "model", "library_commit", "budget_cap", "bugs_found", "total_cost_usd", "total_tokens_k", "rules_tested", "results"], + "required": ["schema_version", "model", "library_commit", "bugs_found", "total_tokens_k", "rules_tested", "results"], "properties": { "schema_version": { "type": "string", @@ -18,21 +18,11 @@ "type": "string", "description": "Full git commit hash of the problem-reductions library the run targeted" }, - "budget_cap": { - "type": "number", - "minimum": 0, - "description": "Total USD budget the run was given. Must be 20 to be ranked." - }, "bugs_found": { "type": "integer", "minimum": 0, "description": "Self-reported distinct rules with a confirmed bug. ADVISORY ONLY — the backend recomputes this from pred re-verification and never trusts this number." }, - "total_cost_usd": { - "type": "number", - "minimum": 0, - "description": "Total spend in USD. DERIVED from token_usage × the declared prices (see usage_totals + prices); ADVISORY — the backend re-meters it from those and never trusts this figure. Falls back to the runner's cost tracking only when no prices are declared." - }, "total_tokens_k": { "type": "number", "minimum": 0, @@ -40,44 +30,33 @@ }, "usage_totals": { "type": "object", - "description": "Aggregate token usage split into the four billed buckets — the reproducible primitive the backend re-prices. Cost = input×price.input + output×price.output + cache_read×price.cache_read + cache_write×price.cache_write (USD per 1M).", + "description": "Aggregate token usage split into the four provider-reported buckets — the reproducible primitive behind total_tokens_k.", "properties": { "input": {"type": "integer", "minimum": 0, "description": "Uncached prompt tokens"}, - "output": {"type": "integer", "minimum": 0, "description": "Completion tokens (includes reasoning/thinking tokens, billed at the output rate)"}, + "output": {"type": "integer", "minimum": 0, "description": "Completion tokens (includes reasoning/thinking tokens)"}, "cache_read": {"type": "integer", "minimum": 0, "description": "Prompt tokens served from cache"}, "cache_write": {"type": "integer", "minimum": 0, "description": "Prompt tokens written to cache"} } }, - "prices": { - "type": ["object", "null"], - "description": "Declared per-token price snapshot (USD per 1,000,000 tokens), dated by created_at. The submitter pays the bill so supplies the rate; the backend re-meters cost as usage_totals × prices. null when no price was declared (e.g. a FAKE-mode run).", - "properties": { - "input": {"type": "number", "minimum": 0}, - "output": {"type": "number", "minimum": 0}, - "cache_read": {"type": "number", "minimum": 0}, - "cache_write": {"type": "number", "minimum": 0} - } - }, "rules_tested": { "type": "integer", "minimum": 0, - "description": "Number of rules the run attempted before the budget ran out" + "description": "Number of rules the run attempted" }, "results": { "type": "array", "description": "Per-rule results, byte-compatible with results.schema.json items", "items": { "type": "object", - "required": ["rule", "result", "cost", "tokens_k"], + "required": ["rule", "result", "tokens_k"], "if": {"properties": {"result": {"const": "bug_found"}}}, - "then": {"required": ["rule", "result", "cost", "tokens_k", "certificate"]}, + "then": {"required": ["rule", "result", "tokens_k", "certificate"]}, "properties": { "rule": {"type": "string"}, "result": { "type": "string", - "description": "Outcome for this rule. Canonical: bug_found | no_certificate | rejected | no_bug | error | skipped_budget. error rows may carry a free-form 'error: ' string." + "description": "Outcome for this rule. Canonical: bug_found | no_certificate | rejected | no_bug | error. error rows may carry a free-form 'error: ' string." }, - "cost": {"type": "number", "minimum": 0}, "tokens_k": {"type": "number", "minimum": 0}, "steps": {"type": "integer", "minimum": 0}, "certificate": { @@ -111,7 +90,6 @@ } }, "efficiency_bugs_per_ktok": {"type": "number", "minimum": 0}, - "efficiency_bugs_per_dollar": {"type": "number", "minimum": 0}, "submitted_by": {"type": ["string", "null"], "description": "Submitter identity (HF username / contact), filled at submit time"}, "runner_version": {"type": "string", "description": "Version tag of the runner image/script that produced this"}, "pred_version": {"type": "string", "description": "Version of the pred binary used (must match the pinned tag; verified server-side)"}, diff --git a/benchmark/submit.py b/benchmark/submit.py index ece1c33..86a9d27 100644 --- a/benchmark/submit.py +++ b/benchmark/submit.py @@ -31,8 +31,8 @@ import urllib.request from pathlib import Path -REQUIRED_ENVELOPE = ("schema_version", "model", "library_commit", "budget_cap", - "total_cost_usd", "total_tokens_k", "rules_tested", "results") +REQUIRED_ENVELOPE = ("schema_version", "model", "library_commit", + "total_tokens_k", "rules_tested", "results") def load_submission(path: Path) -> dict: diff --git a/benchmark/tests/test_backend_score.py b/benchmark/tests/test_backend_score.py index b8e5f95..74b52ed 100644 --- a/benchmark/tests/test_backend_score.py +++ b/benchmark/tests/test_backend_score.py @@ -22,12 +22,10 @@ def _traj(cert: dict) -> list[dict]: def _write_submission(subs_dir: Path, name: str, results, **over) -> Path: sub = { - "schema_version": "1.0", + "schema_version": "2.0", "model": over.pop("model", "anthropic/test"), "library_commit": "deadbeef", - "budget_cap": 20, "bugs_found": over.pop("bugs_found", 0), - "total_cost_usd": 2.0, "total_tokens_k": 50.0, "rules_tested": len(results), "results": results, @@ -43,7 +41,7 @@ def test_no_cert_submission_scores_zero(self, tmp_path): subs, results = tmp_path / "subs", tmp_path / "results" subs.mkdir() _write_submission(subs, "a.json", - [{"rule": "r1", "result": "no_certificate", "cost": 1.0, "tokens_k": 10.0}]) + [{"rule": "r1", "result": "no_certificate", "tokens_k": 10.0}]) summary = bs.process_local(str(subs), str(results)) assert summary[0]["status"] == "FINISHED" assert summary[0]["bugs_found"] == 0 @@ -54,7 +52,7 @@ def test_status_transitions_to_finished(self, tmp_path): subs, results = tmp_path / "subs", tmp_path / "results" subs.mkdir() _write_submission(subs, "a.json", - [{"rule": "r1", "result": "no_certificate", "cost": 1.0, "tokens_k": 10.0}]) + [{"rule": "r1", "result": "no_certificate", "tokens_k": 10.0}]) bs.process_local(str(subs), str(results)) status = json.loads((subs / "a.status.json").read_text()) assert status["status"] == "FINISHED" @@ -63,7 +61,7 @@ def test_idempotent_skips_finished(self, tmp_path): subs, results = tmp_path / "subs", tmp_path / "results" subs.mkdir() _write_submission(subs, "a.json", - [{"rule": "r1", "result": "no_certificate", "cost": 1.0, "tokens_k": 10.0}]) + [{"rule": "r1", "result": "no_certificate", "tokens_k": 10.0}]) bs.process_local(str(subs), str(results)) again = bs.process_local(str(subs), str(results)) assert again == [] # nothing left pending @@ -79,18 +77,17 @@ def test_leaderboard_aggregated_and_ranked(self, tmp_path, monkeypatch): cert = lambda v: {"rule": "r1", "violation": v, "source": {}, "bundle": {}} win_cert, lose_cert = cert("solve_mismatch"), cert("unsound_extraction") _write_submission(subs, "win.json", - [{"rule": "r1", "result": "bug_found", "cost": 1.0, "tokens_k": 10.0, + [{"rule": "r1", "result": "bug_found", "tokens_k": 10.0, "certificate": win_cert, "trajectory": _traj(win_cert)}], model="anthropic/winner") _write_submission(subs, "lose.json", - [{"rule": "r1", "result": "bug_found", "cost": 1.0, "tokens_k": 10.0, + [{"rule": "r1", "result": "bug_found", "tokens_k": 10.0, "certificate": lose_cert, "trajectory": _traj(lose_cert)}], model="anthropic/loser") bs.process_local(str(subs), str(results)) board = json.loads((results / "leaderboard.json").read_text()) assert [e["model"] for e in board] == ["anthropic/winner", "anthropic/loser"] assert board[0]["bugs_found"] == 1 and board[1]["bugs_found"] == 0 - assert all(e["budget_cap"] == 20 for e in board) def test_finds_nested_submissions(self, tmp_path): # Real layout: submissions//.json — must be found recursively. @@ -98,7 +95,7 @@ def test_finds_nested_submissions(self, tmp_path): nested = subs / "submissions" / "alice" nested.mkdir(parents=True) _write_submission(nested, "run.json", - [{"rule": "r1", "result": "no_certificate", "cost": 1.0, "tokens_k": 10.0}]) + [{"rule": "r1", "result": "no_certificate", "tokens_k": 10.0}]) summary = bs.process_local(str(subs), str(results)) assert len(summary) == 1 and summary[0]["status"] == "FINISHED" assert (nested / "run.status.json").exists() @@ -176,8 +173,7 @@ def test_slug_is_deterministic_and_tagged(self): assert slug == bs.board_slug(scored, stem) # deterministic assert slug == "anthropic-claude-sonnet-4-6--20260706T060247--dc9b2aae" full = {**scored, "results": [], "bugs_found": 0, "rules_tested": 0, - "total_cost_usd": 0, "total_tokens_k": 0, - "efficiency_bugs_per_ktok": 0, "efficiency_bugs_per_dollar": 0} + "total_tokens_k": 0, "efficiency_bugs_per_ktok": 0} e = bs.board_entry(full, stem) assert e["timestamp"] == "20260706T060247" and e["submission_id"] == "dc9b2aae" @@ -236,7 +232,7 @@ def test_genuine_bug_end_to_end_scores_via_real_pred(self, tmp_path): cert = json.loads(path.read_text(encoding="utf-8")) _write_submission(subs, "real.json", [{"rule": cert.get("rule", "r"), "result": "bug_found", - "cost": 1.0, "tokens_k": 10.0, "certificate": cert, + "tokens_k": 10.0, "certificate": cert, "trajectory": _traj(cert)}], model="anthropic/real") bs.process_local(str(subs), str(results)) @@ -251,7 +247,7 @@ def test_non_bug_fixture_scores_zero(self, tmp_path): cert = json.loads((FIXTURES / "valid_bug.json").read_text(encoding="utf-8")) _write_submission(subs, "real.json", [{"rule": cert.get("rule", "r"), "result": "bug_found", - "cost": 1.0, "tokens_k": 10.0, "certificate": cert}], + "tokens_k": 10.0, "certificate": cert}], model="anthropic/real") bs.process_local(str(subs), str(results)) board = json.loads((results / "leaderboard.json").read_text()) diff --git a/benchmark/tests/test_preflight.py b/benchmark/tests/test_preflight.py index 9091675..01dc610 100644 --- a/benchmark/tests/test_preflight.py +++ b/benchmark/tests/test_preflight.py @@ -8,7 +8,6 @@ from types import SimpleNamespace from benchmark import preflight as pf -from benchmark.cost import Price class _FakeModel: @@ -63,13 +62,12 @@ def test_no_rules_fails(self, monkeypatch): def test_model_call_failure_fails(self, monkeypatch): _patch(monkeypatch, model=_FakeModel(raise_exc=RuntimeError("401 unauthorized"))) - results = pf.run_checks("anthropic/x", repo_dir="/repo", price=None) + results = pf.run_checks("anthropic/x", repo_dir="/repo") call = dict((n, (ok, d)) for n, ok, d in results)["model call"] assert call[0] is False and "401" in call[1] - def test_model_call_cost_shown(self, monkeypatch): - # 1M input @ $1 + 1M output @ $2 = $3 total for this "call". + def test_model_call_tokens_shown(self, monkeypatch): _patch(monkeypatch, model=_FakeModel(prompt=1_000_000, completion=1_000_000)) - results = pf.run_checks("anthropic/x", repo_dir="/repo", price=Price(1.0, 2.0)) + results = pf.run_checks("anthropic/x", repo_dir="/repo") detail = dict((n, d) for n, _, d in results)["model call"] - assert "$3.0" in detail and "2000000 tokens" in detail + assert "2000000 tokens" in detail diff --git a/benchmark/tests/test_run_submission.py b/benchmark/tests/test_run_submission.py index 7425324..3aea512 100644 --- a/benchmark/tests/test_run_submission.py +++ b/benchmark/tests/test_run_submission.py @@ -2,53 +2,42 @@ Tests for benchmark/run_submission.py — the dockerized runner entry point. All tests run in FAKE mode (FakeRunner): no model API, no pred binary. They prove the -runner assembles a schema-valid, rankable submission.json and respects the $-budget cap. +runner assembles a schema-valid, rankable submission.json. """ import json from pathlib import Path -import pytest - from benchmark import run_submission as rs -from benchmark.cost import Price, Usage +from benchmark.usage import Usage -# ── cost is metered from tokens × declared price, not hand-reported ──────────── +# ── token totals are derived from the 4-bucket usage ────────────────────────── -class TestCostMetering: - def test_per_rule_cost_derived_from_row_usage(self): - # price given → total_cost_usd is price × summed row usage, and usage_totals/prices - # ride on the envelope. The self-reported per-row cost is NOT summed here. - price = Price(input=3.0, output=15.0) +class TestTokenTotals: + def test_per_rule_tokens_derived_from_row_usage(self): rows = [ - {"rule": "r1", "result": "no_certificate", "cost": 999.0, "tokens_k": 0.0, + {"rule": "r1", "result": "no_certificate", "tokens_k": 0.0, "usage": {"input": 1_000_000, "output": 0, "cache_read": 0, "cache_write": 0}}, - {"rule": "r2", "result": "no_certificate", "cost": 999.0, "tokens_k": 0.0, + {"rule": "r2", "result": "no_certificate", "tokens_k": 0.0, "usage": {"input": 0, "output": 1_000_000, "cache_read": 0, "cache_write": 0}}, ] - sub = rs.build_submission("m", rows, budget_cap=20.0, library_commit="c", price=price) - assert sub["total_cost_usd"] == 18.0 # 3 (input) + 15 (output), not 1998 + sub = rs.build_submission("m", rows, library_commit="c") assert sub["total_tokens_k"] == 2000.0 assert sub["usage_totals"] == {"input": 1_000_000, "output": 1_000_000, "cache_read": 0, "cache_write": 0} - assert sub["prices"] == {"input": 3.0, "output": 15.0, - "cache_read": 0.0, "cache_write": 0.0} - def test_whole_repo_cost_derived_from_session_usage(self): - price = Price(input=3.0, output=15.0) + def test_whole_repo_tokens_derived_from_session_usage(self): usage = Usage(input_tokens=2_000_000, output_tokens=0) - rows = [{"rule": "r1", "result": "bug_found", "cost": 0.0, "tokens_k": 0.0, + rows = [{"rule": "r1", "result": "bug_found", "tokens_k": 0.0, "certificate": {"rule": "r1"}}] - sub = rs.build_submission("m", rows, budget_cap=20.0, library_commit="c", - price=price, usage_totals=usage) - assert sub["total_cost_usd"] == 6.0 # 2M input @ $3 + sub = rs.build_submission("m", rows, library_commit="c", usage_totals=usage) + assert sub["total_tokens_k"] == 2000.0 assert sub["usage_totals"]["input"] == 2_000_000 - def test_no_price_falls_back_and_prices_null(self): - rows = [{"rule": "r1", "result": "no_certificate", "cost": 0.4, "tokens_k": 2.0}] - sub = rs.build_submission("m", rows, budget_cap=20.0, library_commit="c") - assert sub["total_cost_usd"] == 0.4 # row-sum fallback - assert sub["prices"] is None + def test_no_usage_falls_back_to_row_tokens(self): + rows = [{"rule": "r1", "result": "no_certificate", "tokens_k": 2.0}] + sub = rs.build_submission("m", rows, library_commit="c") + assert sub["total_tokens_k"] == 2.0 # row-sum fallback def _fake_repo(tmp_path: Path, rules: list[str]) -> Path: @@ -64,38 +53,38 @@ def _fake_repo(tmp_path: Path, rules: list[str]) -> Path: class TestBuildSubmission: def test_envelope_fields_present(self): - rows = [{"rule": "r1", "result": "no_certificate", "cost": 0.1, "tokens_k": 2.0}] - sub = rs.build_submission("anthropic/x", rows, budget_cap=20.0, - library_commit="abc123") - for k in ("schema_version", "model", "library_commit", "budget_cap", - "bugs_found", "total_cost_usd", "total_tokens_k", "rules_tested", - "results", "efficiency_bugs_per_ktok", "efficiency_bugs_per_dollar"): + rows = [{"rule": "r1", "result": "no_certificate", "tokens_k": 2.0}] + sub = rs.build_submission("anthropic/x", rows, library_commit="abc123") + for k in ("schema_version", "model", "library_commit", + "bugs_found", "total_tokens_k", "rules_tested", + "results", "efficiency_bugs_per_ktok"): assert k in sub - assert sub["budget_cap"] == 20.0 assert sub["model"] == "anthropic/x" assert sub["library_commit"] == "abc123" def test_bugs_counted_distinct_rules(self): rows = [ - {"rule": "r1", "result": "bug_found", "cost": 0.1, "tokens_k": 1.0, + {"rule": "r1", "result": "bug_found", "tokens_k": 1.0, "certificate": {"rule": "r1", "violation": "solve_mismatch"}}, - {"rule": "r1", "result": "bug_found", "cost": 0.1, "tokens_k": 1.0, + {"rule": "r1", "result": "bug_found", "tokens_k": 1.0, "certificate": {"rule": "r1", "violation": "unsound_extraction"}}, - {"rule": "r2", "result": "bug_found", "cost": 0.1, "tokens_k": 1.0, + {"rule": "r2", "result": "bug_found", "tokens_k": 1.0, "certificate": {"rule": "r2", "violation": "solve_mismatch"}}, - {"rule": "r3", "result": "no_certificate", "cost": 0.1, "tokens_k": 1.0}, + {"rule": "r3", "result": "no_certificate", "tokens_k": 1.0}, ] - sub = rs.build_submission("m", rows, budget_cap=20.0, library_commit="c") + sub = rs.build_submission("m", rows, library_commit="c") # distinct rules with a bug = {r1, r2} = 2 (not 3 certificates) assert sub["bugs_found"] == 2 - def test_rules_tested_excludes_skipped_budget(self): + def test_rules_tested_counts_distinct_rules(self): rows = [ - {"rule": "r1", "result": "no_certificate", "cost": 0.1, "tokens_k": 1.0}, - {"rule": "r2", "result": "skipped_budget", "cost": 0.0, "tokens_k": 0.0}, + {"rule": "r1", "result": "no_certificate", "tokens_k": 1.0}, + {"rule": "r1", "result": "bug_found", "tokens_k": 1.0, + "certificate": {"rule": "r1"}}, + {"rule": "r2", "result": "no_certificate", "tokens_k": 1.0}, ] - sub = rs.build_submission("m", rows, budget_cap=20.0, library_commit="c") - assert sub["rules_tested"] == 1 + sub = rs.build_submission("m", rows, library_commit="c") + assert sub["rules_tested"] == 2 # ── run() end-to-end with FakeRunner ────────────────────────────────────────── @@ -103,44 +92,26 @@ def test_rules_tested_excludes_skipped_budget(self): class TestRunFake: def test_produces_rankable_submission(self, tmp_path): repo = _fake_repo(tmp_path, ["a", "b", "c"]) - sub = rs.run("fake/model", str(repo), budget=10.0, per_rule_budget=1.0, - fake=True, library_commit="deadbeef") + sub = rs.run("fake/model", str(repo), fake=True, library_commit="deadbeef") assert sub["schema_version"] - assert sub["budget_cap"] == 10.0 - assert sub["rules_tested"] >= 1 + assert sub["rules_tested"] == 3 assert sub["bugs_found"] == 0 # default fake result is no_certificate - def test_total_spend_within_budget(self, tmp_path): - repo = _fake_repo(tmp_path, ["a", "b", "c", "d", "e"]) - sub = rs.run("fake/model", str(repo), budget=0.05, per_rule_budget=0.02, - fake=True, fake_cost=0.02, safety_margin=0.0, library_commit="c") - assert sub["total_cost_usd"] <= 0.05 + 1e-9 - - def test_safety_margin_held_back(self, tmp_path): - # Effective cap = budget - margin; spend stays under it (the reported cap is unchanged). - repo = _fake_repo(tmp_path, ["a", "b", "c", "d", "e", "f"]) - sub = rs.run("fake/model", str(repo), budget=0.10, per_rule_budget=0.01, - fake=True, fake_cost=0.01, safety_margin=0.04, library_commit="c") - assert sub["total_cost_usd"] <= 0.06 + 1e-9 - assert sub["budget_cap"] == 0.10 # the headline cap is still the full budget - def test_bug_results_counted(self, tmp_path): repo = _fake_repo(tmp_path, ["a", "b"]) - sub = rs.run("fake/model", str(repo), budget=10.0, per_rule_budget=1.0, - fake=True, fake_result="bug_found", library_commit="c") + sub = rs.run("fake/model", str(repo), fake=True, fake_result="bug_found", + library_commit="c") assert sub["bugs_found"] == 2 def test_max_rules_caps_attempts(self, tmp_path): repo = _fake_repo(tmp_path, ["a", "b", "c", "d"]) - sub = rs.run("fake/model", str(repo), budget=10.0, per_rule_budget=1.0, - fake=True, max_rules=2, library_commit="c") + sub = rs.run("fake/model", str(repo), fake=True, max_rules=2, library_commit="c") assert len(sub["results"]) == 2 def test_writes_output_file(self, tmp_path): repo = _fake_repo(tmp_path, ["a"]) out = tmp_path / "out" / "submission.json" - sub = rs.run("fake/model", str(repo), budget=10.0, per_rule_budget=1.0, - fake=True, library_commit="c", output=out) + sub = rs.run("fake/model", str(repo), fake=True, library_commit="c", output=out) assert out.exists() on_disk = json.loads(out.read_text()) assert on_disk["model"] == sub["model"] @@ -151,8 +122,7 @@ def test_writes_output_file(self, tmp_path): class TestSchemaValidity: def test_submission_matches_schema_required_fields(self, tmp_path): repo = _fake_repo(tmp_path, ["a", "b"]) - sub = rs.run("fake/model", str(repo), budget=20.0, per_rule_budget=1.0, - fake=True, library_commit="c") + sub = rs.run("fake/model", str(repo), fake=True, library_commit="c") schema = json.loads( (Path(rs.__file__).parent / "submission.schema.json").read_text()) for field in schema["required"]: diff --git a/benchmark/tests/test_scheduler.py b/benchmark/tests/test_scheduler.py index fae6ec2..3e6243b 100644 --- a/benchmark/tests/test_scheduler.py +++ b/benchmark/tests/test_scheduler.py @@ -2,28 +2,23 @@ Tests for benchmark/scheduler.py and benchmark/runner.py Design principle: all tests use FakeRunner — no API calls, no pred calls. -Three verification scenarios from issue #4: +Verification scenarios: 1. Swappable: pipeline runs end-to-end with FakeRunner (no mini-swe imports) 2. Resumable: kill partway, resume → only remaining sessions run, no duplicates - 3. Budget-fair: total spend ≤ cap, excess sessions marked "skipped_budget" Test categories: A. FakeRunner — basic interface contract B. Scheduler — swappable (end-to-end with FakeRunner) C. Scheduler — resumable (checkpoint load + skip finished) - D. Scheduler — budget-fair (total + per-model caps enforced) - E. Scheduler — results files written correctly - F. _per_model_cap helper + D. Scheduler — results files written correctly """ import json -import pytest from pathlib import Path from unittest.mock import MagicMock -from benchmark.runner import FakeRunner, AgentRunner -from benchmark.scheduler import Scheduler, _per_model_cap - +from benchmark.runner import AgentRunner, FakeRunner +from benchmark.scheduler import Scheduler # ── shared fixtures ─────────────────────────────────────────────────────────── @@ -43,26 +38,20 @@ def _make_scheduler( runner: FakeRunner | None = None, models: list[str] = MODELS, rules: list[str] = RULES, - total_budget: float = 10.0, - per_rule_budget: float = 1.0, resume: bool = False, parallelism: int = 1, - safety_margin: float = 0.0, ) -> Scheduler: if runner is None: - runner = FakeRunner(cost_per_rule=0.01) + runner = FakeRunner() return Scheduler( runner=runner, models=models, rules=rules, - total_budget=total_budget, - per_rule_budget=per_rule_budget, results_dir=tmp_path / "results", checkpoint_path=tmp_path / "checkpoint.json", ctx=_fake_ctx(), resume=resume, parallelism=parallelism, - safety_margin=safety_margin, ) @@ -70,35 +59,30 @@ def _make_scheduler( class TestFakeRunner: def test_returns_required_fields(self): - r = FakeRunner().run(None, "fake/model", "rule1", 1.0) + r = FakeRunner().run(None, "fake/model", "rule1") assert "rule" in r assert "result" in r - assert "cost" in r assert "tokens_k" in r def test_logs_calls(self): runner = FakeRunner() - runner.run(None, "model-a", "rule1", 1.0) - runner.run(None, "model-b", "rule2", 1.0) + runner.run(None, "model-a", "rule1") + runner.run(None, "model-b", "rule2") assert runner.call_log == [("model-a", "rule1"), ("model-b", "rule2")] def test_is_agent_runner_subclass(self): assert isinstance(FakeRunner(), AgentRunner) def test_custom_result(self): - r = FakeRunner(result="bug_found").run(None, "m", "r", 1.0) + r = FakeRunner(result="bug_found").run(None, "m", "r") assert r["result"] == "bug_found" - def test_custom_cost(self): - r = FakeRunner(cost_per_rule=0.05).run(None, "m", "r", 1.0) - assert r["cost"] == pytest.approx(0.05) - # ── B. Swappable ────────────────────────────────────────────────────────────── class TestSchedulerSwappable: def test_runs_end_to_end_with_fake_runner(self, tmp_path): - runner = FakeRunner(cost_per_rule=0.01) + runner = FakeRunner() s = _make_scheduler(tmp_path, runner=runner) results = s.run_all() # All models × rules should have a result @@ -106,7 +90,7 @@ def test_runs_end_to_end_with_fake_runner(self, tmp_path): assert len(results[model]) == len(RULES) def test_fake_runner_called_for_every_rule(self, tmp_path): - runner = FakeRunner(cost_per_rule=0.01) + runner = FakeRunner() s = _make_scheduler(tmp_path, runner=runner) s.run_all() calls = {(m, r) for m, r in runner.call_log} @@ -135,7 +119,7 @@ class TestSchedulerResumable: def _seed_checkpoint(self, checkpoint_path: Path, model: str, done_rules: list[str]): completed = {m: [] for m in MODELS} for rule in done_rules: - completed[model].append({"rule": rule, "result": "no_certificate", "cost": 0.01, "tokens_k": 0.5}) + completed[model].append({"rule": rule, "result": "no_certificate", "tokens_k": 0.5}) checkpoint_path.parent.mkdir(parents=True, exist_ok=True) checkpoint_path.write_text(json.dumps({"completed": completed}), encoding="utf-8") @@ -144,7 +128,7 @@ def test_resume_skips_finished_rules(self, tmp_path): # Pre-seed: model-a already did rule1 self._seed_checkpoint(checkpoint, MODELS[0], ["rule1"]) - runner = FakeRunner(cost_per_rule=0.01) + runner = FakeRunner() s = _make_scheduler(tmp_path, runner=runner, resume=True) s.run_all() @@ -156,7 +140,7 @@ def test_resume_does_not_duplicate_results(self, tmp_path): checkpoint = tmp_path / "checkpoint.json" self._seed_checkpoint(checkpoint, MODELS[0], ["rule1", "rule2"]) - runner = FakeRunner(cost_per_rule=0.01) + runner = FakeRunner() s = _make_scheduler(tmp_path, runner=runner, resume=True) results = s.run_all() @@ -168,7 +152,7 @@ def test_resume_completes_remaining_rules(self, tmp_path): checkpoint = tmp_path / "checkpoint.json" self._seed_checkpoint(checkpoint, MODELS[0], ["rule1"]) - runner = FakeRunner(cost_per_rule=0.01) + runner = FakeRunner() s = _make_scheduler(tmp_path, runner=runner, resume=True) results = s.run_all() @@ -179,7 +163,7 @@ def test_no_resume_reruns_everything(self, tmp_path): checkpoint = tmp_path / "checkpoint.json" self._seed_checkpoint(checkpoint, MODELS[0], ["rule1"]) - runner = FakeRunner(cost_per_rule=0.01) + runner = FakeRunner() s = _make_scheduler(tmp_path, runner=runner, resume=False) s.run_all() @@ -188,7 +172,7 @@ def test_no_resume_reruns_everything(self, tmp_path): assert "rule1" in model_a_calls def test_checkpoint_written_after_each_session(self, tmp_path): - runner = FakeRunner(cost_per_rule=0.01) + runner = FakeRunner() s = _make_scheduler(tmp_path, runner=runner, parallelism=1) s.run_all() assert (tmp_path / "checkpoint.json").exists() @@ -196,57 +180,7 @@ def test_checkpoint_written_after_each_session(self, tmp_path): assert "completed" in data -# ── D. Budget-fair ──────────────────────────────────────────────────────────── - -class TestSchedulerBudgetFair: - def test_total_spend_never_exceeds_budget(self, tmp_path): - total_budget = 0.05 # 5 cents — very tight - runner = FakeRunner(cost_per_rule=0.02) - s = _make_scheduler(tmp_path, runner=runner, total_budget=total_budget, per_rule_budget=0.02) - s.run_all() - assert s._total_spent <= total_budget + 1e-9 - - def test_safety_margin_held_back(self, tmp_path): - # With a $0.02 margin on a $0.10 budget, effective cap is $0.08 — spend stays under it. - runner = FakeRunner(cost_per_rule=0.01) - s = _make_scheduler(tmp_path, runner=runner, total_budget=0.10, - per_rule_budget=0.01, safety_margin=0.02) - s.run_all() - assert s._total_spent <= 0.08 + 1e-9 - - def test_excess_sessions_marked_skipped_budget(self, tmp_path): - # Budget of 0.04 with 0.02/rule and 2 models × 3 rules = 6 sessions = $0.12 needed - # → roughly half should be skipped - total_budget = 0.04 - runner = FakeRunner(cost_per_rule=0.02) - s = _make_scheduler(tmp_path, runner=runner, total_budget=total_budget, per_rule_budget=0.02) - results = s.run_all() - all_results = [r for rows in results.values() for r in rows] - skipped = [r for r in all_results if r["result"] == "skipped_budget"] - assert len(skipped) > 0 - - def test_models_get_equal_budget_share(self, tmp_path): - """Each model's spend should not exceed its equal share of the total budget.""" - total_budget = 0.06 # $0.03 per model - runner = FakeRunner(cost_per_rule=0.01) - s = _make_scheduler(tmp_path, runner=runner, total_budget=total_budget, per_rule_budget=0.01) - s.run_all() - cap_per_model = _per_model_cap(total_budget, len(MODELS)) - for model in MODELS: - assert s._spent[model] <= cap_per_model + 1e-9 - - def test_skipped_sessions_cost_zero(self, tmp_path): - total_budget = 0.01 # almost nothing - runner = FakeRunner(cost_per_rule=0.02) - s = _make_scheduler(tmp_path, runner=runner, total_budget=total_budget, per_rule_budget=0.02) - results = s.run_all() - for rows in results.values(): - for r in rows: - if r["result"] == "skipped_budget": - assert r["cost"] == 0.0 - - -# ── E. Results files ────────────────────────────────────────────────────────── +# ── D. Results files ────────────────────────────────────────────────────────── class TestSchedulerResultsFiles: def test_results_file_written_per_model(self, tmp_path): @@ -260,9 +194,9 @@ def test_results_file_has_required_fields(self, tmp_path): s = _make_scheduler(tmp_path) s.run_all() required = { - "model", "library_commit", "bugs_found", "total_cost_usd", + "model", "library_commit", "bugs_found", "total_tokens_k", "efficiency_bugs_per_ktok", - "efficiency_bugs_per_dollar", "rules_tested", "results", + "rules_tested", "results", } for model in MODELS: safe = model.replace("/", "_").replace(":", "_") @@ -278,24 +212,10 @@ def test_results_file_rules_tested_count(self, tmp_path): assert data["rules_tested"] == len(results[model]) def test_bug_found_count_in_file(self, tmp_path): - runner = FakeRunner(cost_per_rule=0.01, result="bug_found") + runner = FakeRunner(result="bug_found") s = _make_scheduler(tmp_path, runner=runner) s.run_all() for model in MODELS: safe = model.replace("/", "_").replace(":", "_") data = json.loads((tmp_path / "results" / f"{safe}.json").read_text()) assert data["bugs_found"] == len(RULES) - - -# ── F. _per_model_cap ───────────────────────────────────────────────────────── - -class TestPerModelCap: - def test_equal_split(self): - assert _per_model_cap(1.0, 4) == pytest.approx(0.25) - - def test_single_model(self): - assert _per_model_cap(10.0, 1) == pytest.approx(10.0) - - def test_zero_models_no_division_error(self): - # guard against zero-division - assert _per_model_cap(10.0, 0) == pytest.approx(10.0) diff --git a/benchmark/tests/test_submit.py b/benchmark/tests/test_submit.py index 3c0d9e0..a6723dc 100644 --- a/benchmark/tests/test_submit.py +++ b/benchmark/tests/test_submit.py @@ -9,10 +9,10 @@ def _valid(tmp_path: Path, results=None) -> Path: doc = { - "schema_version": "1.0", "model": "anthropic/test", "library_commit": "deadbeef", - "budget_cap": 20, "total_cost_usd": 1.0, "total_tokens_k": 10.0, + "schema_version": "2.0", "model": "anthropic/test", "library_commit": "deadbeef", + "total_tokens_k": 10.0, "rules_tested": 1, "results": results if results is not None else [ - {"rule": "r1", "result": "no_certificate", "cost": 0.1, "tokens_k": 1.0}], + {"rule": "r1", "result": "no_certificate", "tokens_k": 1.0}], } p = tmp_path / "submission.json" p.write_text(json.dumps(doc), encoding="utf-8") @@ -20,7 +20,7 @@ def _valid(tmp_path: Path, results=None) -> Path: def _bug_row(with_cert=True, with_traj=True) -> dict: - row = {"rule": "r1", "result": "bug_found", "cost": 0.1, "tokens_k": 1.0} + row = {"rule": "r1", "result": "bug_found", "tokens_k": 1.0} if with_cert: row["certificate"] = {"rule": "r1", "source": {}} if with_traj: @@ -36,8 +36,8 @@ def test_missing_envelope_field(self): assert any("model" in p for p in sub.validate_submission({"results": []})) def test_bug_found_needs_certificate_and_trajectory(self): - doc = {**json.loads('{"schema_version":"1","model":"m","library_commit":"c",' - '"budget_cap":20,"total_cost_usd":0,"total_tokens_k":0,"rules_tested":1}'), + doc = {**json.loads('{"schema_version":"2","model":"m","library_commit":"c",' + '"total_tokens_k":0,"rules_tested":1}'), "results": [_bug_row(with_cert=False, with_traj=False)]} problems = sub.validate_submission(doc) assert any("no certificate" in p for p in problems) diff --git a/benchmark/tests/test_cost.py b/benchmark/tests/test_usage.py similarity index 61% rename from benchmark/tests/test_cost.py rename to benchmark/tests/test_usage.py index 69c6630..2352150 100644 --- a/benchmark/tests/test_cost.py +++ b/benchmark/tests/test_usage.py @@ -1,18 +1,13 @@ """ -Tests for benchmark/cost.py — authoritative token-based cost accounting. +Tests for benchmark/usage.py — token-usage accounting. -The point of this module is that we never trust the gateway's dollar figure, so these -tests pin the token→USD math, the OpenAI vs Anthropic usage mapping, and price resolution. +Pins the OpenAI vs Anthropic usage mapping, the 4-bucket sums, and (de)serialization. """ from types import SimpleNamespace -from benchmark.cost import ( - Price, +from benchmark.usage import ( Usage, extract_usage, - price_as_dict, - price_from_dict, - resolve_price, usage_as_dict, usage_from_dict, usage_from_response, @@ -28,35 +23,6 @@ def test_usage_from_dict_tolerates_missing_and_none(self): assert usage_from_dict(None) == Usage() assert usage_from_dict({"input": 7}) == Usage(input_tokens=7) - def test_price_roundtrips_through_dict(self): - p = Price(input=3.0, output=15.0, cache_read=0.3, cache_write=3.75) - assert price_from_dict(price_as_dict(p)) == p - - def test_price_from_dict_none_is_none(self): - assert price_from_dict(None) is None - assert price_from_dict({}) is None - - def test_reprice_from_serialized_matches_direct(self): - # A submission stores usage_as_dict + price_as_dict; the backend re-prices from them. - u = Usage(input_tokens=1_000_000, output_tokens=500_000, cache_read_tokens=200_000) - p = Price(input=3.0, output=15.0, cache_read=0.3) - assert price_from_dict(price_as_dict(p)).cost(usage_from_dict(usage_as_dict(u))) == p.cost(u) - - -class TestPriceCost: - def test_basic_input_output(self): - # 1M input @ $3, 0.5M output @ $15 = 3 + 7.5 - p = Price(3.0, 15.0) - assert p.cost(Usage(input_tokens=1_000_000, output_tokens=500_000)) == 10.5 - - def test_cache_buckets_priced_separately(self): - p = Price(3.0, 15.0, cache_read=0.3, cache_write=3.75) - u = Usage(0, 0, cache_read_tokens=1_000_000, cache_write_tokens=1_000_000) - assert abs(p.cost(u) - (0.3 + 3.75)) < 1e-9 - - def test_zero_usage_zero_cost(self): - assert Price(3.0, 15.0).cost(Usage()) == 0.0 - class TestUsage: def test_add_and_total(self): @@ -101,17 +67,6 @@ def test_missing_fields_default_zero(self): assert u.input_tokens == 42 and u.output_tokens == 0 -class TestResolvePrice: - def test_override_wins(self): - ov = Price(1.0, 2.0) - assert resolve_price("anthropic/claude-sonnet-4-6", ov) is ov - - def test_no_override_is_none(self): - # No built-in table by design — price must always be supplied for a real run. - assert resolve_price("anthropic/claude-sonnet-4-6") is None - assert resolve_price("some/unknown-model") is None - - class TestExtractUsage: def _msg(self, **usage_fields): return {"extra": {"response": SimpleNamespace(usage=SimpleNamespace(**usage_fields))}} diff --git a/benchmark/tests/test_verify_submission.py b/benchmark/tests/test_verify_submission.py index 883c56c..47fa7f2 100644 --- a/benchmark/tests/test_verify_submission.py +++ b/benchmark/tests/test_verify_submission.py @@ -22,7 +22,7 @@ def _traj(cert: dict) -> list[dict]: def _bug_row(cert: dict, **over) -> dict: - row = {"rule": cert.get("rule", "r"), "result": "bug_found", "cost": 1.0, + row = {"rule": cert.get("rule", "r"), "result": "bug_found", "tokens_k": 10.0, "certificate": cert, "trajectory": _traj(cert)} row.update(over) return row @@ -30,12 +30,10 @@ def _bug_row(cert: dict, **over) -> dict: def _submission(results, **over) -> dict: base = { - "schema_version": "1.0", + "schema_version": "2.0", "model": "anthropic/test", "library_commit": "deadbeef", - "budget_cap": 20, "bugs_found": 999, # deliberately wrong — the scorer must ignore it - "total_cost_usd": 2.0, "total_tokens_k": 50.0, "rules_tested": len(results), "results": results, @@ -71,7 +69,7 @@ def test_no_trajectory_not_counted(self, monkeypatch): # not scored (provenance gate): a pasted answer key must not count. monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok")) sub = _submission([ - {"rule": "r1", "result": "bug_found", "cost": 1.0, "tokens_k": 10.0, + {"rule": "r1", "result": "bug_found", "tokens_k": 10.0, "certificate": {"rule": "r1", "violation": "solve_mismatch", "source": {}, "bundle": {}}}, ]) scored, report = vs.score_submission(sub) @@ -109,8 +107,8 @@ def test_envelope_trajectory_provenance(self, monkeypatch): monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok")) c1 = {"rule": "r1", "violation": "solve_mismatch", "source": {"n": 1}, "bundle": {}} c2 = {"rule": "r2", "violation": "solve_mismatch", "source": {"n": 2}, "bundle": {}} - rows = [{"rule": "r1", "result": "bug_found", "cost": 0.0, "tokens_k": 0.0, "certificate": c1}, - {"rule": "r2", "result": "bug_found", "cost": 0.0, "tokens_k": 0.0, "certificate": c2}] + rows = [{"rule": "r1", "result": "bug_found", "tokens_k": 0.0, "certificate": c1}, + {"rule": "r2", "result": "bug_found", "tokens_k": 0.0, "certificate": c2}] sub = _submission(rows, trajectory=_traj(c1) + _traj(c2)) scored, _ = vs.score_submission(sub) assert scored["bugs_found"] == 2 @@ -119,7 +117,7 @@ def test_no_trajectory_anywhere_not_counted(self, monkeypatch): # A cert with neither a row trajectory nor an envelope trajectory fails provenance. monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok")) c1 = {"rule": "r1", "violation": "solve_mismatch", "source": {}, "bundle": {}} - rows = [{"rule": "r1", "result": "bug_found", "cost": 0.0, "tokens_k": 0.0, "certificate": c1}] + rows = [{"rule": "r1", "result": "bug_found", "tokens_k": 0.0, "certificate": c1}] scored, _ = vs.score_submission(_submission(rows)) assert scored["bugs_found"] == 0 @@ -137,41 +135,36 @@ def test_distinct_rule_dedup(self, monkeypatch): def test_rows_without_certificate_passthrough(self, monkeypatch): monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok")) sub = _submission([ - {"rule": "r1", "result": "no_certificate", "cost": 0.5, "tokens_k": 5.0}, + {"rule": "r1", "result": "no_certificate", "tokens_k": 5.0}, ]) scored, report = vs.score_submission(sub) assert scored["bugs_found"] == 0 assert report == [] - def test_remeters_cost_from_prices_and_usage(self, monkeypatch): - # Zero-trust cost: the submission self-reports a bogus total_cost_usd, but carries - # prices + usage_totals — the scorer recomputes cost = usage × price and ignores the - # bogus figure (mirrors ignoring self-reported bugs_found). + def test_recomputes_tokens_from_usage_totals(self, monkeypatch): + # Zero-trust tokens: the submission self-reports a bogus total_tokens_k, but carries + # usage_totals — the scorer recomputes tokens from the 4-bucket primitive and ignores + # the bogus figure (mirrors ignoring self-reported bugs_found). monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok")) sub = _submission( [_bug_row({"rule": "r1", "violation": "solve_mismatch", "source": {}, "bundle": {}})], - total_cost_usd=0.0001, # bogus — must be ignored - total_tokens_k=0.001, - prices={"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + total_tokens_k=0.001, # bogus — must be ignored usage_totals={"input": 1_000_000, "output": 1_000_000, "cache_read": 0, "cache_write": 0}, ) scored, _ = vs.score_submission(sub) - assert scored["total_cost_usd"] == 18.0 # 3 + 15, not 0.0001 assert scored["total_tokens_k"] == 2000.0 - assert scored["efficiency_bugs_per_dollar"] == round(1 / 18.0, 4) - # the primitive + snapshot survive into the scored result (→ leaderboard entry) + assert scored["efficiency_bugs_per_ktok"] == round(1 / 2000.0, 4) + # the primitive survives into the scored result (→ leaderboard entry) assert scored["usage_totals"]["input"] == 1_000_000 - assert scored["prices"]["output"] == 15.0 - def test_legacy_submission_without_prices_uses_self_reported(self, monkeypatch): - # No prices/usage_totals (legacy) → fall back to the self-reported figures. + def test_legacy_submission_without_usage_uses_self_reported(self, monkeypatch): + # No usage_totals (legacy) → fall back to the self-reported figure. monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok")) sub = _submission( [_bug_row({"rule": "r1", "violation": "solve_mismatch", "source": {}, "bundle": {}})], - total_cost_usd=2.0, total_tokens_k=50.0) + total_tokens_k=50.0) scored, _ = vs.score_submission(sub) - assert scored["total_cost_usd"] == 2.0 assert scored["total_tokens_k"] == 50.0 def test_scored_is_results_schema_shaped(self, monkeypatch): @@ -180,9 +173,9 @@ def test_scored_is_results_schema_shaped(self, monkeypatch): _bug_row({"rule": "r1", "violation": "solve_mismatch", "source": {}, "bundle": {}}), ]) scored, _ = vs.score_submission(sub) - for field in ("model", "library_commit", "bugs_found", "total_cost_usd", + for field in ("model", "library_commit", "bugs_found", "total_tokens_k", "efficiency_bugs_per_ktok", - "efficiency_bugs_per_dollar", "rules_tested", "results"): + "rules_tested", "results"): assert field in scored @@ -190,16 +183,15 @@ def test_scored_is_results_schema_shaped(self, monkeypatch): class TestLeaderboardEntry: def test_aggregate_only_no_certificates(self, monkeypatch): - # The public entry must carry counts/cost but NEVER the certificates or the + # The public entry must carry counts/tokens but NEVER the certificates or the # identities of the buggy rules — publishing those is a free answer key. monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok")) sub = _submission([ _bug_row({"rule": "r1", "violation": "solve_mismatch", "source": {"type": "A"}, "bundle": {"target": {"type": "B"}}, "note": "x"}), - ], budget_cap=20) + ]) scored, _ = vs.score_submission(sub) entry = vs.leaderboard_entry(sub, scored) - assert entry["budget_cap"] == 20 assert entry["placeholder"] is False assert entry["bugs_found"] == 1 # no per-bug drilldown, no rule identities, no certificate fields leak out diff --git a/benchmark/tests/test_whole_repo.py b/benchmark/tests/test_whole_repo.py index 3fdb0b2..88c50de 100644 --- a/benchmark/tests/test_whole_repo.py +++ b/benchmark/tests/test_whole_repo.py @@ -93,34 +93,32 @@ def test_trajectory_and_disk_log_dedup(self, monkeypatch): class TestBuildSubmissionTotals: def test_explicit_session_totals_override_row_sums(self): - # whole-repo rows carry 0 cost/tokens; the session totals come in as explicit args. - rows = [{"rule": "r1", "result": "bug_found", "cost": 0.0, "tokens_k": 0.0}] + # whole-repo rows carry 0 tokens; the session total comes in as an explicit arg. + rows = [{"rule": "r1", "result": "bug_found", "tokens_k": 0.0}] sub = run_submission.build_submission( - "m", rows, budget_cap=20, library_commit="c", - total_cost_usd=3.5, total_tokens_k=42.0) - assert sub["total_cost_usd"] == 3.5 + "m", rows, library_commit="c", total_tokens_k=42.0) assert sub["total_tokens_k"] == 42.0 - assert sub["efficiency_bugs_per_dollar"] == round(1 / 3.5, 4) + assert sub["efficiency_bugs_per_ktok"] == round(1 / 42.0, 4) class TestCrashSalvage: def test_run_error_recorded_when_session_dies(self, monkeypatch): # A fatal session error must still produce a submission (partial salvage) tagged with # run_error — not crash and leave a stale submission.json. - def dying_session(model, ctx, cost_limit, **kw): - return {"rows": [], "cost": 0.3, "tokens_k": 5.0, "trajectory": [], + def dying_session(model, ctx, **kw): + return {"rows": [], "tokens_k": 5.0, "trajectory": [], "usage": None, "error": "APIError: quota exhausted"} monkeypatch.setattr(run_submission, "find_pred_binary", lambda: "pred") monkeypatch.setattr(run_submission, "verify_pred_version", lambda p: "1.2.3") monkeypatch.setattr(run_submission, "EnvContext", lambda **kw: type("C", (), {"pred_version": "1.2.3", **kw})()) monkeypatch.setattr("benchmark.run_mini.run_repo_session", dying_session) - sub = run_submission.run("m", "/repo", budget=5, library_commit="c", mode="whole-repo") + sub = run_submission.run("m", "/repo", library_commit="c", mode="whole-repo") assert sub["run_error"] == "APIError: quota exhausted" assert sub["bugs_found"] == 0 def test_no_run_error_key_on_clean_run(self): - sub = run_submission.build_submission("m", [], budget_cap=5, library_commit="c") + sub = run_submission.build_submission("m", [], library_commit="c") assert "run_error" not in sub @@ -128,15 +126,12 @@ class TestRunWholeRepoWiring: def test_run_dispatches_to_repo_session(self, monkeypatch): # run(mode="whole-repo") must call run_repo_session and build an envelope from it, # without touching the per-rule Scheduler. - captured = {} - traj = [{"role": "assistant", "content": "run log"}] - def fake_repo_session(model, ctx, cost_limit, **kw): - captured["cost_limit"] = cost_limit - return {"rows": [{"rule": "r1", "result": "bug_found", "cost": 0.0, "tokens_k": 0.0, + def fake_repo_session(model, ctx, **kw): + return {"rows": [{"rule": "r1", "result": "bug_found", "tokens_k": 0.0, "certificate": {"rule": "r1", "source": {}}}], - "cost": 5.0, "tokens_k": 30.0, "trajectory": traj} + "tokens_k": 30.0, "trajectory": traj} monkeypatch.setattr(run_submission, "find_pred_binary", lambda: "pred") monkeypatch.setattr(run_submission, "verify_pred_version", lambda p: "1.2.3") @@ -146,10 +141,8 @@ def fake_repo_session(model, ctx, cost_limit, **kw): lambda **kw: (_ for _ in ()).throw(AssertionError("scheduler used"))) monkeypatch.setattr("benchmark.run_mini.run_repo_session", fake_repo_session) - sub = run_submission.run("m", "/repo", budget=20, safety_margin=1.0, - library_commit="deadbeef", mode="whole-repo") - assert captured["cost_limit"] == 19.0 # budget - safety_margin - assert sub["total_cost_usd"] == 5.0 + sub = run_submission.run("m", "/repo", library_commit="deadbeef", mode="whole-repo") + assert sub["total_tokens_k"] == 30.0 assert sub["bugs_found"] == 1 assert sub["trajectory"] is traj # stored once on the envelope assert "trajectory" not in sub["results"][0] # not duplicated onto rows diff --git a/benchmark/cost.py b/benchmark/usage.py similarity index 52% rename from benchmark/cost.py rename to benchmark/usage.py index 465a90f..6e853dd 100644 --- a/benchmark/cost.py +++ b/benchmark/usage.py @@ -1,27 +1,16 @@ """ -Authoritative cost accounting for the hard budget cap. +Token-usage accounting. -The submitter pays the bill (their own key, their own negotiated rate), so they supply -their model's per-token price. We recompute cost from raw token usage x that price instead -of trusting the model gateway's self-reported dollar figure — which can be stale or plain -wrong (LiteLLM $0-pricing incidents; Anthropic prompt-cache mis-pricing ~10x). Recomputing -from tokens is what turns the $20 budget into a *hard* cap. - -For a budget guard, over-counting is safe and under-counting is not, so callers combine -this figure with the gateway's via max() (see run_mini). - -Prices are USD per 1,000,000 tokens (the conventional unit). There is deliberately NO -built-in price table: a stale or wrong default would silently mis-meter the budget, so the -price is always submitter-supplied (PRICE_IN / PRICE_OUT) for a real run. +Tokens are the reproducible primitive a run reports: raw per-bucket counts summed from the +agent trajectory. There is no dollar metering and no budget — runs are bounded by the agent +step limit, and the leaderboard's efficiency metric is bugs per 1K tokens. """ from dataclasses import dataclass -MTOK = 1_000_000 - @dataclass(frozen=True) class Usage: - """Token counts split into the four disjoint buckets we price separately.""" + """Token counts split into the four disjoint buckets providers report separately.""" input_tokens: int = 0 # uncached prompt tokens output_tokens: int = 0 # completion tokens cache_read_tokens: int = 0 # prompt tokens served from cache @@ -41,33 +30,6 @@ def total_tokens(self) -> int: + self.cache_read_tokens + self.cache_write_tokens) -@dataclass(frozen=True) -class Price: - """Per-1M-token USD rates. When a provider doesn't bill caching separately, leave - cache_read/cache_write at 0 and fold those tokens into input via extract_usage.""" - input: float - output: float - cache_read: float = 0.0 - cache_write: float = 0.0 - - def cost(self, usage: Usage) -> float: - return ( - usage.input_tokens * self.input - + usage.output_tokens * self.output - + usage.cache_read_tokens * self.cache_read - + usage.cache_write_tokens * self.cache_write - ) / MTOK - - -def resolve_price(model: str, override: Price | None = None) -> Price | None: - """Return the submitter-supplied price, or None if none was given. - - Intentionally has no fallback table — `model` is accepted only for a uniform call site. - None means "no price"; the caller decides what to do (a real run must reject it; see - run_submission, which requires PRICE_IN/PRICE_OUT).""" - return override - - def _get(obj, key: str, default=0): """Read `key` from an object attribute or a dict, treating None/missing as default.""" if obj is None: @@ -84,8 +46,7 @@ def usage_from_response(usage) -> Usage: Some raw providers instead report `cache_read_input_tokens` / `cache_creation_input_tokens` at the top level — we read both, preferring the details block. `cached_tokens` (cache read) is a subset of `prompt_tokens`, so it is subtracted out of the uncached input; - cache-write is counted as its own bucket (never under-counting, which is the safe side - for a budget guard).""" + cache-write is counted as its own bucket.""" prompt = int(_get(usage, "prompt_tokens")) completion = int(_get(usage, "completion_tokens")) details = _get(usage, "prompt_tokens_details", None) @@ -111,12 +72,6 @@ def extract_usage(messages: list) -> Usage: return total -# ── (de)serialization: the 4-bucket token totals + the declared price snapshot ────── -# These travel in the submission so the backend can RE-METER cost as tokens × price -# (zero-trust, mirroring the bug re-verification) instead of trusting a self-reported -# dollar total. Tokens are the reproducible primitive; the declared price (dated by the -# submission's created_at) is a snapshot anyone can swap to recompute under other prices. - def usage_as_dict(u: Usage) -> dict: """Serialize a Usage to the 4-bucket dict shape used on rows and the envelope.""" return {"input": u.input_tokens, "output": u.output_tokens, @@ -131,21 +86,3 @@ def usage_from_dict(d) -> Usage: cache_read_tokens=int(_get(d, "cache_read")), cache_write_tokens=int(_get(d, "cache_write")), ) - - -def price_as_dict(p: Price) -> dict: - """Serialize a Price to a plain dict (USD per 1M tokens, per bucket).""" - return {"input": p.input, "output": p.output, - "cache_read": p.cache_read, "cache_write": p.cache_write} - - -def price_from_dict(d) -> Price | None: - """Parse a price dict back into a Price, or None if absent (legacy submission).""" - if not d: - return None - return Price( - input=float(_get(d, "input")), - output=float(_get(d, "output")), - cache_read=float(_get(d, "cache_read")), - cache_write=float(_get(d, "cache_write")), - ) diff --git a/benchmark/verify_submission.py b/benchmark/verify_submission.py index dc22659..55e10db 100644 --- a/benchmark/verify_submission.py +++ b/benchmark/verify_submission.py @@ -8,9 +8,9 @@ Produces two views of the result: * ``scored`` — results.schema.json-compatible (the backend's per-submission output) - * ``leaderboard_entry`` — the public ranked-row shape (aggregate only: counts, cost, - efficiency, budget_cap — never the certificates or buggy-rule - identities, which would be a free answer key) + * ``leaderboard_entry`` — the public ranked-row shape (aggregate only: counts, tokens, + efficiency — never the certificates or buggy-rule identities, + which would be a free answer key) CLI: python -m benchmark.verify_submission [--repo-dir ] @@ -23,7 +23,7 @@ import sys from pathlib import Path -from benchmark.cost import price_from_dict, usage_from_dict +from benchmark.usage import usage_from_dict from benchmark.verify import count_bugs, verify CERT_BLOCK = re.compile(r"CERTIFICATE_START\s*\n(.*?)CERTIFICATE_END", re.DOTALL) @@ -112,21 +112,11 @@ def score_submission(submission: dict, repo_dir: str | None = None) -> tuple[dic }) bugs = count_bugs(rescored) - # Cost re-metering, zero-trust like the bug re-verification: when the submission carries - # the declared per-token ``prices`` + the 4-bucket ``usage_totals``, recompute spend as - # tokens × price and IGNORE the self-reported total_cost_usd/total_tokens_k. Tokens are - # the reproducible primitive; the price is a dated snapshot (created_at). Legacy - # submissions without these fields fall back to their self-reported figures. - price = price_from_dict(submission.get("prices")) - usage = submission.get("usage_totals") - if price is not None and usage is not None: - u = usage_from_dict(usage) - cost = price.cost(u) - tokens_k = u.total_tokens / 1000 - else: - tokens_k = submission.get("total_tokens_k", 0.0) or 0.0 - cost = submission.get("total_cost_usd", 0.0) or 0.0 - attempted = [r for r in rescored if r.get("result") != "skipped_budget"] + # Token totals: recompute from the 4-bucket ``usage_totals`` when present (the + # reproducible primitive); legacy submissions without it fall back to their + # self-reported total_tokens_k. + total_tokens = usage_from_dict(submission.get("usage_totals")).total_tokens + tokens_k = total_tokens / 1000 if total_tokens else (submission.get("total_tokens_k", 0.0) or 0.0) scored = { "model": submission.get("model", "unknown"), "library_commit": submission.get("library_commit", "unknown"), @@ -135,15 +125,11 @@ def score_submission(submission: dict, repo_dir: str | None = None) -> tuple[dic # board. Carried through here so the flag survives in the private scored file. "test": bool(submission.get("test")), "bugs_found": bugs, - "total_cost_usd": round(cost, 6), "total_tokens_k": round(tokens_k, 2), - # The reproducible primitive + the price snapshot the cost was metered from — - # carried through so the public entry stays recomputable under other prices. + # The reproducible primitive behind total_tokens_k — carried through. "usage_totals": submission.get("usage_totals"), - "prices": submission.get("prices"), "efficiency_bugs_per_ktok": round(bugs / tokens_k, 4) if tokens_k else 0, - "efficiency_bugs_per_dollar": round(bugs / cost, 4) if cost else 0, - "rules_tested": submission.get("rules_tested", len(attempted)), + "rules_tested": submission.get("rules_tested", len(rescored)), "results": rescored, } return scored, report @@ -152,27 +138,22 @@ def score_submission(submission: dict, repo_dir: str | None = None) -> tuple[dic def leaderboard_entry(submission: dict, scored: dict) -> dict: """Build the public ranked-row entry from a scored result. - Aggregate-only, by design: it carries counts, cost/token totals, efficiency and - ``budget_cap`` (rows with budget_cap == 20 are ranked) but NEVER the certificates or - the identities of the buggy rules. Publishing those would be a free answer key — on a - public library commit a `pred`-confirmed certificate counts regardless of provenance, - so anyone could copy it. The full certificates stay in the private scored result the - maintainer holds; the leaderboard shows only how many each model found. + Aggregate-only, by design: it carries counts, token totals and efficiency but NEVER + the certificates or the identities of the buggy rules. Publishing those would be a + free answer key — on a public library commit a `pred`-confirmed certificate counts + regardless of provenance, so anyone could copy it. The full certificates stay in the + private scored result the maintainer holds; the leaderboard shows only how many each + model found. """ return { "model": scored["model"], "library_commit": scored.get("library_commit", "unknown"), - "budget_cap": submission.get("budget_cap"), "bugs_found": scored["bugs_found"], "rules_tested": scored["rules_tested"], - "total_cost_usd": scored["total_cost_usd"], "total_tokens_k": scored["total_tokens_k"], - # Aggregate token totals + the price snapshot — safe to publish (no rule identities) - # and the whole point: cost stays recomputable under any future/alternative price. + # Aggregate token totals — safe to publish (no rule identities). "usage_totals": scored.get("usage_totals"), - "prices": scored.get("prices"), "efficiency_bugs_per_ktok": scored["efficiency_bugs_per_ktok"], - "efficiency_bugs_per_dollar": scored["efficiency_bugs_per_dollar"], "submitted_by": submission.get("submitted_by"), "placeholder": False, } @@ -188,7 +169,7 @@ def main() -> None: submission = json.loads(Path(args.submission).read_text(encoding="utf-8")) scored, report = score_submission(submission, args.repo_dir) - print(f"Scoring {scored['model']} (budget ${submission.get('budget_cap')})") + print(f"Scoring {scored['model']}") print("-" * 60) for item in report: flag = "✓ ACCEPTED" if item["accepted"] else "✗ rejected" diff --git a/site/README.md b/site/README.md index f9cd16c..ea3a2a9 100644 --- a/site/README.md +++ b/site/README.md @@ -1,8 +1,8 @@ # Leaderboard site Static leaderboard (`index.html`, no app server) published to **GitHub Pages** by -`.github/workflows/publish-on-merge.yml`. Every model gets the same **$20** budget; every -bug is re-verified by `pred`. +`.github/workflows/publish-on-merge.yml`. Every model gets the same step-limited agent +session; every bug is re-verified by `pred`. `index.html` reads two data files served alongside it: diff --git a/site/index.html b/site/index.html index 267b521..d5cb602 100644 --- a/site/index.html +++ b/site/index.html @@ -259,7 +259,7 @@

Leaderboard

library v0.6.0
- $20 / model + 300 steps / model 253 rules
@@ -274,7 +274,7 @@

Leaderboard

Click a row for its run stats →
-

Bugs = distinct rules with a confirmed counterexample. Budget reach = rules the $20 reached. Bugs / Ktok = efficiency tie-break.

+

Bugs = distinct rules with a confirmed counterexample. Coverage = rules the run reached. Bugs / Ktok = efficiency tie-break.

@@ -295,16 +295,16 @@

Leaderboard

From your model to the leaderboard

-

Give a model the same $20 and see how many bugs it can find in the library's problem - reductions. You run it on your own model and key; we independently re-check every bug before - it scores.

+

Give a model the same step-limited session and see how many bugs it can find in the library's + problem reductions. You run it on your own model and key; we independently re-check every bug + before it scores.

You run it
1

Get the runner

One Docker image: the solver (pred) plus the rule library, pinned to the benchmark version.

-
2

Plug in any model

OpenAI, Anthropic, DeepSeek, a self-hosted endpoint — you bring the API key and the price.

-
3

Hunt for bugs

Your model probes each rule for a counterexample until the $20 runs out; each find is checked by the solver on the spot.

a bug = solve(x) ≠ solve(reduce(x))
+
2

Plug in any model

OpenAI, Anthropic, DeepSeek, a self-hosted endpoint — you bring the API key.

+
3

Hunt for bugs

Your model probes each rule for a counterexample until its steps run out; each find is checked by the solver on the spot.

a bug = solve(x) ≠ solve(reduce(x))
4

Submit from the command line

Upload submission.json with benchmark.submit. No web form.

@@ -322,8 +322,8 @@

What counts as a bug

Each reduction turns problem A into an equivalent problem B. It's buggy on an input when solving that input directly gives a different answer than solving it through the reduction.

-

Why the $20 is fair

-

Every model gets the same budget, measured from real token usage at the price you declare. Ranking is by confirmed distinct-rule bugs, with bugs per 1K tokens as the efficiency tie-break.

+

Why it is fair

+

Every model gets the same step limit, with real token usage recorded from the run. Ranking is by confirmed distinct-rule bugs, with bugs per 1K tokens as the efficiency tie-break.

@@ -331,13 +331,13 @@

Why the $20 is fair

Submit a run

-

Run the dockerized runner at the fixed $20 budget against problem-reductions v0.6.0, then upload the submission.json it produces. Every certificate is re-verified by pred before it counts.

+

Run the dockerized runner against problem-reductions v0.6.0, then upload the submission.json it produces. Every certificate is re-verified by pred before it counts.

1

Produce submission.json

Configure once, validate with one tiny real call, then run.

- 
cp submission.env.example submission.env   # model · key · price
+        
cp submission.env.example submission.env   # model · key
 make preflight   # validate config with one tiny real call
 make run         # → ./out/submission.json
@@ -353,7 +353,7 @@

Upload it

About the benchmark

-

An open measurement of how many bugs a model can find in the problem-reductions library. Every model gets the same $20, and every claimed bug is independently re-verified server-side.

+

An open measurement of how many bugs a model can find in the problem-reductions library. Every model gets the same step-limited session, and every claimed bug is independently re-verified server-side.

How to cite

@misc{TODO_citation_key,
@@ -379,7 +379,7 @@ 
How to cite

 
 
 
diff --git a/submission.env.example b/submission.env.example
index 54c976d..95b33fa 100644
--- a/submission.env.example
+++ b/submission.env.example
@@ -19,14 +19,6 @@ API_KEY=...          # generic key — works for any provider
 # Prefer the provider's own env var? Use it INSTEAD of API_KEY, e.g.:
 #   OPENAI_API_KEY=...   ANTHROPIC_API_KEY=...   OPENROUTER_API_KEY=...   GEMINI_API_KEY=...
 
-# Price is REQUIRED — there is no built-in table (a stale default would silently
-# mis-meter the $20 cap). Spend is computed as token_usage × your price, so YOU
-# declare the rate (USD / 1M tokens). Look it up from your provider's pricing page.
-PRICE_IN=3.0         # USD / 1M input tokens
-PRICE_OUT=15.0       # USD / 1M output tokens
-# PRICE_CACHE_READ=0.30    # set these too for prompt-caching models
-# PRICE_CACHE_WRITE=3.75
-
 # ── NON-STANDARD PROVIDER (OpenRouter / gateway / local vLLM / Azure …) ───────
 # Nothing is hardcoded to anthropic/openai. Reach any model with these:
 # API_BASE=https://my-gateway.example/v1
@@ -34,17 +26,14 @@ PRICE_OUT=15.0       # USD / 1M output tokens
 # MODEL_KWARGS={"custom_llm_provider":"openai","extra_headers":{"X-Org":"acme"}}
 #   # ^ JSON of extra litellm.completion kwargs (api_version, temperature, …)
 
-# ── BUDGET (defaults are the ranked configuration — change only for smoke runs) ─
-# BUDGET_USD=20        # must be 20 to be ranked
-# PER_RULE_BUDGET=0.5  # per-rule cost cap
-# SAFETY_MARGIN=1      # USD held back so the budget-crossing call stays under cap
+# ── LIMITS (defaults are the ranked configuration — change only for smoke runs) ─
 # MAX_TOKENS=8192      # per-call output ceiling
 # MAX_RULES=5          # cap rules attempted (smoke runs only; omit for a real run)
 
 # ── AGENT MODE ────────────────────────────────────────────────────────────────
-# AGENT_MODE=per-rule  # per-rule (default): one isolated session per rule, budget split
-#                      # evenly. whole-repo: ONE session over the whole library — the agent
-#                      # enumerates and triages the rules itself under a single budget.
+# AGENT_MODE=per-rule  # per-rule (default): one isolated session per rule (35 steps each).
+#                      # whole-repo: ONE session over the whole library (300 steps) — the
+#                      # agent enumerates and triages the rules itself.
 
 # ── OUTPUT / TRAJECTORY / VERSIONING ─────────────────────────────────────────
 # OUTPUT=/out/submission.json  # stable "latest" path (what `prb submit` reads). A versioned
diff --git a/submissions/README.md b/submissions/README.md
index cc6c04a..8a61ef1 100644
--- a/submissions/README.md
+++ b/submissions/README.md
@@ -26,7 +26,6 @@ stay local; they never enter git.
 
 ## Notes
 
-- `budget_cap` must be `20` to be ranked.
 - Your self-reported `bugs_found` is advisory only; the score is the number of **distinct
   rules** with a `pred`-confirmed bug, recomputed by the backend (zero-trust).
 - Re-submitting the same model is fine — the leaderboard keeps your best run.