From ec6226ae6335691df4c602a6783d1278ea3c47d9 Mon Sep 17 00:00:00 2001 From: igerber Date: Fri, 16 Oct 2026 14:48:33 -0400 Subject: [PATCH] feat(reviewer-eval): pre-register the gpt-5.6-sol -> gpt-6-astra codex reviewer campaign PR-1 of 2: no production model change. Pre-registers the 2026-10 campaign (DECISION_RULE.md; the 2026-07 rule is archived under campaigns/), splits the grader-facing GRADING_RUBRIC.md off it, adds the reviewer_verdict.py and reviewer_audit.py engines plus run_eval.py audit/verdict, runs every harness codex call under an isolated CI-faithful CODEX_HOME, records seven-key git provenance per artifact and in the manifest, seals artifacts at run time and binds the blinded bundle, and makes review worktree leaves salted opaque hashes. Harness CLI re-pinned to codex-cli 0.162.1 (arm E = gpt-6-astra @ xhigh added), CI codex-version pinned to the same and lockstep-tested with DEFAULT_MODEL, the plan-review pin, the control arm and the prose pins. openai_review.py: call_codex parses the codex stderr session events into session_commands (+ counts, truncation flag) on every exit path and prints a summary line; --timeout caps the codex backend too (0 = none, negatives rejected); PRICING corrected for gpt-5.6-* and extended with gpt-6-*; gpt-6 is a reasoning family; blinding scrubs gpt-6/astra and Unicode-dash spellings. --- .claude/commands/ai-review-local.md | 15 +- .claude/scripts/openai_review.py | 189 ++- .claude/skills/plan-review/SKILL.md | 4 + .github/workflows/ai_pr_review.yml | 6 + .github/workflows/rust-test.yml | 20 + DEFERRED.md | 1 + TODO.md | 3 +- .../20261016-reviewer-eval-gpt6-prereg.md | 28 + tests/fixtures/README.md | 34 + tests/fixtures/codex_stderr_0_162_1.txt | 45 + .../codex_stderr_0_162_1_websearch.txt | 17 + tests/test_evals_engine.py | 14 + tests/test_evals_runtime.py | 1380 ++++++++++++++++- tests/test_openai_review.py | 370 ++++- tests/test_plan_review_skill.py | 17 + tests/test_reviewer_audit.py | 554 +++++++ tests/test_reviewer_verdict.py | 1090 +++++++++++++ tools/eval_core/compare.py | 33 +- tools/eval_core/models.py | 4 +- tools/eval_core/runner.py | 38 +- tools/reviewer-eval/DECISION_RULE.md | 351 ++++- tools/reviewer-eval/GRADING_RUBRIC.md | 198 +++ tools/reviewer-eval/README.md | 122 +- .../reviewer-eval/adapters/codex_reviewer.py | 195 ++- tools/reviewer-eval/adapters/worktree.py | 80 +- .../campaigns/2026-07-gpt56-DECISION_RULE.md | 87 ++ .../campaigns/2026-07-gpt56-VERDICT.md | 5 + tools/reviewer-eval/config/configs.json | 24 +- tools/reviewer-eval/reviewer_audit.py | 499 ++++++ tools/reviewer-eval/reviewer_verdict.py | 626 ++++++++ tools/reviewer-eval/run_eval.py | 1280 ++++++++++++++- 31 files changed, 7095 insertions(+), 234 deletions(-) create mode 100644 changelog.d/20261016-reviewer-eval-gpt6-prereg.md create mode 100644 tests/fixtures/README.md create mode 100644 tests/fixtures/codex_stderr_0_162_1.txt create mode 100644 tests/fixtures/codex_stderr_0_162_1_websearch.txt create mode 100644 tests/test_reviewer_audit.py create mode 100644 tests/test_reviewer_verdict.py create mode 100644 tools/reviewer-eval/GRADING_RUBRIC.md create mode 100644 tools/reviewer-eval/campaigns/2026-07-gpt56-DECISION_RULE.md create mode 100644 tools/reviewer-eval/reviewer_audit.py create mode 100644 tools/reviewer-eval/reviewer_verdict.py diff --git a/.claude/commands/ai-review-local.md b/.claude/commands/ai-review-local.md index 7ca427c66..7f14d284e 100644 --- a/.claude/commands/ai-review-local.md +++ b/.claude/commands/ai-review-local.md @@ -16,7 +16,7 @@ Two backends are supported: | Backend | Latency | Cost | Quality | |---|---|---|---| -| `api` (`gpt-5.6-sol`) | 30-60s | ~$0.10-1.00/run, metered via `OPENAI_API_KEY` | Single-shot — won't grep, can't load files on its own initiative | +| `api` (`gpt-5.6-sol`) | 30-60s | metered at the `PRICING` rates in `openai_review.py` via `OPENAI_API_KEY`; `--backend api --dry-run` prints the estimate for your diff | Single-shot — won't grep, can't load files on its own initiative | | `codex` (any auth) | 3-15 min | depends on your `codex login` mode (subscription vs API key) — see codex docs | Agentic — matches CI Codex reviewer, can grep / load files / multi-turn | Choose with `--backend {auto,codex,api}` (default `auto`): @@ -67,13 +67,14 @@ Notes: - `--force-fresh`: Skip delta-diff mode, run a full fresh review even if previous state exists - `--full-registry`: Include the entire REGISTRY.md instead of selective sections - `--model `: Override the model (default: `gpt-5.6-sol`). Applies to both backends. -- `--timeout `: HTTP request timeout. If omitted, defaults to 900 for reasoning models (gpt-5.4/gpt-5.5/gpt-5.6*, *-pro, o1/o3/o4) and 300 otherwise. *Api backend only.* +- `--timeout `: applies to both backends: on the api backend an HTTP request timeout (900 s default for reasoning models: gpt-5.4/gpt-5.5/gpt-5.6*/gpt-6*, *-pro, o1/o3/o4; 300 otherwise), on the codex backend a hard wall-clock cap on the codex process (uncapped when omitted). `0` means no timeout on either backend. - `--dry-run`: Print the compiled prompt without invoking the chosen backend (no API call, no codex subprocess) **Reasoning models** (`gpt-5.4-pro`, `o3`, `o4-mini`, etc.) on the api backend: Reviews may take 10-15 minutes. For deep reviews with reasoning models, combine -`--token-budget` with `--model`: +`--token-budget` with `--model` (the cost estimate models the <=272K-input +pricing tier only; a 500K budget can bill at the higher tier): ``` /ai-review-local --backend api --model gpt-5.4-pro --token-budget 500000 --context deep ``` @@ -417,8 +418,8 @@ would be silently ignored. Note: `--force-fresh` is a skill-only flag — it controls whether delta diffs are generated in Step 4 and is NOT passed to the script. -**Reasoning model handling:** If the model starts with `gpt-5.4`/`gpt-5.5`/`gpt-5.6`, contains `-pro`, or starts with -`o1`/`o3`/`o4` (e.g., `gpt-5.6-sol`, `gpt-5.4-pro`, `o3`, `o4-mini`): +**Reasoning model handling:** If the model starts with `gpt-5.4`/`gpt-5.5`/`gpt-5.6`/`gpt-6`, contains `-pro`, or starts with +`o1`/`o3`/`o4` (e.g., `gpt-5.6-sol`, `gpt-6-astra`, `gpt-5.4-pro`, `o3`, `o4-mini`): - The script auto-resolves `--timeout` to 900s for reasoning models when omitted, so no extra flag is required unless overriding - Run the Bash command with `run_in_background: true` (bypasses the 600s Bash tool timeout cap) @@ -535,7 +536,7 @@ runs `--force-fresh` or when a rebase invalidates the tracked commit. # Force the agentic codex backend (matches CI quality) /ai-review-local --backend codex -# Force the fast api backend (single-shot, $0.05-0.50/run) +# Force the fast api backend (single-shot; metered at the `PRICING` rates in openai_review.py — `--backend api --dry-run` prints the estimate for your diff) /ai-review-local --backend api # Api backend, minimal context (diff only) @@ -556,7 +557,7 @@ runs `--force-fresh` or when a rebase invalidates the tracked commit. # Different model with full registry /ai-review-local --backend api --model gpt-4.1 --full-registry -# Deep api review with reasoning model (10-15 min) +# Deep api review with reasoning model (10-15 min; the estimate is the <=272K-input tier only) /ai-review-local --backend api --model gpt-5.4-pro --token-budget 500000 --context deep ``` diff --git a/.claude/scripts/openai_review.py b/.claude/scripts/openai_review.py index 2d355b235..0bf9837f6 100644 --- a/.claude/scripts/openai_review.py +++ b/.claude/scripts/openai_review.py @@ -862,17 +862,23 @@ def apply_token_budget( # Cost estimation # --------------------------------------------------------------------------- -# Pricing per 1M tokens: (input, output) in USD. -# Source: https://platform.openai.com/docs/pricing -# MAINTENANCE: Update when OpenAI changes pricing. +# Pricing per 1M tokens: (input, output) in USD, standard tier, <=272K-input +# requests only (requests above 272K input bill at a higher tier, ~2x, which +# estimate_cost does not model). +# Source: https://developers.openai.com/api/docs/pricing +# MAINTENANCE: Update when OpenAI changes pricing. The gpt-5.6-* and gpt-6-* +# rows were verified 2026-10-15; older rows are unchanged since their last check. PRICING = { "gpt-5.4": (2.50, 15.00), "gpt-5.4-pro": (30.00, 180.00), "gpt-5.5": (5.00, 30.00), "gpt-5.5-pro": (30.00, 180.00), - "gpt-5.6-sol": (5.00, 30.00), - "gpt-5.6-terra": (2.50, 15.00), - "gpt-5.6-luna": (1.00, 6.00), + "gpt-5.6-sol": (4.00, 20.00), + "gpt-5.6-terra": (2.00, 12.00), + "gpt-5.6-luna": (0.20, 1.20), + "gpt-6-astra": (10.00, 50.00), + "gpt-6-sol": (2.00, 10.00), + "gpt-6-luna": (0.10, 0.50), "gpt-4.1": (2.00, 8.00), "gpt-4.1-mini": (0.40, 1.60), "o3": (2.00, 8.00), @@ -1145,13 +1151,16 @@ def compile_prompt( def _is_reasoning_model(model: str) -> bool: """Return True for models that use internal chain-of-thought reasoning.""" - return model.startswith(("o1", "o3", "o4", "gpt-5.4", "gpt-5.5", "gpt-5.6")) or "-pro" in model + return ( + model.startswith(("o1", "o3", "o4", "gpt-5.4", "gpt-5.5", "gpt-5.6", "gpt-6")) + or "-pro" in model + ) def _resolve_timeout(timeout: "int | None", model: str) -> int: """Auto-resolve omitted --timeout based on model class. - Reasoning models (o1/o3/o4/gpt-5.4/gpt-5.5/gpt-5.6*/*-pro) get REASONING_TIMEOUT (900s). + Reasoning models (o1/o3/o4/gpt-5.4/gpt-5.5/gpt-5.6*/gpt-6*/*-pro) get REASONING_TIMEOUT (900s). Non-reasoning models get DEFAULT_TIMEOUT (300s). Explicit values are passed through unchanged. """ @@ -1324,10 +1333,12 @@ def _detect_backend(requested: str) -> str: return "api" -# Reasoning-effort values the codex backend accepts, verified live against -# codex-cli 0.144.5 / the Responses API effort enum on 2026-07-18. `ultra` is -# deliberately excluded: it is not accepted on this API path (the enum tops out -# at `max`) and is out of scope for the reviewer either way. +# Reasoning-effort values the codex backend accepts. `xhigh` verified live on +# codex-cli 0.162.1 (2026-10); minimal/low/medium/high/max are carried over from +# the codex-cli 0.144.5 / Responses API verification of 2026-07-18 and were not +# re-verified. `ultra` was observed accepted only on codex-cli 0.153.4 via a +# ChatGPT login, is not verified on 0.162.1 nor on the API path, and is +# intentionally excluded. _CODEX_EFFORTS = frozenset({"minimal", "low", "medium", "high", "xhigh", "max"}) @@ -1369,6 +1380,43 @@ def _build_codex_cmd( ] +# Non-shell tool-event headers codex exec prints on stderr in human mode. A +# stderr line that is EXACTLY one of these (after rstrip) or matches +# OTHER_EVENT_PREFIX_RE counts toward usage["session_other_events"]; a run with a +# non-zero count used a channel the shell log cannot show, so the reviewer-eval +# session-log audit classifies it `unaudited`. Learned on codex-cli 0.162.1 +# (2026-10): web search prints `web search: ` (a PREFIX line, not a bare +# word), hence the regex; the bare-word names are the pre-registered initial set. +# Lives HERE (not in the harness adapter) because this script is stdlib-only and +# is loaded BY the adapter; the adapter folds this set into its backend-contract +# hash so a change here busts the resume cache like any call_codex edit. +OTHER_EVENT_HEADERS = frozenset( + { + "web_search", + "web search", + "spawn_agent", + "mcp", + "mcp_tool_call", + "apply_patch", + "view_image", + "browser", + "computer_use", + "tool", + } +) +OTHER_EVENT_PREFIX_RE = re.compile( + r"^(web search|web_search|mcp|mcp_tool_call|tool|spawn_agent|browser|computer_use)[ :_]" +) + + +def _non_negative_int(value: str) -> int: + """argparse type for --timeout: an int >= 0 (0 = no timeout on both backends).""" + n = int(value) + if n < 0: + raise argparse.ArgumentTypeError(f"--timeout must be >= 0, got {n}") + return n + + def call_codex( prompt: str, model: str, @@ -1393,10 +1441,74 @@ def call_codex( reporting on non-zero exit. Codex's stdout (when not using --json) is typically empty for `exec`; we forward it to user stderr just in case. - Raises RuntimeError on non-zero exit or empty output. Cleans up the - tempfile in `finally`, including on KeyboardInterrupt (which propagates + The captured stderr is ALSO parsed, on every exit path (normal, non-zero + exit, timeout) before any RuntimeError is raised, into the session-event + summary the reviewer-eval harness persists: ``session_commands`` (the shell + command lines codex executed, one entry per ``exec`` block: the + `` -lc '...' in `` start line through the line before codex's + status line ``succeeded/failed/exited N/timed out in Nms:``; an entry with + no status line is closed by the next start line, by a 200-line cap with + ``[truncated]`` appended, or by end of stream as-is), the start-line and + ``exec``-line counts, the non-shell event count (OTHER_EVENT_HEADERS / + OTHER_EVENT_PREFIX_RE) and whether the stderr drain was still alive after + the join (``session_commands_truncated``). One stderr line + ``session_commands: entries=.. start_lines=.. exec_lines=.. other_events=.. + truncated=..`` is printed so a caller can check the parser against the live + format without an artifact. The parser is INLINE (a nested function) so it + is part of the ``call_codex`` source the harness fingerprints. + + Raises RuntimeError on non-zero exit, timeout, or empty output. Cleans up + the tempfile in `finally`, including on KeyboardInterrupt (which propagates after best-effort termination). """ + _START_RE = re.compile(r"^\S*/(zsh|bash|sh) -lc ") + _STATUS_RE = re.compile( + r"^ (succeeded|failed|exited \d+|timed out)\b.* in \d+(\.\d+)?(ms|s):?\s*$" + ) + _ENTRY_CAP = 200 + + def _parse_session(text: str) -> dict: + entries: "list[str]" = [] + start_lines = 0 + exec_lines = 0 + other_events = 0 + current: "list[str] | None" = None + for raw in text.splitlines(): + line = raw.rstrip() + if line == "exec": + exec_lines += 1 + if line in OTHER_EVENT_HEADERS or OTHER_EVENT_PREFIX_RE.match(line): + other_events += 1 + if _START_RE.match(raw): + start_lines += 1 + if current is not None: + # Unterminated: closed by the next start line. The `exec` + # header that introduced THIS block is not the previous + # entry's output, so drop it from the closed entry's tail. + if len(current) > 1 and current[-1].rstrip() == "exec": + current.pop() + entries.append("\n".join(current)) + current = [raw] + continue + if current is None: + continue + if _STATUS_RE.match(raw): + entries.append("\n".join(current)) + current = None + continue + current.append(raw) + if len(current) >= _ENTRY_CAP: + entries.append("\n".join(current) + "\n[truncated]") + current = None + if current is not None: + entries.append("\n".join(current)) # end of stream: kept as-is + return { + "session_commands": entries, + "session_commands_start_lines": start_lines, + "session_commands_exec_lines": exec_lines, + "session_other_events": other_events, + } + fd, tmp_path = tempfile.mkstemp(suffix=".md", prefix="codex-review-") os.close(fd) try: @@ -1448,16 +1560,13 @@ def _feed_stdin(): stdin_thread = threading.Thread(target=_feed_stdin, daemon=True) stdin_thread.start() + timed_out = False try: proc.wait(timeout=timeout_s) except subprocess.TimeoutExpired: proc.kill() proc.wait() - stdin_thread.join(timeout=2) - raise RuntimeError( - f"codex exec timed out after {timeout_s}s (model={model}, " - f"effort={effort}):\n{stderr_buf.getvalue()}" - ) + timed_out = True except KeyboardInterrupt: proc.terminate() try: @@ -1469,6 +1578,26 @@ def _feed_stdin(): stderr_thread.join(timeout=2) stdin_thread.join(timeout=2) + # Parse + summarize on EVERY exit path, before any raise below. + session = _parse_session(stderr_buf.getvalue()) + session["session_commands_truncated"] = bool(stderr_thread.is_alive()) + print( + "session_commands: entries={} start_lines={} exec_lines={} other_events={} " + "truncated={}".format( + len(session["session_commands"]), + session["session_commands_start_lines"], + session["session_commands_exec_lines"], + session["session_other_events"], + session["session_commands_truncated"], + ), + file=sys.stderr, + ) + if timed_out: + raise RuntimeError( + f"codex exec timed out after {timeout_s}s (model={model}, " + f"effort={effort}):\n{stderr_buf.getvalue()}" + ) + if proc.returncode != 0: raise RuntimeError( f"codex exec failed (exit {proc.returncode}):\n" f"{stderr_buf.getvalue()}" @@ -1479,11 +1608,13 @@ def _feed_stdin(): ) with open(tmp_path) as f: content = f.read() - return content, { + usage: dict = { "backend": "codex", "input_tokens": None, "output_tokens": None, } + usage.update(session) + return content, usage finally: try: os.unlink(tmp_path) @@ -1555,7 +1686,8 @@ def call_openai( ) try: - with urllib.request.urlopen(req, timeout=timeout) as resp: + # 0 is the documented "no timeout" value; urlopen(timeout=0) is NOT that. + with urllib.request.urlopen(req, timeout=(None if timeout == 0 else timeout)) as resp: result = json.loads(resp.read().decode("utf-8")) except urllib.error.HTTPError as e: body = "" @@ -1744,13 +1876,15 @@ def main() -> None: ) parser.add_argument( "--timeout", - type=int, + type=_non_negative_int, default=None, help=( - f"HTTP request timeout in seconds (api backend only). If omitted, " - f"defaults to {REASONING_TIMEOUT} for reasoning models (gpt-5.4/gpt-5.5/gpt-5.6*, " - f"*-pro, o1/o3/o4) and {DEFAULT_TIMEOUT} otherwise. Ignored under " - f"--backend codex (codex manages its own time budget)." + f"Timeout in seconds; applies to both backends. On the api backend an " + f"HTTP request timeout: if omitted, defaults to {REASONING_TIMEOUT} for " + f"reasoning models (gpt-5.4/gpt-5.5/gpt-5.6*/gpt-6*, *-pro, o1/o3/o4) and " + f"{DEFAULT_TIMEOUT} otherwise. On the codex backend a hard wall-clock cap " + f"on the codex process (uncapped when omitted). 0 means no timeout on " + f"either backend; negative values are rejected." ), ) parser.add_argument( @@ -2100,10 +2234,13 @@ def main() -> None: # See the SENSITIVE_FILE_PATTERNS comment block for the rationale on # why this is a notice and not an enforcement gate. _print_sensitive_notice(codex_repo_root, _scan_sensitive_files(codex_repo_root)) + # 0 means "no timeout" on both backends: proc.wait(timeout=0) would + # kill codex at once, so 0 (and an omitted flag) map to None here. review_content, usage = call_codex( prompt=prompt, model=args.model, repo_root=codex_repo_root, + timeout_s=(None if not args.timeout else args.timeout), ) else: review_content, usage = call_openai(prompt, args.model, api_key, timeout=args.timeout) diff --git a/.claude/skills/plan-review/SKILL.md b/.claude/skills/plan-review/SKILL.md index 57bd442a6..6cdd86fb9 100644 --- a/.claude/skills/plan-review/SKILL.md +++ b/.claude/skills/plan-review/SKILL.md @@ -153,6 +153,10 @@ deliberate-single note. > session; treat this as the accepted, pre-existing codex surface, not a > plan-review-specific gate. +Run this Bash call with `run_in_background: true` and act on its completion +notification by following the exit-code bullets below; the Bash tool caps +foreground calls at 600 s, below the codex ceiling. + ```bash python3 ".claude/skills/plan-review/codex_review.py" \ --prompt-file "/reviewer_prompt.txt" \ diff --git a/.github/workflows/ai_pr_review.yml b/.github/workflows/ai_pr_review.yml index 34ee6edea..836d224cb 100644 --- a/.github/workflows/ai_pr_review.yml +++ b/.github/workflows/ai_pr_review.yml @@ -523,6 +523,12 @@ jobs: prompt-file: .github/codex/prompts/pr_review_compiled.md sandbox: read-only safety-strategy: drop-sudo + # Pinned so the campaign CLI == the CI CLI (tools/reviewer-eval + # config/configs.json `cli_version`). The action installs BOTH + # @openai/codex and @openai/codex-responses-api-proxy at this version, + # so bump only to a version that exists for both, together with the + # configs.json pin + a harness smoke (the lockstep test enforces equality). + codex-version: 0.162.1 # Recommended by OpenAI for review quality/consistency: model: gpt-5.6-sol effort: xhigh diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 985afc311..a751e9279 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -38,6 +38,11 @@ on: - '.github/workflows/ai_pr_review.yml' - '.github/codex/prompts/pr_review.md' - '.claude/scripts/openai_review.py' + # Production model-pin lockstep surfaces read by tests/test_openai_review.py + # (a skill-only, command-doc-only or CLAUDE.md-only pin edit must run it). + - '.claude/skills/plan-review/**' + - '.claude/commands/ai-review-local.md' + - 'CLAUDE.md' pull_request: branches: [main] types: [opened, synchronize, reopened, labeled, unlabeled] @@ -68,6 +73,11 @@ on: - '.github/workflows/ai_pr_review.yml' - '.github/codex/prompts/pr_review.md' - '.claude/scripts/openai_review.py' + # Production model-pin lockstep surfaces read by tests/test_openai_review.py + # (a skill-only, command-doc-only or CLAUDE.md-only pin edit must run it). + - '.claude/skills/plan-review/**' + - '.claude/commands/ai-review-local.md' + - 'CLAUDE.md' permissions: contents: read @@ -199,6 +209,16 @@ jobs: python -c "import diff_diff; print('Location:', diff_diff.__file__)" python -c "from diff_diff import HAS_RUST_BACKEND; print('HAS_RUST_BACKEND:', HAS_RUST_BACKEND); assert HAS_RUST_BACKEND, 'Rust backend not available'" + - name: Run eval-harness suites in the checkout (py3.9 floor) + # The matrix copies tests/ to /tmp and runs there, so the harness suites + # (which need tools/, .claude/scripts, .github/workflows and .git) skip + # on every leg; this is the floor's only import+run of those modules. + # -rs prints skip reasons so a silent skip is visible in the log. + if: matrix.python-version == '3.9' + env: + PYTHONPATH: . + run: pytest tests/test_openai_review.py tests/test_evals_runtime.py tests/test_evals_engine.py tests/test_evals_adapters.py tests/test_reviewer_verdict.py tests/test_reviewer_audit.py tests/test_plan_review_eval.py tests/test_plan_review_skill.py -q -rs + - name: Copy tests to isolated location (Unix) if: runner.os != 'Windows' run: cp -r tests /tmp/tests diff --git a/DEFERRED.md b/DEFERRED.md index 98ac11f5c..b39b7b4f4 100644 --- a/DEFERRED.md +++ b/DEFERRED.md @@ -123,6 +123,7 @@ For survey-specific limitations (`NotImplementedError` paths), see the | `HeterogeneousAdoptionDiD` `covariates=` (Theorem 6 multivariate-covariate extension) not implemented — `fit(covariates=...)` raises `NotImplementedError` via the shipped future-work trap (locked by the `test_had.py` / `test_methodology_had.py` L73 tests); the deferred work is the Theorem 6 extension itself. | `had.py` | Phase 2a | Low | | MultiPeriodDiD wild bootstrap not supported (falls back to analytical, n_bootstrap-independent) — user-facing edge-case limitation; the 4.0 removal replaces the fallback with a raise (v4-design §4.1), and the merged TWFE event-study mode already RAISES since 3.9 (Phase 3(a)) — this row now governs only the deprecated class itself. | `estimators.py` (MultiPeriodDiD.fit wild-fallback block) | — | Low | | `predict()` raises `NotImplementedError` — rarely needed; user-facing limitation. | `estimators.py:890-911` | — | Low | +| `gpt-6-sol` cost-fallback probe for the Codex reviewer: a ChatGPT-account `codex login` rejects `gpt-6-sol` (400, "not supported when using Codex with a ChatGPT account"), so the local harness cannot run a cheaper-tier arm; needs an API-key codex login (metered) before an arm can be added. | `tools/reviewer-eval/config/configs.json` | #849 | Low | ## Version-gated (v4) diff --git a/TODO.md b/TODO.md index 2b01ad53f..e04d90bce 100644 --- a/TODO.md +++ b/TODO.md @@ -101,8 +101,9 @@ generic sparse-FE, QR+SVD rank-detection redundancy, `check_finite` bypass — m | Codex reviewer isolation (repo-wide): `codex --sandbox read-only` blocks writes but does NOT confine READS to the worktree/repo (verified: a probe read `/etc/hosts`), and prompt guards are not a security boundary. Affects `/ai-review-local` (which already documents and accepts this surface) and plan-review-eval dual arms equally. Evaluate OS-level isolation for codex invocations (container / `sandbox-exec` profile / dedicated low-privilege account exposing only the worktree). | `.claude/scripts/openai_review.py`, `tools/plan-review-eval/` | plan-review-eval local review R6 | Heavy | Medium | | plan-review-eval dual arms: terminate the surviving peer subprocess when the first reviewer of a dual pair fails (today an early Claude failure still waits out the codex peer — up to its 3600s ceiling — before the run becomes INFRA_ERROR; needs process handles threaded through `_call_claude`/`call_codex` or a shared cancellation event). | `tools/plan-review-eval/plan_adapters/plan_reviewer.py` | plan-review-eval local review R3 | Mid | Low | | Reimplement the retired `/review-plan` `--updated` (delta re-review, `## Delta Assessment` section) and `--pr ` (PR-comment feedback-coverage) modes in the plan-review skill — descoped from the initial skill ship (the advertised-but-unimplemented flags were removed). `--updated` needs prior-review load + a delta diff; `--pr` needs a `gh` comment fetch + a new reviewer-prompt token. | `.claude/skills/plan-review/SKILL.md` | plan-review skill local review | Mid | Low | -| plan-review skill dual-mode peer cancellation: when reviewer 1 (Opus subagent) fails, terminate the concurrently-running codex `codex_review.py` subprocess instead of waiting out its 1200s ceiling (and vice-versa). Interactive analogue of the eval-harness peer-cancellation row above; needs the codex process handle surfaced from `codex_review.py` so the skill can signal it. | `.claude/skills/plan-review/SKILL.md`, `.claude/skills/plan-review/codex_review.py` | plan-review skill local review R2 | Mid | Low | +| plan-review skill dual-mode peer cancellation: when reviewer 1 (Opus subagent) fails, terminate the concurrently-running codex `codex_review.py` subprocess instead of waiting out its 2400s ceiling (and vice-versa). Interactive analogue of the eval-harness peer-cancellation row above; needs the codex process handle surfaced from `codex_review.py` so the skill can signal it. | `.claude/skills/plan-review/SKILL.md`, `.claude/skills/plan-review/codex_review.py` | plan-review skill local review R2 | Mid | Low | | Clean-negative precision re-run: Campaign 1's `s3_negative` plans were base_sha-contaminated (real defects at the pinned base), so hallucination rate cleared (dual 3.4%) but trivia-flooding on a *genuinely* clean plan is unmeasured. Small future run (~30 s3-style reviews on 3-5 constructed-clean plans, not the full 120-matrix). | `tools/plan-review-eval/` | campaign-1 verdict | Mid | Low | | `trop-silent-drop` criteria regression: the rewritten `criteria.md` dropped a silent-failure catch the OLD criteria had (arm A caught it, B/C missed — campaign-1 A-vs-B contrast). Patch the criteria + re-validate (re-opens the criteria identity, so gated behind a re-validation run). | `.claude/skills/plan-review/criteria.md`, `tools/plan-review-eval/` | campaign-1 verdict | Mid | Medium | | The Stata `jwdid` parity arms cover BALANCED panels only. `jwdid_alltreated` pins the all-eventually-treated cell set, `N` and SE ratio on the `mpdta` subset (191 units, 955 rows, 764 estimated), but every arm is a balanced frame, so REGISTRY's parity claim is scoped to that case. The comparison-support predicate's UNBALANCED behavior -- where `G_max` may be unobserved at later periods so the closed form `t < G_max - anticipation` does not hold and support is set by whichever cohorts are actually observed -- is verified against the predicate directly, not against Stata. Add an unbalanced `jwdid` arm to close that gap. | `benchmarks/stata/generate_etwfe_cs_golden.do`, `tests/test_etwfe_cs_stata_parity.py` | #729-followup | Mid | Low | | DurationDiD tutorial notebook (Appendix C-style simulation showing the sign reversal of standard DiD on an absorbing outcome, CD vs PH, the fitting window, and the pretest), registered in `docs/tutorials/index.rst` with a drift test; deferred from the estimator PR, which relies on the executed API-page examples | `docs/tutorials/` | #822 | Mid | Low | +| Answer-key isolation for reviewer-eval corpus worktrees: after PR-1's opaque worktree leaves, the remaining channels are the shared object DB and refs (incl. the dangling synthetic commits `verify-corpus` and the run pre-flight leave behind), the enclosing checkout's corpus and run artifacts, the synthetic HEAD commit message (names the bug for S1), other cases' corpus files committed inside 9 of the 11 case trees themselves, and the committed, redacted stderr fixtures `tests/fixtures/codex_stderr_0_162_1*.txt` (command lines of a control review of the smoke case; no prompt, output or review text). Fix = a clone at the case sha OUTSIDE the checkout with a neutral commit message PLUS removal of `tools/reviewer-eval/corpus/` from the materialized tree. Cross-links the "Codex reviewer isolation (repo-wide)" row above. | `tools/reviewer-eval/adapters/worktree.py` | #849 | Mid | Medium | diff --git a/changelog.d/20261016-reviewer-eval-gpt6-prereg.md b/changelog.d/20261016-reviewer-eval-gpt6-prereg.md new file mode 100644 index 000000000..6538171ac --- /dev/null +++ b/changelog.d/20261016-reviewer-eval-gpt6-prereg.md @@ -0,0 +1,28 @@ +### Internal +- **Codex reviewer gpt-6 campaign pre-registration (PR-1 of 2)**: `tools/reviewer-eval` + pre-registers the `gpt-5.6-sol` -> `gpt-6-astra` evaluation (`DECISION_RULE.md`: + symmetric regression gate over every ground-truth bug, FP gate, downgrade rule, + verifier-confirmed leaks void a cell and contest the gate, stop conditions; the + 2026-07 rule is archived under `campaigns/`) and splits the grader-facing text into + `GRADING_RUBRIC.md` (marker-based `LEAK?` flag whose list renders from + `reviewer_audit.LEAK_RULES`). New engines: `reviewer_verdict.py` (gates, precedence, + binding to the sealed artifacts, protocol guards, PROVISIONAL/DEFINITIVE) and + `reviewer_audit.py` (post-unblinding session-log audit, informational and + over-matching by design). `run_eval.py` gains `audit` and `verdict`, runs every + codex call under an isolated CI-faithful `CODEX_HOME`, records seven-key git + provenance per artifact and in the manifest, seals artifacts at run time + (`artifacts_sha`), binds the blinded bundle (`bundle_id`, `run_ids_sha`, + rubric/prompt sha) and refuses post-run stages on a dirty or moved tree; `run` + holds an exclusive per-subdir lock for the whole attempt, records the OK cells of + aborted attempts too and refuses a rerun that would re-dispatch any cell an + earlier attempt recorded OK; review worktree leaves are salted opaque hashes. Harness CLI re-pinned to + codex-cli 0.162.1 (arm `E` = gpt-6-astra @ xhigh added; CI `codex-version` pinned + to the same, lockstep-tested with `DEFAULT_MODEL`, the plan-review pin, the + control arm and the prose pins). +- **`openai_review.py`**: `call_codex` parses codex's stderr session events into + `session_commands` (+ start/exec/other-event counts, truncation flag) on every + exit path and prints a one-line summary; `--timeout` now also caps the codex + backend's wall clock when passed explicitly (previously ignored there), `0` + means no timeout on both backends and negatives are rejected; `PRICING` + corrected for gpt-5.6-* and extended with gpt-6-*; gpt-6* counts as a + reasoning family; blinding scrubs gpt-6/astra and Unicode-dash spellings. diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md new file mode 100644 index 000000000..fed6ee02c --- /dev/null +++ b/tests/fixtures/README.md @@ -0,0 +1,34 @@ +# Test fixtures + +## `codex_stderr_0_162_1*.txt` - sanitized codex exec stderr transcripts + +Event-structure transcripts of `codex exec` (codex-cli 0.162.1) used by +`tests/test_openai_review.py` to pin the `call_codex` session parser against +the live stderr format: + +- `codex_stderr_0_162_1.txt` - the 2026-10 control-arm harness smoke + (`run_eval.py smoke`, gpt-5.6-sol @ xhigh, isolated `CODEX_HOME`): 8 shell + commands, no non-shell event. +- `codex_stderr_0_162_1_websearch.txt` - the step-0 web-search probe + (gpt-6-astra @ xhigh, prompt asking for a web search then `echo ok`, empty + `--cd` dir): codex printed `web search: ` PREFIX lines, so the + `session_other_events` counter uses `OTHER_EVENT_PREFIX_RE` as well as the + bare-word `OTHER_EVENT_HEADERS` set. + +A committed transcript keeps ONLY the event structure: every event-header line +(`codex`, `exec`, `user`, `thinking`, `tokens used`, `web search: ...`, any +other bare-word header), every `exec` START line (` -lc '...' in `, +command text kept so the parser test is real) and every STATUS line +(` succeeded in Nms:` etc.). Every other run of consecutive lines is collapsed +into ONE placeholder named by the header that precedes it: `` +after `user` (the prompt echo carries the case diff), `` after +`codex`/`thinking` (review prose), `` after a status line +(command output), `` otherwise (harness logging, token counts). +Absolute paths in the kept lines are rewritten to ``. Line counts +therefore differ from the raw transcript; header, start and status counts do +not, which is all the fixture tests assert. The sanitizer is a one-off snippet +run in the authoring session (not committed code); re-create a fixture by +capturing `2> ` from a real run and applying the same rules. + +There is deliberately NO 0.153.4 transcript: the format was first observed on +that CLI, but the live pin is 0.162.1 and the older CLI is no longer installed. diff --git a/tests/fixtures/codex_stderr_0_162_1.txt b/tests/fixtures/codex_stderr_0_162_1.txt new file mode 100644 index 000000000..848274185 --- /dev/null +++ b/tests/fixtures/codex_stderr_0_162_1.txt @@ -0,0 +1,45 @@ + +user + +codex + +exec +/bin/zsh -lc "sed -n '1,240p' docs/methodology/REGISTRY.md && sed -n '1500,1665p' diff_diff/estimators.py && rg -n \"MultiPeriodDiD|fixed_effects|absorb|coef_dict|var_names\" tests diff_diff docs TODO.md DEFERRED.md changelog.d 2>/dev/null" in + exited 2 in 144ms: + +codex + +exec +/bin/zsh -lc "git diff -- diff_diff/estimators.py && nl -ba diff_diff/estimators.py | sed -n '1410,1785p' && nl -ba tests/test_estimators_vcov_type.py | sed -n '1190,1380p' && rg -n \"skip.*time|fe == time|duplicate var_names|coefficients.*vcov|mpd_absorbed\" docs/methodology/REGISTRY.md TODO.md DEFERRED.md tests diff_diff" in + exited 2 in 1813ms: + +exec +/bin/zsh -lc "nl -ba diff_diff/estimators.py | sed -n '1775,1985p'; nl -ba tests/test_estimators_vcov_type.py | sed -n '1208,1378p'; rg -n \"class MultiPeriodDiD|def fit\\(\" diff_diff/estimators.py | head -20; rg -n \"fixed_effects.*time|time.*fixed_effects|MultiPeriodDiD\\(.*fixed_effects|fixed_effects=.*period\" tests/test_estimators.py tests/test_estimators_vcov_type.py tests -g '*.py'" in + succeeded in 0ms: + +exec +/bin/zsh -lc 'PYTHONDONTWRITEBYTECODE=1 pytest -q -p no:cacheprovider tests/test_estimators_vcov_type.py::TestMultiPeriodDiDAbsorbedFERParity::test_absorb_hc2_result_surface_invariants_multi_absorb' in + exited 1 in 326ms: + +codex + +exec +/bin/zsh -lc "rg -n \"\\.coefficients|coefficients\\[|list\\(.*coefficients|vcov.*coeff|coef.*vcov|interaction_indices\" diff_diff tests -g '*.py' | rg \"MultiPeriod|multi|Honest|honest|Business|report|coefficients|interaction_indices\"; nl -ba diff_diff/estimators.py | sed -n '1960,2025p'; sed -n '1,180p' changelog.d/README.md; git status --short; git diff --name-only; rg -n \"MultiPeriodDiDResults|interaction_indices\" diff_diff -g '*.py'" in + succeeded in 3189ms: + +exec +/bin/zsh -lc "find . -maxdepth 3 -iname '*changelog*' -o -iname 'news.d' | sort; nl -ba docs/methodology/REGISTRY.md | sed -n '2538,2562p'; nl -ba diff_diff/results.py | sed -n '589,710p'; nl -ba diff_diff/honest_did.py | sed -n '620,665p'; nl -ba diff_diff/pretrends.py | sed -n '575,610p'; nl -ba diff_diff/diagnostic_report.py | sed -n '1250,1290p'; nl -ba tests/test_estimators.py | sed -n '2124,2155p'" in + succeeded in 0ms: + +exec +/bin/zsh -lc "rg -n \"def _detect_rank_deficiency|column_names|rank_deficient_action|dropped\" diff_diff/linalg.py | head -80; nl -ba diff_diff/linalg.py | sed -n '420,600p'; nl -ba diff_diff/linalg.py | sed -n '700,875p'; nl -ba CHANGELOG.md | sed -n '1,60p'; rg -n \"changelog\\.d|fragment\" pyproject.toml .github scripts docs README.md CONTRIBUTING.md 2>/dev/null | head -100" in + succeeded in 0ms: + +exec +/bin/zsh -lc "PYTHONDONTWRITEBYTECODE=1 python -c 'import json,pandas as pd; from diff_diff import MultiPeriodDiD; d=json.load(open(\"benchmarks/data/clubsandwich_cr2_golden.json\"))[\"mpd_absorbed_fe_did\"]; data=pd.DataFrame({\"unit\":d[\"unit\"],\"period\":d[\"period\"],\"treated\":d[\"treated\"],\"y\":d[\"y\"]}); target=int(d[\"target_period\"]); [(lambda r,k: print(k,\"ncoef\",len(r.coefficients),\"vcov\",r.vcov.shape,\"period\",target,r.coefficients.get(f\"period_{target}\"),\"effect\",r.period_effects[target].effect,\"se\",r.period_effects[target].se))(MultiPeriodDiD(vcov_type=\"hc2\").fit(data,outcome=\"y\",treatment=\"treated\",time=\"period\",reference_period=int(d[\"reference_period\"]),unit=\"unit\",**{k:[\"unit\",\"period\"]}),k) for k in (\"absorb\",\"fixed_effects\")]'" in + succeeded in 1120ms: + +codex + +tokens used + diff --git a/tests/fixtures/codex_stderr_0_162_1_websearch.txt b/tests/fixtures/codex_stderr_0_162_1_websearch.txt new file mode 100644 index 000000000..d7fd96875 --- /dev/null +++ b/tests/fixtures/codex_stderr_0_162_1_websearch.txt @@ -0,0 +1,17 @@ + +user + +codex + +web search: +web search: https://example.com +codex + +exec +/bin/zsh -lc 'echo ok' in + succeeded in 0ms: + +codex + +tokens used + diff --git a/tests/test_evals_engine.py b/tests/test_evals_engine.py index 6a605fa14..6ac0703b6 100644 --- a/tests/test_evals_engine.py +++ b/tests/test_evals_engine.py @@ -225,6 +225,20 @@ def test_sanitize_model_refs_scrubs_names_tiers_and_versions(): assert " 5.5 " not in f" {out} " +def test_sanitize_model_refs_scrubs_gpt6_family_and_unicode_dashes(): + from eval_core.compare import sanitize_model_refs + + text = ( + "GPT-6 Astra beats gpt-6-astra and gpt-6-astra-mini; Astra said so. " + "GPT\u20116 and GPT\u20136 too. Version 6.2 of the library is fine." + ) + out = sanitize_model_refs(text, ["gpt-6-astra", "gpt-5.6-sol"]) + low = out.lower() + assert "gpt" not in low and "astra" not in low + assert "6.2" in out, "no bare 6.x numeric pattern (only the 5.x one is grandfathered)" + assert "[model-redacted]" in out + + def test_sanitize_preserves_claude_paths_and_prose(): """Claude redaction is MODEL-shaped only: repository paths (`.claude/...`, `CLAUDE.md`) and generic prose ("the Claude workflow") must survive intact diff --git a/tests/test_evals_runtime.py b/tests/test_evals_runtime.py index c9e9d2fa5..660078946 100644 --- a/tests/test_evals_runtime.py +++ b/tests/test_evals_runtime.py @@ -10,7 +10,9 @@ import functools import importlib.util +import os import pathlib +import re import sys import pytest @@ -65,6 +67,82 @@ def _pinned_cli_version(): return raw["arms"][0]["cli_version"] +_STUB_PROVENANCE = { + "repo_head_sha": "feedface", + "repo_tree_sha": "cafebabe", + "repo_dirty": False, + "repo_branch": "test-branch", + "repo_common_dir": "/tmp/x/.git", + "repo_worktree_paths": [], + "repo_ref_names": ["test-branch", "origin/main"], +} + + +class _FakeIsolatedHome: + """Test double for ``isolated_codex_home()``: no credential copy, but the + SAME observable contract (sets ``ACTIVE_ISOLATED_HOME`` + ``CODEX_HOME`` to a + tmp dir while active, restores both on exit, counts entries).""" + + def __init__(self, tmp_path): + self.home = str(tmp_path / "fake-codex-home") + self.entries = 0 + self._prev = None + self._prev_active = None + + def __call__(self): + return self + + def __enter__(self): + import os as _os + + from adapters import codex_reviewer as cr + + _os.makedirs(self.home, exist_ok=True) + self.entries += 1 + self._prev = _os.environ.get("CODEX_HOME") + self._prev_active = cr.ACTIVE_ISOLATED_HOME + _os.environ["CODEX_HOME"] = self.home + cr.ACTIVE_ISOLATED_HOME = self.home + return self.home + + def __exit__(self, *exc): + import os as _os + + from adapters import codex_reviewer as cr + + if self._prev is None: + _os.environ.pop("CODEX_HOME", None) + else: + _os.environ["CODEX_HOME"] = self._prev + cr.ACTIVE_ISOLATED_HOME = self._prev_active + return False + + +def _stub_run_git(monkeypatch, tmp_path): + """Keep cmd_run/cmd_smoke tests off the dev/CI checkout's git state and off + ``~/.codex/auth.json``: stub the manifest provenance (fixed seven keys), the + hidden-input guard (no-op) and the isolated codex home (``_FakeIsolatedHome``). + Returns the fake home so a test can assert it was entered.""" + run_eval = _run_eval() + fake = _FakeIsolatedHome(tmp_path) + monkeypatch.setattr(run_eval, "_manifest_provenance", lambda repo_root: dict(_STUB_PROVENANCE)) + monkeypatch.setattr(run_eval, "_assert_no_hidden_inputs", lambda repo_root: None) + monkeypatch.setattr(run_eval, "isolated_codex_home", fake) + # The frozen-ref helpers would touch the dev checkout's refs: stub them with + # an in-memory ref keyed by contrast. + refs: dict = {} + monkeypatch.setattr( + run_eval, "_read_frozen_ref", lambda root, contrast: refs.get(tuple(contrast)) + ) + monkeypatch.setattr( + run_eval, + "_anchor_frozen_ref", + lambda root, contrast, head: refs.setdefault(tuple(contrast), head), + ) + fake.refs = refs + return fake + + def _make_reviewer(monkeypatch, review_md="## Overall Assessment\n✅ Looks good\n"): """A CodexReviewer with call_codex + worktree stubbed (no codex, no git).""" from adapters import codex_reviewer as cr @@ -92,6 +170,11 @@ def _make_reviewer(monkeypatch, review_md="## Overall Assessment\n✅ Looks good raising=True, ) monkeypatch.setattr(worktree, "cleanup", lambda *a, **k: None, raising=True) + # review() samples git provenance through the module; keep the helper's + # "no codex, no git" contract by stubbing it as one unit. + monkeypatch.setattr( + worktree, "git_provenance", lambda root: dict(_STUB_PROVENANCE), raising=True + ) return r @@ -450,6 +533,7 @@ def test_compare_renders_from_run_snapshot_not_corpus(tmp_path, monkeypatch): def test_run_rejects_unknown_configs(tmp_path, monkeypatch): + _stub_run_git(monkeypatch, tmp_path) run_eval = _run_eval() monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) @@ -505,6 +589,7 @@ def test_case_tag_changes_with_scoring_metadata(): def test_run_and_smoke_fail_closed_on_empty_corpus(tmp_path, monkeypatch): """run/smoke must NOT report success (or write a manifest) on zero selected cases.""" + _stub_run_git(monkeypatch, tmp_path) run_eval = _run_eval() monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) @@ -685,6 +770,7 @@ def test_run_fails_closed_on_infra_error(tmp_path, monkeypatch): """cmd_run must exit non-zero and write a FAILURE-MARKER manifest (never a valid run_ids manifest) when any run hits INFRA_ERROR, so `compare` can't present a partial run as a valid A/B.""" + _stub_run_git(monkeypatch, tmp_path) import json as _json run_eval = _run_eval() @@ -711,6 +797,7 @@ def test_run_fails_closed_on_infra_error(tmp_path, monkeypatch): def test_smoke_cli_default_limits_to_one_case(tmp_path, monkeypatch): + _stub_run_git(monkeypatch, tmp_path) run_eval = _run_eval() from eval_core.models import STRATUM_SYNTHETIC, Case @@ -795,6 +882,7 @@ def test_failed_rerun_invalidates_stale_manifest(tmp_path, monkeypatch): """A successful run writes a manifest; a later FAILED rerun into the same subdir must invalidate it (mark failed) so `compare` refuses instead of rendering the stale experiment.""" + _stub_run_git(monkeypatch, tmp_path) import json as _json run_eval = _run_eval() @@ -854,6 +942,7 @@ def test_run_aborts_on_invalid_case_before_any_codex_call(tmp_path, monkeypatch) """smoke/run must fail closed on a CorpusLoader.verify() failure BEFORE any Codex call, so a stale/malformed case is never reviewed/graded against stale ground truth.""" + _stub_run_git(monkeypatch, tmp_path) run_eval = _run_eval() from eval_core.models import STRATUM_SYNTHETIC, Case @@ -1028,6 +1117,7 @@ def test_run_early_abort_writes_failure_marker(tmp_path, monkeypatch): """An early abort (e.g. ConfoundMismatch) after the manifest is removed must leave a {failed:true} marker — NOT a missing manifest, which `compare` would treat as 'no manifest -> compare ALL' over the prior run's stale artifacts.""" + _stub_run_git(monkeypatch, tmp_path) import json as _json run_eval = _run_eval() @@ -1205,6 +1295,7 @@ def test_run_matrix_fails_closed_when_pinned_cli_unavailable(monkeypatch): def test_run_and_compare_reject_subdir_traversal(tmp_path, monkeypatch): """--subdir flows into filesystem paths; a `..` traversal must be rejected so the harness can't read/write outside runs/.""" + _stub_run_git(monkeypatch, tmp_path) run_eval = _run_eval() monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) @@ -1229,6 +1320,7 @@ def test_resolve_configs_rejects_duplicate_ids(): def test_run_marks_failed_on_corpus_load_error(tmp_path, monkeypatch): """A corpus-load exception (e.g. the duplicate-id guard) must leave a failure marker so compare refuses the stale subdir — not leave a prior manifest live.""" + _stub_run_git(monkeypatch, tmp_path) import json as _json run_eval = _run_eval() @@ -1255,6 +1347,7 @@ def _boom(self, strata): def test_run_bad_configs_invalidates_existing_manifest(tmp_path, monkeypatch): """A run attempt that exits at input validation (bad --configs) must still invalidate a PRIOR successful manifest, so compare doesn't render the stale one.""" + _stub_run_git(monkeypatch, tmp_path) import json as _json run_eval = _run_eval() @@ -1535,6 +1628,7 @@ def _rr(rid, marker): def test_smoke_clears_cache_for_live_run(tmp_path, monkeypatch): """smoke is a live plumbing check, so it must clear cached artifacts and actually exercise codex rather than resume a stale success.""" + _stub_run_git(monkeypatch, tmp_path) run_eval = _run_eval() from eval_core.models import STRATUM_SYNTHETIC, Case @@ -1701,8 +1795,9 @@ def _arm(id_, model="m", effort="xhigh", role=None, **kw): def test_make_configs_resolves_four_arms_in_order(): - """The live configs.json defines the 4-arm gpt-5.6 matrix; ids resolve in the - requested order and unknown ids still fail closed.""" + """The live configs.json still carries the historical 2026-07 A-D subset (the + 4-arm gpt-5.6 matrix); ids resolve in the requested order and unknown ids still + fail closed.""" run_eval = _run_eval() cfgs = run_eval._make_configs(["A", "B", "C", "D"]) @@ -1936,6 +2031,7 @@ def test_parse_k_per_fail_closed(): def test_cmd_run_k_per_end_to_end(tmp_path, monkeypatch): """cmd_run with --k 2 --k-per B=1 must execute A twice and B once per case, record k/k_per in the manifest, and refuse a bad --k-per up front.""" + _stub_run_git(monkeypatch, tmp_path) import json as _json run_eval = _run_eval() @@ -2012,6 +2108,7 @@ def review(self, case, config, repeat_idx): def _run_ok_experiment(tmp_path, monkeypatch, subdir="blind"): """Drive a real cmd_run (stubbed reviewer) so cmd_compare sees a valid manifest; returns the runs dir.""" + _stub_run_git(monkeypatch, tmp_path) run_eval = _run_eval() from eval_core.models import STRATUM_SYNTHETIC, Case, ReviewOutput @@ -2057,6 +2154,7 @@ def test_cmd_compare_blinded_writes_sealed_bundle(tmp_path, monkeypatch): import json as _json run_eval = _run_eval() + monkeypatch.setattr(run_eval, "_assert_tree_identity", lambda *a, **k: None) runs_dir = _run_ok_experiment(tmp_path, monkeypatch) rc = run_eval.cmd_compare(_ns(subdir="blind", allow_mixed=False, blinded=True)) @@ -2082,6 +2180,7 @@ def test_cmd_compare_blinded_mapping_stable_across_rerenders(tmp_path, monkeypat import json as _json run_eval = _run_eval() + monkeypatch.setattr(run_eval, "_assert_tree_identity", lambda *a, **k: None) runs_dir = _run_ok_experiment(tmp_path, monkeypatch, subdir="stable") assert run_eval.cmd_compare(_ns(subdir="stable", allow_mixed=False, blinded=True)) == 0 @@ -2095,6 +2194,7 @@ def test_cmd_compare_blinded_refusals(tmp_path, monkeypatch): """--blinded is manifest-scoped by construction: refuse --allow-mixed and refuse when the manifest is missing.""" run_eval = _run_eval() + monkeypatch.setattr(run_eval, "_assert_tree_identity", lambda *a, **k: None) runs_dir = _run_ok_experiment(tmp_path, monkeypatch, subdir="refuse") assert ( @@ -2128,6 +2228,7 @@ def test_smoke_default_selects_control_arm(tmp_path, monkeypatch): """Bare `smoke` (no --configs) must exercise the sole role=control arm from configs.json - a control flip must never leave the default smoking a stale arm (local review P2 on the gpt-5.6 swap).""" + _stub_run_git(monkeypatch, tmp_path) import json as _json run_eval = _run_eval() @@ -2162,3 +2263,1278 @@ def _capture_run_matrix(cases, configs, *a, **k): control = [c["id"] for c in live["arms"] if c.get("role") == "control"] assert captured["config_ids"] == control, "bare smoke must run exactly the control arm" assert run_eval._control_id() == control[0] + + +# --------------------------------------------------------------------------- # +# 2026-10 gpt-6 campaign: configs, provenance, isolated home, sealing, guards. +# --------------------------------------------------------------------------- # + + +def _git(repo, *args): + import subprocess + + return subprocess.run( + ["git", "-c", "user.name=t", "-c", "user.email=t@local", *args], + cwd=str(repo), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def _tmp_repo(tmp_path, name="repo"): + """A committed tmp git repo (inline identity, no global config needed).""" + repo = tmp_path / name + repo.mkdir() + _git(repo, "init", "-q", "-b", "trunk") + (repo / "README.md").write_text("hello\n") + (repo / ".gitignore").write_text("__pycache__/\n") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "--no-verify", "-m", "init") + return repo + + +def test_live_configs_resolve_control_and_candidate(): + run_eval = _run_eval() + + cfgs = run_eval._resolve_configs("B,E") + assert cfgs is not None and [c.id for c in cfgs] == ["B", "E"] + b, e = cfgs + assert b.model != e.model and b.effort == e.effort == "xhigh" + assert b.sandbox == e.sandbox and b.action_version == e.action_version + assert b.cli_version == e.cli_version == "codex-cli 0.162.1" + assert run_eval._control_id() == "B" + assert run_eval._treatment_fields() == ("model", "effort") + # Bare `run` is the registered campaign shape end to end: the COMPLETE default + # argument combination passes the candidate-exposure gate (configs B,E at k=2, + # no --k-per, no --strata), so `python run_eval.py run` is not configured to refuse. + ns = run_eval._build_parser().parse_args(["run"]) + assert ns.configs == "B,E" and ns.k == 2 and ns.k_per == "" and ns.strata is None + assert ( + run_eval._candidate_exposure_violations( + cfgs, ns.k, run_eval._parse_k_per(ns.k_per, cfgs), ns.strata, "run" + ) + == [] + ) + + +def test_git_provenance_seven_keys_on_tmp_repo(tmp_path): + from adapters import worktree + + repo = _tmp_repo(tmp_path) + prov = worktree.git_provenance(str(repo)) + assert sorted(prov) == sorted( + [ + "repo_head_sha", + "repo_tree_sha", + "repo_dirty", + "repo_branch", + "repo_common_dir", + "repo_worktree_paths", + "repo_ref_names", + ] + ) + assert prov["repo_head_sha"] == _git(repo, "rev-parse", "HEAD") + assert prov["repo_tree_sha"] == _git(repo, "rev-parse", "HEAD^{tree}") + assert prov["repo_dirty"] is False + assert prov["repo_branch"] == "trunk" + assert os.path.isabs(prov["repo_common_dir"]) + assert os.path.realpath(prov["repo_common_dir"]) == os.path.realpath(str(repo / ".git")) + assert prov["repo_ref_names"] == ["trunk"] + # Own-harness worktrees are excluded; a linked worktree elsewhere is included. + own = repo / "tools" / "reviewer-eval" / "runs" / ".worktrees" + own.mkdir(parents=True) + _git(repo, "worktree", "add", "--detach", str(own / "x"), "HEAD") + other = tmp_path / "linked-elsewhere" + _git(repo, "worktree", "add", "--detach", str(other), "HEAD") + prov2 = worktree.git_provenance(str(repo)) + assert [os.path.realpath(p) for p in prov2["repo_worktree_paths"]] == [ + os.path.realpath(str(other)) + ] + assert os.path.realpath(str(repo)) not in [ + os.path.realpath(p) for p in prov2["repo_worktree_paths"] + ] + (repo / "stray.txt").write_text("x") + assert worktree.git_provenance(str(repo))["repo_dirty"] is True + # Non-git dir: fail-soft values, None (never []) for the list fields. + empty = tmp_path / "not-a-repo" + empty.mkdir() + prov3 = worktree.git_provenance(str(empty)) + assert prov3["repo_head_sha"] == "" and prov3["repo_dirty"] is None + assert prov3["repo_worktree_paths"] is None and prov3["repo_ref_names"] is None + + +def test_review_records_provenance_worktree_session_and_home(monkeypatch): + from adapters import codex_reviewer as cr + + r = _make_reviewer(monkeypatch) + monkeypatch.setattr( + r._mod, + "call_codex", + lambda prompt, model, repo_root, effort="xhigh", timeout_s=None: ( + "## ok", + { + "backend": "codex", + "session_commands": ["/bin/zsh -lc \"sed -n '1,3p' a.py\" in /tmp/x"], + }, + ), + raising=True, + ) + from eval_core.models import Config + + cfg = Config(id="B", model="m", effort="xhigh", cli_version=_pinned_cli_version()) + out = r.review(_case(), cfg, 0) + for k, v in _STUB_PROVENANCE.items(): + assert out.usage[k] == v + assert out.usage["worktree_dir"] == "/tmp/reviewer-eval-test/wt" + assert out.usage["session_commands"] == ["/bin/zsh -lc \"sed -n '1,3p' a.py\" in /tmp/x"] + assert out.usage["codex_home_path"] is None and out.usage["codex_home_isolated"] is False + monkeypatch.setattr(cr, "ACTIVE_ISOLATED_HOME", "/tmp/codex-home-t") + monkeypatch.setenv("CODEX_HOME", "/tmp/codex-home-t") + out2 = r.review(_case(), cfg, 1) + assert out2.usage["codex_home_path"] == "/tmp/codex-home-t" + assert out2.usage["codex_home_isolated"] is True + + +def test_review_worktree_key_is_salted_opaque_hash(monkeypatch): + import hashlib + + from eval_core.models import Config + + r = _make_reviewer(monkeypatch) + seen = [] + monkeypatch.setattr( + r, + "build_prompt_for_case", + lambda case, worktree_key=None: ( + seen.append(worktree_key) or ("PROMPT", "/tmp/reviewer-eval-test/wt", "deadbeef") + ), + raising=True, + ) + cfg_b = Config(id="B", model="m", effort="xhigh", cli_version=_pinned_cli_version()) + cfg_e = Config(id="E", model="n", effort="xhigh", cli_version=_pinned_cli_version()) + case = _case() + r.review(case, cfg_b, 0) + r.review(case, cfg_b, 1) + r.review(case, cfg_e, 0) + assert len(seen) == 3 and len(set(seen)) == 3 + for key in seen: + assert re.fullmatch(r"[0-9a-f]{32}", key), key + unsalted = hashlib.sha256(f"{case.id}.B.r0".encode()).hexdigest()[:32] + namespaced = hashlib.sha256(f"{r._wt_namespace}.{case.id}.B.r0".encode()).hexdigest()[:32] + assert seen[0] not in (unsalted, namespaced), "the salt must be unrecomputable" + r2 = _make_reviewer(monkeypatch) + seen2 = [] + monkeypatch.setattr( + r2, + "build_prompt_for_case", + lambda case, worktree_key=None: ( + seen2.append(worktree_key) or ("PROMPT", "/tmp/reviewer-eval-test/wt", "deadbeef") + ), + raising=True, + ) + r2.review(case, cfg_b, 0) + assert seen2[0] != seen[0], "two instances must not share review worktree keys" + + +def _fake_real_home(tmp_path, monkeypatch, with_auth=True): + real = tmp_path / "real-codex" + real.mkdir() + if with_auth: + (real / "auth.json").write_text('{"token": "entry"}') + (real / "config.toml").write_text('[mcp_servers.x]\ncommand = "node"\n') + (real / "memories").mkdir() + monkeypatch.setenv("CODEX_HOME", str(real)) + return real + + +def test_isolated_codex_home_mirrors_ci_and_restores_env(tmp_path, monkeypatch): + from adapters import codex_reviewer as cr + + real = _fake_real_home(tmp_path, monkeypatch) + with cr.isolated_codex_home() as home: + assert sorted(os.listdir(home)) == ["auth.json", "config.toml"] + assert (pathlib.Path(home) / "config.toml").read_text() == cr.ISOLATED_CONFIG_TOML + assert 'sandbox_mode = "read-only"' in cr.ISOLATED_CONFIG_TOML + assert 'approval_policy = "never"' in cr.ISOLATED_CONFIG_TOML + assert "[features]" not in cr.ISOLATED_CONFIG_TOML and "mcp" not in cr.ISOLATED_CONFIG_TOML + assert cr.ACTIVE_ISOLATED_HOME == home and os.environ["CODEX_HOME"] == home + assert home != str(real) + assert cr.ACTIVE_ISOLATED_HOME is None + assert os.environ["CODEX_HOME"] == str(real) + assert not os.path.exists(home) + # (a) isolated copy unchanged -> real file untouched, nothing else written. + assert (real / "auth.json").read_text() == '{"token": "entry"}' + assert sorted(os.listdir(real)) == ["auth.json", "config.toml", "memories"] + + +def test_isolated_codex_home_saves_rotated_token_beside_real(tmp_path, monkeypatch): + from adapters import codex_reviewer as cr + + real = _fake_real_home(tmp_path, monkeypatch) + with cr.isolated_codex_home() as home: + (pathlib.Path(home) / "auth.json").write_text('{"token": "rotated"}') + # (b) real file byte-identical to entry; exactly one campaign copy holds the new bytes. + assert (real / "auth.json").read_text() == '{"token": "entry"}' + saved = sorted(real.glob("auth.json.campaign-*")) + assert len(saved) == 1 and saved[0].read_text() == '{"token": "rotated"}' + # The rerun seeds from the NEWEST candidate (the rotated copy). + os.utime(saved[0], None) + with cr.isolated_codex_home() as home2: + assert (pathlib.Path(home2) / "auth.json").read_text() == '{"token": "rotated"}' + + +def test_isolated_codex_home_requires_credential(tmp_path, monkeypatch): + from adapters import codex_reviewer as cr + + _fake_real_home(tmp_path, monkeypatch, with_auth=False) + with pytest.raises(RuntimeError, match="no codex credential"): + with cr.isolated_codex_home(): + pass + + +def test_cmd_run_enters_isolated_home_once_and_writes_provenance(tmp_path, monkeypatch): + """The happy path enters the isolated home exactly once around the matrix, and + the manifest carries the seven provenance keys, codex_home_path, the + aggregates and the artifacts seal.""" + import json as _json + + run_eval = _run_eval() + runs_dir = _run_ok_experiment(tmp_path, monkeypatch, subdir="prov") + fake = run_eval.isolated_codex_home # the _FakeIsolatedHome the helper installed + assert fake.entries == 1, "the isolated home is entered exactly once around the matrix" + manifest = _json.loads((runs_dir / "prov-manifest.json").read_text()) + for k, v in _STUB_PROVENANCE.items(): + assert manifest[k] == v + assert manifest["codex_home_path"] == fake.home + assert manifest["artifacts_sha"] + assert manifest["run_head_shas"] == [] and manifest["run_head_missing"] == 2 + assert manifest["run_dirty_any"] is False + assert manifest["run_marker_mismatch"] == 2 and manifest["run_not_isolated"] == 2 + + +def _ok_results(usages): + from eval_core.models import RunResult + + return [ + RunResult(case_id="c", config_id="A", repeat_idx=i, usage=u, run_id=f"k{i}") + for i, u in enumerate(usages) + ] + + +def test_provenance_aggregates_precedence(): + run_eval = _run_eval() + prov = {"repo_branch": "trunk", "repo_common_dir": "/r/.git"} + base = { + "repo_head_sha": "h1", + "repo_dirty": False, + "repo_branch": "trunk", + "repo_common_dir": "/r/.git", + "codex_home_isolated": True, + "repo_ref_names": ["trunk"], + "repo_worktree_paths": [], + } + # (i) full provenance collects. + agg = run_eval._provenance_aggregates(_ok_results([base, base]), prov) + assert agg == { + "run_head_shas": ["h1"], + "run_head_missing": 0, + "run_dirty_any": False, + "run_marker_mismatch": 0, + "run_not_isolated": 0, + } + # (ii) legacy prompt_sha-only stubs. + agg = run_eval._provenance_aggregates(_ok_results([{"prompt_sha": "p"}] * 3), prov) + assert agg["run_head_shas"] == [] and agg["run_head_missing"] == 3 + assert agg["run_dirty_any"] is False + # (iii) head sha + repo_dirty None -> None. + assert ( + run_eval._provenance_aggregates(_ok_results([{**base, "repo_dirty": None}]), prov)[ + "run_dirty_any" + ] + is None + ) + # (iv) any True -> True. + assert ( + run_eval._provenance_aggregates(_ok_results([base, {**base, "repo_dirty": True}]), prov)[ + "run_dirty_any" + ] + is True + ) + # (v)/(vi) marker mismatch: a different or empty branch counts. + assert ( + run_eval._provenance_aggregates( + _ok_results([base, {**base, "repo_branch": "other"}]), prov + )["run_marker_mismatch"] + == 1 + ) + assert ( + run_eval._provenance_aggregates(_ok_results([base, {**base, "repo_branch": ""}]), prov)[ + "run_marker_mismatch" + ] + == 1 + ) + # not isolated: flag false, or None ref/worktree lists. + assert ( + run_eval._provenance_aggregates( + _ok_results([{**base, "codex_home_isolated": False}]), prov + )["run_not_isolated"] + == 1 + ) + assert ( + run_eval._provenance_aggregates(_ok_results([{**base, "repo_ref_names": None}]), prov)[ + "run_not_isolated" + ] + == 1 + ) + assert ( + run_eval._provenance_aggregates(_ok_results([{**base, "repo_worktree_paths": None}]), prov)[ + "run_not_isolated" + ] + == 1 + ) + # (vii) a manifest whose own repo_branch is "" -> the Campaign gate expression prints BAD. + m = { + **prov, + "repo_branch": "", + "run_head_missing": 0, + "run_dirty_any": False, + "run_head_shas": ["h"], + "repo_head_sha": "h", + "repo_dirty": False, + "run_marker_mismatch": 0, + "run_not_isolated": 0, + "repo_tree_sha": "t", + "repo_ref_names": [], + "repo_worktree_paths": [], + } + ok = ( + m.get("run_head_missing") == 0 + and m.get("run_dirty_any") is False + and (m.get("run_head_shas") or []) == [m.get("repo_head_sha")] + and m.get("repo_dirty") is False + and m.get("run_marker_mismatch") == 0 + and m.get("run_not_isolated") == 0 + and bool(m.get("repo_branch")) + and bool(m.get("repo_common_dir")) + and bool(m.get("repo_tree_sha")) + and isinstance(m.get("repo_ref_names"), list) + and isinstance(m.get("repo_worktree_paths"), list) + ) + assert ok is False + + +def test_cmd_run_manifest_carries_real_write_time_provenance(tmp_path, monkeypatch): + """With the REAL _manifest_provenance (not the stub) and find_repo_root pointed + at a tmp git repo, the manifest's write-time fields equal git_provenance(tmp).""" + import json as _json + + from adapters import worktree + + run_eval = _run_eval() + repo = _tmp_repo(tmp_path) + fake = _FakeIsolatedHome(tmp_path) + monkeypatch.setattr(run_eval, "isolated_codex_home", fake) + monkeypatch.setattr(run_eval, "_assert_no_hidden_inputs", lambda repo_root: None) + monkeypatch.setattr(run_eval, "find_repo_root", lambda: str(repo)) + runs_dir = _run_ok_experiment_no_stub(tmp_path, monkeypatch, subdir="real") + manifest = _json.loads((runs_dir / "real-manifest.json").read_text()) + live = worktree.git_provenance(str(repo)) + for k in ("repo_head_sha", "repo_tree_sha", "repo_dirty", "repo_branch", "repo_ref_names"): + assert manifest[k] == live[k], k + + +def _run_ok_experiment_no_stub(tmp_path, monkeypatch, subdir): + """_run_ok_experiment minus _stub_run_git (callers stub what they need).""" + run_eval = _run_eval() + from eval_core.models import STRATUM_SYNTHETIC, Case, ReviewOutput + + monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) + monkeypatch.setattr( + run_eval.CorpusLoader, + "load_cases", + lambda self, strata: [Case(id="c", stratum=STRATUM_SYNTHETIC)], + raising=True, + ) + monkeypatch.setattr(run_eval.CorpusLoader, "verify", lambda self, case: None, raising=True) + + class _Ok: + def cli_version(self): + return _pinned_cli_version() + + def experiment_tag(self, config): + return f"tag-{config.id}" + + def case_tag(self, case): + return "ctag" + + def prompt_sha_for(self, case): + return "psha" + + def review(self, case, config, repeat_idx): + return ReviewOutput( + review_markdown=f"As {config.model}, I see no issues.", + cli_version=_pinned_cli_version(), + latency_s=0.0, + usage={"prompt_sha": "psha"}, + ) + + monkeypatch.setattr(run_eval, "_build_reviewer", lambda repo_root: _Ok(), raising=True) + rc = run_eval.cmd_run( + _ns(configs="A,B", strata=["s1_synthetic"], subdir=subdir, k=1, max_parallel=1) + ) + assert rc == 0 + return tmp_path / "runs" + + +def test_cmd_run_refuses_hidden_corpus_input_before_reviewer(tmp_path, monkeypatch): + run_eval = _run_eval() + from eval_core.models import STRATUM_SYNTHETIC, Case + + repo = _tmp_repo(tmp_path) + hidden = repo / "tools" / "reviewer-eval" / "corpus" / "cases" / "s9_x" / "extra" + hidden.mkdir(parents=True) + (hidden / "case.json").write_text("{}") + (repo / ".git" / "info").mkdir(exist_ok=True) + (repo / ".git" / "info" / "exclude").write_text("tools/reviewer-eval/corpus/cases/s9_x/\n") + assert _git(repo, "status", "--porcelain", "--untracked-files=all") == "" + monkeypatch.setattr(run_eval, "find_repo_root", lambda: str(repo)) + monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) + monkeypatch.setattr(run_eval, "isolated_codex_home", _FakeIsolatedHome(tmp_path)) + monkeypatch.setattr( + run_eval.CorpusLoader, + "load_cases", + lambda self, strata: [Case(id="c", stratum=STRATUM_SYNTHETIC)], + raising=True, + ) + monkeypatch.setattr(run_eval.CorpusLoader, "verify", lambda self, case: None, raising=True) + + def _never(repo_root): + raise AssertionError("_build_reviewer must not be reached") + + monkeypatch.setattr(run_eval, "_build_reviewer", _never, raising=True) + with pytest.raises(SystemExit, match="IGNORED FILE"): + run_eval.cmd_run(_ns(configs="A,B", strata=None, subdir="h", k=1, max_parallel=1)) + + +def test_assert_tree_identity_guard(tmp_path, monkeypatch): + run_eval = _run_eval() + + repo = _tmp_repo(tmp_path) + manifest = { + "repo_head_sha": _git(repo, "rev-parse", "HEAD"), + "repo_tree_sha": _git(repo, "rev-parse", "HEAD^{tree}"), + } + run_eval._assert_tree_identity(str(repo), manifest) # clean: passes + # Bytecode FILES under tools/eval_core do not trip it ... + pyc = repo / "tools" / "eval_core" / "__pycache__" + pyc.mkdir(parents=True) + (pyc / "x.cpython-314.pyc").write_bytes(b"\x00") + (repo / ".DS_Store").write_bytes(b"\x00") + (repo / ".git" / "info").mkdir(exist_ok=True) + (repo / ".git" / "info" / "exclude").write_text(".DS_Store\n") + run_eval._assert_tree_identity(str(repo), manifest) + # ... but an excluded corpus file does, and so does an excluded __pycache__/case.json dir. + extra = repo / "tools" / "reviewer-eval" / "corpus" + extra.mkdir(parents=True) + (extra / "extra.json").write_text("{}") + (repo / ".git" / "info" / "exclude").write_text( + ".DS_Store\ntools/reviewer-eval/corpus/extra.json\n" + ) + with pytest.raises(SystemExit, match="IGNORED FILE"): + run_eval._assert_tree_identity(str(repo), manifest) + (extra / "extra.json").unlink() + cache_case = repo / "tools" / "reviewer-eval" / "corpus" / "cases" / "s1" / "__pycache__" + cache_case.mkdir(parents=True) + (cache_case / "case.json").write_text("{}") + with pytest.raises(SystemExit, match="IGNORED FILE"): + run_eval._assert_tree_identity(str(repo), manifest) + import shutil + + shutil.rmtree(repo / "tools" / "reviewer-eval") + run_eval._assert_tree_identity(str(repo), manifest) + # Dirty tree. + (repo / "stray.txt").write_text("x") + with pytest.raises(SystemExit, match="not clean"): + run_eval._assert_tree_identity(str(repo), manifest) + (repo / "stray.txt").unlink() + # Tree-sha mismatch. + with pytest.raises(SystemExit, match="repo_tree_sha"): + run_eval._assert_tree_identity(str(repo), {**manifest, "repo_tree_sha": "0" * 40}) + # Same tree, amended commit -> HEAD mismatch. + _git(repo, "commit", "-q", "--amend", "--no-verify", "-m", "amended") + assert _git(repo, "rev-parse", "HEAD^{tree}") == manifest["repo_tree_sha"] + with pytest.raises(SystemExit, match="repo_head_sha"): + run_eval._assert_tree_identity(str(repo), manifest) + + +def _sealed_experiment_in_tmp_repo(tmp_path, monkeypatch, subdir="seal"): + """A real cmd_run (stubbed reviewer) whose manifest's HEAD/tree are then + REWRITTEN to a clean tmp git repo, so the tree-identity guard can pass there.""" + import json as _json + + run_eval = _run_eval() + repo = _tmp_repo(tmp_path) + monkeypatch.setattr(run_eval, "find_repo_root", lambda: str(repo)) + runs_dir = _run_ok_experiment(tmp_path, monkeypatch, subdir=subdir) + mpath = runs_dir / f"{subdir}-manifest.json" + manifest = _json.loads(mpath.read_text()) + manifest["repo_head_sha"] = _git(repo, "rev-parse", "HEAD") + manifest["repo_tree_sha"] = _git(repo, "rev-parse", "HEAD^{tree}") + mpath.write_text(_json.dumps(manifest)) + monkeypatch.setattr(run_eval, "_assert_no_hidden_inputs", lambda repo_root: None) + return repo, runs_dir, manifest + + +def test_compare_blinded_refuses_edited_artifact_and_checks_identity_at_exit(tmp_path, monkeypatch): + import json as _json + + run_eval = _run_eval() + repo, runs_dir, manifest = _sealed_experiment_in_tmp_repo(tmp_path, monkeypatch) + # Restore the REAL guard (the ok-experiment helper no-ops it). + monkeypatch.setattr( + run_eval, "_assert_tree_identity", _run_eval().__dict__["_assert_tree_identity"] + ) + assert run_eval.cmd_compare(_ns(subdir="seal", allow_mixed=False, blinded=True)) == 0 + blinding = _json.loads((runs_dir / "seal" / "blinding.json").read_text()) + assert blinding["artifacts_sha"] == manifest["artifacts_sha"] + assert blinding["run_ids_sha"] == run_eval._run_ids_sha(manifest["run_ids"]) + # Edit one artifact's session_commands after run -> refused with both hashes. + art = next(p for p in (runs_dir / "seal").glob("*.json") if p.name != "blinding.json") + d = _json.loads(art.read_text()) + d["usage"]["session_commands"] = ["/bin/zsh -lc 'git log --all' in /x"] + art.write_text(_json.dumps(d)) + assert run_eval.cmd_compare(_ns(subdir="seal", allow_mixed=False, blinded=True)) == 1 + art.write_text(_json.dumps(_json.loads(art.read_text()))) # leave edited; next tests re-run + + +def test_stage_exit_identity_recheck_catches_mid_stage_edit(tmp_path, monkeypatch): + run_eval = _run_eval() + repo, runs_dir, manifest = _sealed_experiment_in_tmp_repo( + tmp_path, monkeypatch, subdir="exitck" + ) + real_guard = run_eval._assert_tree_identity + + def _dirtying_audit(runs, subdir_path, repo_root, markers, run_ids_sha, sealed): + (repo / "mid-stage.txt").write_text("edit while the stage runs") + return { + "run_ids_sha": run_ids_sha, + "artifacts_sha": sealed, + "branch": "", + "hits": [], + "unaudited": [], + "no_commands": [], + "coverage": {"covered": 0, "no_commands": 0, "unaudited": 0, "total": 0}, + "vacuous": True, + } + + monkeypatch.setattr(run_eval.reviewer_audit, "audit", _dirtying_audit) + monkeypatch.setattr(run_eval, "_assert_tree_identity", real_guard) + with pytest.raises(SystemExit, match="stage exit"): + run_eval.cmd_audit(_ns(subdir="exitck", output=str(tmp_path / "hits.json"))) + + +# --------------------------------------------------------------------------- # +# Local-review fixes: credential mode, candidate-exposure gate, stop conditions. +# --------------------------------------------------------------------------- # + + +def test_rotated_credential_saved_owner_only(tmp_path, monkeypatch): + import stat + + from adapters import codex_reviewer as cr + + real = _fake_real_home(tmp_path, monkeypatch) + with cr.isolated_codex_home() as home: + (pathlib.Path(home) / "auth.json").write_text('{"token": "rotated"}') + saved = sorted(real.glob("auth.json.campaign-*")) + assert len(saved) == 1 + mode = stat.S_IMODE(saved[0].stat().st_mode) + assert mode & 0o077 == 0, f"credential copy must be owner-only, got {oct(mode)}" + + +def _gate_setup(tmp_path, monkeypatch): + run_eval = _run_eval() + from eval_core.models import STRATUM_SYNTHETIC, Case + + _stub_run_git(monkeypatch, tmp_path) + monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) + monkeypatch.setattr( + run_eval.CorpusLoader, + "load_cases", + lambda self, strata: [Case(id="c", stratum=STRATUM_SYNTHETIC)], + raising=True, + ) + monkeypatch.setattr(run_eval.CorpusLoader, "verify", lambda self, case: None, raising=True) + + def _never(repo_root): + raise AssertionError("_build_reviewer must not be reached") + + monkeypatch.setattr(run_eval, "_build_reviewer", _never, raising=True) + return run_eval + + +@pytest.mark.parametrize( + "ns", + [ + dict(configs="E", strata=None, subdir="x", k=2, max_parallel=1), + dict(configs="B,E", strata=None, subdir="x", k=1, max_parallel=1), + dict(configs="B,E", strata=["s1_synthetic"], subdir="x", k=2, max_parallel=1), + dict(configs="B,E", strata=None, subdir="x", k=2, k_per="E=1", max_parallel=1), + dict(configs="A,B,E", strata=None, subdir="x", k=2, max_parallel=1), + ], +) +def test_run_refuses_candidate_exposure_outside_registered_campaign( + tmp_path, monkeypatch, ns, capsys +): + run_eval = _gate_setup(tmp_path, monkeypatch) + assert run_eval.cmd_run(_ns(**ns)) == 1 + assert "pre-registration" in capsys.readouterr().err + assert not (tmp_path / "runs" / "x-attempts.jsonl").exists() + + +def test_smoke_refuses_candidate_arm(tmp_path, monkeypatch, capsys): + run_eval = _gate_setup(tmp_path, monkeypatch) + assert run_eval.cmd_smoke(_ns(configs="E", strata=None, k=1, limit=1, max_parallel=1)) == 1 + assert "may not be smoked" in capsys.readouterr().err + + +def test_registered_campaign_shape_passes_the_gate(): + run_eval = _run_eval() + cfgs = run_eval._resolve_configs("B,E") + assert run_eval._candidate_exposure_violations(cfgs, 2, {}, None, "run") == [] + assert ( + run_eval._candidate_exposure_violations( + run_eval._resolve_configs("A,B"), 1, {}, ["s1_synthetic"], "run" + ) + == [] + ) + + +def test_stop_conditions_from_attempts_log(tmp_path, monkeypatch): + import json as _json + + run_eval = _run_eval() + monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) + (tmp_path / "runs").mkdir() + path = tmp_path / "runs" / "g-attempts.jsonl" + + def end(failures): + return _json.dumps( + {"kind": "end", "started_at": "t", "ok": 1, "total": 2, "failures": failures} + ) + + start = _json.dumps({"kind": "start", "head": "h", "at": "t", "configs": ["B", "E"], "k": 2}) + # 4 dispatched attempts -> the cap (a START without END does not count). + path.write_text( + "\n".join([start, end([]), start, end([]), start, end([]), start, end([]), start]) + "\n" + ) + stops = run_eval._stop_condition_violations("g") + assert len(stops) == 1 and "cap" in stops[0] + # Two consecutive timeouts on the same cell -> stop; on different cells -> no stop. + t = {"case": "c", "arm": "E", "repeat": 1, "infra_error": "codex exec timed out after 3600s"} + path.write_text("\n".join([start, end([t]), start, end([t])]) + "\n") + stops = run_eval._stop_condition_violations("g") + assert len(stops) == 1 and "2 consecutive" in stops[0] + t2 = {**t, "repeat": 0} + path.write_text("\n".join([start, end([t]), start, end([t2])]) + "\n") + assert run_eval._stop_condition_violations("g") == [] + # Non-timeout failures never trip the consecutive rule. + f = {**t, "infra_error": "worktree add failed"} + path.write_text("\n".join([start, end([f]), start, end([f])]) + "\n") + assert run_eval._stop_condition_violations("g") == [] + + +def test_cmd_run_appends_attempt_records_and_refuses_at_cap(tmp_path, monkeypatch): + import json as _json + + run_eval = _run_eval() + _run_ok_experiment(tmp_path, monkeypatch, subdir="cap") # the one real dispatch + lines = [ + _json.loads(line) + for line in (tmp_path / "runs" / "cap-attempts.jsonl").read_text().splitlines() + ] + assert [r["kind"] for r in lines] == ["start", "end"] + assert lines[0]["head"] == _STUB_PROVENANCE["repo_head_sha"] and lines[1]["failures"] == [] + assert len(lines[1]["ok_cells"]) == 2 and all(c["sha256"] for c in lines[1]["ok_cells"]) + # Three more dispatched attempts (synthetic END records that RESUMED every OK cell). + for _ in range(3): + run_eval._append_attempt("cap", {**lines[0], "at": "later"}) + run_eval._append_attempt("cap", lines[1]) + manifest_before = (tmp_path / "runs" / "cap-manifest.json").read_bytes() + assert b'"failed"' not in manifest_before + with pytest.raises( + AssertionError + ): # the 5th attempt is refused (rc 1), the helper asserts rc == 0 + _run_ok_experiment(tmp_path, monkeypatch, subdir="cap") + assert ( + len((tmp_path / "runs" / "cap-attempts.jsonl").read_text().splitlines()) == 8 + ), "refused attempt appends nothing" + assert ( + tmp_path / "runs" / "cap-manifest.json" + ).read_bytes() == manifest_before, "a refused attempt leaves the last valid manifest intact" + + +def test_aborted_attempt_after_dispatch_counts_toward_cap(tmp_path, monkeypatch): + import json as _json + + run_eval = _run_eval() + from eval_core.models import STRATUM_SYNTHETIC, Case + from eval_core.runner import ConfoundMismatch + + _stub_run_git(monkeypatch, tmp_path) + monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) + monkeypatch.setattr( + run_eval.CorpusLoader, + "load_cases", + lambda self, strata: [Case(id="c", stratum=STRATUM_SYNTHETIC)], + raising=True, + ) + monkeypatch.setattr(run_eval.CorpusLoader, "verify", lambda self, case: None, raising=True) + monkeypatch.setattr(run_eval, "_build_reviewer", lambda repo_root: object(), raising=True) + + def _boom(*a, **k): + k["on_dispatch"]() # the runner has handed the first review to the pool + raise RuntimeError("codex died mid-matrix") + + monkeypatch.setattr("eval_core.runner.run_matrix", _boom, raising=True) + with pytest.raises(RuntimeError): + run_eval.cmd_run(_ns(configs="A,B", strata=None, subdir="ab", k=1, max_parallel=1)) + recs = [ + _json.loads(x) for x in (tmp_path / "runs" / "ab-attempts.jsonl").read_text().splitlines() + ] + assert [r["kind"] for r in recs] == ["start", "end"] and recs[1]["aborted"] == "RuntimeError" + assert _json.loads((tmp_path / "runs" / "ab-manifest.json").read_text())["failed"] is True + + def _pre(*a, **k): + raise ConfoundMismatch("arms differ in sandbox") + + def _planning(*a, **k): + raise RuntimeError("experiment_tag failed for config B") # planning: before dispatch + + for raiser, exc in ((_pre, ConfoundMismatch), (_planning, RuntimeError)): + monkeypatch.setattr("eval_core.runner.run_matrix", raiser, raising=True) + with pytest.raises(exc): + run_eval.cmd_run(_ns(configs="A,B", strata=None, subdir="ab", k=1, max_parallel=1)) + recs = [ + _json.loads(x) for x in (tmp_path / "runs" / "ab-attempts.jsonl").read_text().splitlines() + ] + assert [r["kind"] for r in recs] == [ + "start", + "end", + "start", + "start", + ], "pre-dispatch raise (any type): no END record" + # Three more aborted attempts hit the cap. + monkeypatch.setattr("eval_core.runner.run_matrix", _boom, raising=True) + for _ in range(3): + with pytest.raises(RuntimeError): + run_eval.cmd_run(_ns(configs="A,B", strata=None, subdir="ab", k=1, max_parallel=1)) + assert run_eval.cmd_run(_ns(configs="A,B", strata=None, subdir="ab", k=1, max_parallel=1)) == 1 + + +def test_cmd_run_preflight_refuses_dirty_or_incomplete_provenance(tmp_path, monkeypatch, capsys): + """Before the manifest is touched and before any codex call: a dirty tree or + an incomplete git sample refuses the attempt (no attempt record, no reviewer).""" + run_eval = _run_eval() + _stub_run_git(monkeypatch, tmp_path) + for bad, why in ( + ({**_STUB_PROVENANCE, "repo_ref_names": None}, "incomplete git provenance"), + ({**_STUB_PROVENANCE, "repo_dirty": True}, "working tree is dirty"), + ): + monkeypatch.setattr(run_eval, "_manifest_provenance", lambda repo_root, bad=bad: dict(bad)) + monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) + + def _never(repo_root): + raise AssertionError("_build_reviewer must not be reached") + + monkeypatch.setattr(run_eval, "_build_reviewer", _never, raising=True) + rc = run_eval.cmd_run(_ns(configs="A,B", strata=None, subdir="pre", k=1, max_parallel=1)) + assert rc == 1 and why in capsys.readouterr().err + assert not (tmp_path / "runs" / "pre-attempts.jsonl").exists() + assert not (tmp_path / "runs" / "pre-manifest.json").exists() + + +def test_cmd_run_fails_closed_on_incomplete_write_time_provenance(tmp_path, monkeypatch): + """The preflight sample is clean but the WRITE-TIME sample fails (a transient git + error mid-run): failure marker, exit 2, resume reuses the OK artifacts.""" + import json as _json + + run_eval = _run_eval() + _stub_run_git(monkeypatch, tmp_path) + calls = {"n": 0} + + def _flaky(repo_root): + calls["n"] += 1 + if calls["n"] == 1: + return dict(_STUB_PROVENANCE) + return {**_STUB_PROVENANCE, "repo_ref_names": None} + + monkeypatch.setattr(run_eval, "_manifest_provenance", _flaky) + with pytest.raises(AssertionError): # helper asserts rc == 0; the run returns 2 + _run_ok_experiment_no_stub(tmp_path, monkeypatch, subdir="incomplete") + m = _json.loads((tmp_path / "runs" / "incomplete-manifest.json").read_text()) + assert m["failed"] is True and "repo_ref_names" in m["error"] + assert run_eval._provenance_failures({**_STUB_PROVENANCE, "repo_worktree_paths": None}) == [ + "repo_worktree_paths" + ] + assert run_eval._provenance_failures({**_STUB_PROVENANCE, "repo_head_sha": ""}) == [ + "repo_head_sha" + ] + assert run_eval._provenance_failures({**_STUB_PROVENANCE, "repo_dirty": None}) == ["repo_dirty"] + assert run_eval._provenance_failures(_STUB_PROVENANCE) == [] + + +def test_run_matrix_calls_on_dispatch_once_before_first_review(): + import tempfile + + from eval_core.models import Config, ReviewOutput + from eval_core.runner import run_matrix + from eval_core.store import RunStore + + events = [] + + class _Rev: + def cli_version(self): + return "codex-cli 0.162.1" + + def experiment_tag(self, config): + return "t" + + def case_tag(self, case): + return "c" + + def prompt_sha_for(self, case): + return "p" + + def review(self, case, config, repeat_idx): + events.append("review") + return ReviewOutput( + review_markdown="ok", + cli_version="codex-cli 0.162.1", + latency_s=0.0, + usage={"prompt_sha": "p"}, + ) + + cfg = Config(id="A", model="m", effort="xhigh", cli_version="codex-cli 0.162.1") + with tempfile.TemporaryDirectory() as d: + store = RunStore(d) + run_matrix( + [_case()], + [cfg], + _Rev(), + store, + k=2, + max_parallel=1, + on_dispatch=lambda: events.append("dispatch"), + ) + assert events[0] == "dispatch" and events.count("dispatch") == 1 + assert events.count("review") == 2 + events.clear() + run_matrix( + [_case()], + [cfg], + _Rev(), + store, + k=2, + max_parallel=1, + on_dispatch=lambda: events.append("dispatch"), + ) + assert events == [], "a fully cached matrix dispatches nothing" + + +def test_cmd_run_appends_campaign_ledger_only_for_registered_contrast(tmp_path, monkeypatch): + import json as _json + + run_eval = _run_eval() + _stub_run_git(monkeypatch, tmp_path) + # A,B is not a registered contrast: no ledger record. + _run_ok_experiment(tmp_path, monkeypatch, subdir="ab") + assert not (tmp_path / "runs" / "campaign-ledger.jsonl").exists() + # B,E k=2 IS the registered campaign: one dispatch record on the preflight head. + from eval_core.models import STRATUM_SYNTHETIC, Case, ReviewOutput + + class _Ok: + def cli_version(self): + return _pinned_cli_version() + + def experiment_tag(self, config): + return f"tag-{config.id}" + + def case_tag(self, case): + return "ctag" + + def prompt_sha_for(self, case): + return "psha" + + def review(self, case, config, repeat_idx): + return ReviewOutput( + review_markdown="x", + cli_version=_pinned_cli_version(), + latency_s=0.0, + usage={"prompt_sha": "psha"}, + ) + + monkeypatch.setattr(run_eval, "_build_reviewer", lambda repo_root: _Ok(), raising=True) + monkeypatch.setattr( + run_eval.CorpusLoader, + "load_cases", + lambda self, strata: [Case(id="c", stratum=STRATUM_SYNTHETIC)], + raising=True, + ) + assert ( + run_eval.cmd_run(_ns(configs="B,E", strata=None, subdir="camp", k=2, max_parallel=1)) == 0 + ) + recs = [ + _json.loads(x) + for x in (tmp_path / "runs" / "campaign-ledger.jsonl").read_text().splitlines() + ] + assert recs == [ + { + "at": recs[0]["at"], + "contrast": ["B", "E"], + "head": "feedface", + "kind": "dispatch", + "ledger_reset": False, + "subdir": "camp", + "tree": "cafebabe", + } + ] + + +def test_cached_resume_writes_no_end_record(tmp_path, monkeypatch): + import json as _json + + run_eval = _run_eval() + _run_ok_experiment(tmp_path, monkeypatch, subdir="cache") # dispatches + _run_ok_experiment(tmp_path, monkeypatch, subdir="cache") # fully cached resume + recs = [ + _json.loads(x) + for x in (tmp_path / "runs" / "cache-attempts.jsonl").read_text().splitlines() + ] + assert [r["kind"] for r in recs] == [ + "start", + "end", + "start", + ], "a cached resume is not an attempt" + assert not run_eval._stop_condition_violations("cache") + + +def test_ledger_reset_is_recorded_when_frozen_ref_exists(tmp_path, monkeypatch): + import json as _json + + run_eval = _run_eval() + fake = _stub_run_git(monkeypatch, tmp_path) + fake.refs[("B", "E")] = "feedface" # the ref survived; the ledger did not + from eval_core.models import STRATUM_SYNTHETIC, Case, ReviewOutput + + class _Ok: + def cli_version(self): + return _pinned_cli_version() + + def experiment_tag(self, config): + return f"tag-{config.id}" + + def case_tag(self, case): + return "ctag" + + def prompt_sha_for(self, case): + return "psha" + + def review(self, case, config, repeat_idx): + return ReviewOutput( + review_markdown="x", + cli_version=_pinned_cli_version(), + latency_s=0.0, + usage={"prompt_sha": "psha"}, + ) + + monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) + monkeypatch.setattr(run_eval, "_build_reviewer", lambda repo_root: _Ok(), raising=True) + monkeypatch.setattr( + run_eval.CorpusLoader, + "load_cases", + lambda self, strata: [Case(id="c", stratum=STRATUM_SYNTHETIC)], + raising=True, + ) + monkeypatch.setattr(run_eval.CorpusLoader, "verify", lambda self, case: None, raising=True) + assert ( + run_eval.cmd_run(_ns(configs="B,E", strata=None, subdir="again", k=2, max_parallel=1)) == 0 + ) + recs = [ + _json.loads(x) + for x in (tmp_path / "runs" / "campaign-ledger.jsonl").read_text().splitlines() + ] + assert recs[0]["ledger_reset"] is True and recs[0]["head"] == "feedface" + + +def test_frozen_ref_helpers_on_tmp_repo(tmp_path): + run_eval = _run_eval() + repo = _tmp_repo(tmp_path) + head = _git(repo, "rev-parse", "HEAD") + assert run_eval._read_frozen_ref(str(repo), ("B", "E")) is None + assert run_eval._anchor_frozen_ref(str(repo), ("B", "E"), head) == head + assert run_eval._read_frozen_ref(str(repo), ("B", "E")) == head + assert _git(repo, "rev-parse", "refs/campaign/B-E/frozen") == head + # A later anchor call never moves the ref. + _git(repo, "commit", "-q", "--allow-empty", "--no-verify", "-m", "later") + assert ( + run_eval._anchor_frozen_ref(str(repo), ("B", "E"), _git(repo, "rev-parse", "HEAD")) == head + ) + + +def test_cmd_run_refuses_a_concurrent_attempt_into_the_same_subdir(tmp_path, monkeypatch, capsys): + """Two processes with the same --subdir compute identical run ids and would + interleave artifacts into one store; the second is refused at the per-subdir + lock, before the attempts log or the manifest is touched. The holder here is + a REAL second process; once it exits, the same attempt proceeds.""" + import subprocess + import sys as _sys + + run_eval = _run_eval() + runs = tmp_path / "runs" + runs.mkdir() + monkeypatch.setattr(run_eval, "RUNS_DIR", str(runs)) + lock = run_eval._lock_path("lk") + assert lock == str(runs / "lk.lock") + holder = subprocess.Popen( + [ + _sys.executable, + "-c", + "import fcntl, sys, time\n" + "fh = open(sys.argv[1], 'a')\n" + "fcntl.flock(fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)\n" + "print('held', flush=True)\n" + "time.sleep(120)\n", + lock, + ], + stdout=subprocess.PIPE, + text=True, + ) + try: + assert holder.stdout.readline().strip() == "held" + with pytest.raises(AssertionError): # helper asserts rc == 0; the run returns 1 + _run_ok_experiment(tmp_path, monkeypatch, subdir="lk") + err = capsys.readouterr().err + assert "concurrent run" in err and "lk.lock" in err + assert not (runs / "lk-attempts.jsonl").exists() + assert not (runs / "lk-manifest.json").exists() + assert not (runs / "lk").exists() or not list((runs / "lk").glob("*.json")) + finally: + holder.kill() + holder.wait() + # The holder is gone: the same attempt proceeds and owns every artifact. + _run_ok_experiment(tmp_path, monkeypatch, subdir="lk") + assert len(list((runs / "lk").glob("*.json"))) == 2 + # In-process: a held lock yields False to a second acquirer, True after release. + with run_eval._subdir_lock("lk") as first: + assert first is True + with run_eval._subdir_lock("lk") as second: + assert second is False + with run_eval._subdir_lock("lk") as again: + assert again is True + + +def test_aborted_attempt_records_the_cells_it_persisted_and_resume_protects_them( + tmp_path, monkeypatch, capsys +): + """An attempt whose worker raises immediately AFTER its artifact landed on disk + (before any bookkeeping) aborts; the aborted END record lists every cell on + disk (that one AND the sibling the pool finished while shutting down); + deleting or editing one before the rerun is refused as re-sampling, and a + rerun that preserves them resumes both without a dispatch.""" + import json as _json + + run_eval = _run_eval() + real_store = run_eval.RunStore + calls = {"n": 0} + + class _DiesRightAfterFirstSave(real_store): + # The narrowest window: the artifact's os.replace has landed, nothing + # after it ran. The journal must still see it. + def save(self, key, result): + calls["n"] += 1 + super().save(key, result) + if calls["n"] == 1: + raise RuntimeError("interrupted mid-matrix") + + monkeypatch.setattr(run_eval, "RunStore", _DiesRightAfterFirstSave) + with pytest.raises(RuntimeError, match="interrupted"): + _run_ok_experiment(tmp_path, monkeypatch, subdir="ab2") + recs = [ + _json.loads(x) for x in (tmp_path / "runs" / "ab2-attempts.jsonl").read_text().splitlines() + ] + assert [r["kind"] for r in recs] == ["start", "end"] and recs[1]["aborted"] == "RuntimeError" + arts = sorted((tmp_path / "runs" / "ab2").glob("*.json")) + assert len(arts) == 2, "the raising save landed its artifact; the pool finished the sibling" + assert sorted(c["run_id"] for c in recs[1]["ok_cells"]) == [a.stem for a in arts] + assert all(c["sha256"] for c in recs[1]["ok_cells"]) + monkeypatch.setattr(run_eval, "RunStore", real_store) + saved = arts[0].read_bytes() + arts[0].unlink() # a persisted cell deleted before the rerun: refuse + with pytest.raises(AssertionError): + _run_ok_experiment(tmp_path, monkeypatch, subdir="ab2") + assert "resume integrity" in capsys.readouterr().err + d = _json.loads(saved) + d["review_markdown"] = "edited" + arts[0].write_text(_json.dumps(d)) # edited: refuse too + with pytest.raises(AssertionError): + _run_ok_experiment(tmp_path, monkeypatch, subdir="ab2") + assert "artifact changed" in capsys.readouterr().err + arts[0].write_bytes(saved) + # Preserved: the rerun resumes both cells (fully cached: no dispatch, no END). + _run_ok_experiment(tmp_path, monkeypatch, subdir="ab2") + recs = [ + _json.loads(x) for x in (tmp_path / "runs" / "ab2-attempts.jsonl").read_text().splitlines() + ] + assert [r["kind"] for r in recs] == ["start", "end", "start"] + assert arts[0].read_bytes() == saved + + +def test_coordinator_interruption_keeps_every_finished_review(tmp_path, monkeypatch): + """Two workers finish; the coordinator is interrupted while consuming the FIRST + future (the `done:` progress line raises). Both reviews were persisted by + their workers, so the aborted END record lists both and the rerun resumes + both without a dispatch.""" + import json as _json + + run_eval = _run_eval() + from eval_core.models import STRATUM_SYNTHETIC, Case, ReviewOutput + + _stub_run_git(monkeypatch, tmp_path) + monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) + monkeypatch.setattr( + run_eval.CorpusLoader, + "load_cases", + lambda self, strata: [Case(id="c", stratum=STRATUM_SYNTHETIC)], + raising=True, + ) + monkeypatch.setattr(run_eval.CorpusLoader, "verify", lambda self, case: None, raising=True) + + class _Ok: + def cli_version(self): + return _pinned_cli_version() + + def experiment_tag(self, config): + return f"tag-{config.id}" + + def case_tag(self, case): + return "ctag" + + def prompt_sha_for(self, case): + return "psha" + + def review(self, case, config, repeat_idx): + return ReviewOutput( + review_markdown="fine", + cli_version=_pinned_cli_version(), + latency_s=0.0, + usage={"prompt_sha": "psha"}, + ) + + monkeypatch.setattr(run_eval, "_build_reviewer", lambda repo_root: _Ok(), raising=True) + real_print = print + + def _interrupting_print(*a, **k): + if a and "done:" in str(a[0]): + raise KeyboardInterrupt("operator hit ^C while the coordinator consumed a future") + real_print(*a, **k) + + monkeypatch.setattr(run_eval, "print", _interrupting_print, raising=False) + with pytest.raises(KeyboardInterrupt): + run_eval.cmd_run(_ns(configs="A,B", strata=None, subdir="ki", k=1, max_parallel=2)) + monkeypatch.setattr(run_eval, "print", real_print, raising=False) + arts = sorted((tmp_path / "runs" / "ki").glob("*.json")) + assert len(arts) == 2, "both workers' reviews are on disk" + recs = [ + _json.loads(x) for x in (tmp_path / "runs" / "ki-attempts.jsonl").read_text().splitlines() + ] + assert [r["kind"] for r in recs] == ["start", "end"] + assert recs[1]["aborted"] == "KeyboardInterrupt" + assert sorted(c["run_id"] for c in recs[1]["ok_cells"]) == [a.stem for a in arts] + # The rerun resumes both (fully cached: no dispatch, no END record) and leaves + # the artifacts byte-identical. + before = [a.read_bytes() for a in arts] + assert run_eval.cmd_run(_ns(configs="A,B", strata=None, subdir="ki", k=1, max_parallel=2)) == 0 + assert [a.read_bytes() for a in arts] == before + recs = [ + _json.loads(x) for x in (tmp_path / "runs" / "ki-attempts.jsonl").read_text().splitlines() + ] + assert [r["kind"] for r in recs] == ["start", "end", "start"] + + +def test_resume_integrity_protects_the_cumulative_ok_cell_set(tmp_path, monkeypatch): + """A cell OK in an EARLIER END record stays protected when a later (aborted) + END record lists only the cells it persisted itself.""" + import hashlib as _hashlib + import json as _json + + run_eval = _run_eval() + runs = tmp_path / "runs" + (runs / "cum").mkdir(parents=True) + monkeypatch.setattr(run_eval, "RUNS_DIR", str(runs)) + x = runs / "cum" / "c.A.r0.x.json" + y = runs / "cum" / "c.B.r0.y.json" + x.write_text("{}") + y.write_text("{}") + + def cell(path, arm): + return { + "case": "c", + "arm": arm, + "repeat": 0, + "run_id": path.stem, + "sha256": _hashlib.sha256(path.read_bytes()).hexdigest(), + } + + start = {"kind": "start", "head": "h", "at": "t", "configs": ["A", "B"], "k": 1} + end1 = { + "kind": "end", + "started_at": "t", + "ok": 1, + "total": 2, + "failures": [], + "ok_cells": [cell(x, "A")], + } + end2 = { + "kind": "end", + "started_at": "t2", + "aborted": "RuntimeError", + "ok": None, + "total": None, + "failures": [], + "ok_cells": [cell(y, "B")], + } + (runs / "cum-attempts.jsonl").write_text( + "\n".join(_json.dumps(r) for r in (start, end1, start, end2)) + "\n" + ) + assert run_eval._resume_integrity_violations("cum", str(runs / "cum")) == [] + x.unlink() # listed only by the FIRST END record + probs = run_eval._resume_integrity_violations("cum", str(runs / "cum")) + assert len(probs) == 1 and "c/A/r0" in probs[0] and "missing" in probs[0] + + +def test_rerun_refuses_when_an_ok_artifact_is_missing_or_changed(tmp_path, monkeypatch, capsys): + import json as _json + + _run_ok_experiment(tmp_path, monkeypatch, subdir="res") + art = sorted((tmp_path / "runs" / "res").glob("*.json"))[0] + saved = art.read_bytes() + art.unlink() # a deleted OK artifact would be RE-DISPATCHED: refuse + with pytest.raises(AssertionError): # helper asserts rc == 0; the run returns 1 + _run_ok_experiment(tmp_path, monkeypatch, subdir="res") + assert "resume integrity" in capsys.readouterr().err + art.write_bytes(saved) + d = _json.loads(saved) + d["review_markdown"] = "edited" + art.write_text(_json.dumps(d)) # a changed OK artifact: refuse too + with pytest.raises(AssertionError): + _run_ok_experiment(tmp_path, monkeypatch, subdir="res") + assert "artifact changed" in capsys.readouterr().err + art.write_bytes(saved) + # A genuine infra-only resume that preserves every OK artifact proceeds (cached, no END). + _run_ok_experiment(tmp_path, monkeypatch, subdir="res") + recs = [ + _json.loads(x) for x in (tmp_path / "runs" / "res-attempts.jsonl").read_text().splitlines() + ] + assert [r["kind"] for r in recs] == ["start", "end", "start"] diff --git a/tests/test_openai_review.py b/tests/test_openai_review.py index afac89028..4c9158057 100644 --- a/tests/test_openai_review.py +++ b/tests/test_openai_review.py @@ -1663,10 +1663,22 @@ def test_gpt56_sol_timeout_resolves_to_reasoning(self, review_mod): assert review_mod._resolve_timeout(None, "gpt-5.6-sol") == review_mod.REASONING_TIMEOUT def test_gpt56_pricing_entries_present(self, review_mod): - """The three GPT-5.6 tiers priced per the 2026-07 OpenAI table (per 1M).""" - assert review_mod.PRICING["gpt-5.6-sol"] == (5.00, 30.00) - assert review_mod.PRICING["gpt-5.6-terra"] == (2.50, 15.00) - assert review_mod.PRICING["gpt-5.6-luna"] == (1.00, 6.00) + """The three GPT-5.6 tiers priced per the official table, re-verified + 2026-10-15 (standard tier, <=272K-input requests, per 1M tokens).""" + assert review_mod.PRICING["gpt-5.6-sol"] == (4.00, 20.00) + assert review_mod.PRICING["gpt-5.6-terra"] == (2.00, 12.00) + assert review_mod.PRICING["gpt-5.6-luna"] == (0.20, 1.20) + + def test_gpt6_pricing_entries_present(self, review_mod): + """The gpt-6 rows (2026-10-15) and estimate_cost resolving the candidate.""" + assert review_mod.PRICING["gpt-6-astra"] == (10.00, 50.00) + assert review_mod.PRICING["gpt-6-sol"] == (2.00, 10.00) + assert review_mod.PRICING["gpt-6-luna"] == (0.10, 0.50) + assert review_mod.estimate_cost(1_000_000, 0, "gpt-6-astra").startswith("$10.00") + + def test_gpt6_family_is_reasoning(self, review_mod): + assert review_mod._is_reasoning_model("gpt-6-astra") is True + assert review_mod._resolve_timeout(None, "gpt-6-astra") == review_mod.REASONING_TIMEOUT def test_default_model_is_gpt56_sol(self, review_mod): """DEFAULT_MODEL is the production reviewer pin (kept in lockstep with the @@ -2150,6 +2162,12 @@ class TestRustTestWorkflowPathFilter: ".github/workflows/ai_pr_review.yml", ".github/codex/prompts/pr_review.md", ".claude/scripts/openai_review.py", + # Production model-pin lockstep surfaces (TestWorkflowCodexActionContract + # reads them): a skill-only, command-doc-only or CLAUDE.md-only edit + # must still run the test job. + ".claude/skills/plan-review/**", + ".claude/commands/ai-review-local.md", + "CLAUDE.md", ) @pytest.fixture(scope="class") @@ -2696,6 +2714,78 @@ def test_compiled_prompt_path_agrees_between_build_and_action(self, workflow_tex ) assert consumer.group(1) == ".github/codex/prompts/pr_review_compiled.md" + # --- CLI / model / effort lockstep across every production surface --- + + @staticmethod + def _repo_root(): + assert _SCRIPT_PATH is not None + return _SCRIPT_PATH.parent.parent.parent + + def _configs(self): + import json as _json + + path = self._repo_root() / "tools" / "reviewer-eval" / "config" / "configs.json" + if not path.exists(): + pytest.skip("reviewer-eval harness not present") + return _json.loads(path.read_text()) + + def test_codex_version_pin_matches_harness_cli_version(self, workflow_text): + """The campaign CLI == the CI CLI: the Run Codex step's `codex-version` + must equal the version suffix of EVERY arm's `cli_version`.""" + block = self._require_block(workflow_text, self.RUN_CODEX_STEP) + m = re.search(r"^\s*codex-version:\s*(\d+\.\d+\.\d+)\s*$", block, re.MULTILINE) + assert m, "the Run Codex step must pin `codex-version: X.Y.Z`" + for arm in self._configs()["arms"]: + assert arm["cli_version"] == f"codex-cli {m.group(1)}", ( + f"arm {arm['id']} cli_version {arm['cli_version']!r} != workflow " + f"codex-version {m.group(1)} (bump both together, with a harness smoke)" + ) + + def test_model_and_effort_pins_in_lockstep(self, workflow_text, review_mod): + """Workflow `model` == DEFAULT_MODEL == plan-review CODEX_MODEL == the + configs.json control arm's model == every `gpt-N...` token in CLAUDE.md + and SKILL.md outside attribution lines; workflow `effort` == the control + arm's effort. In a swap PR the control flips with the pins; this test + is what forces the flip.""" + block = self._require_block(workflow_text, self.RUN_CODEX_STEP) + model = re.search(r"^\s*model:\s*(\S+)\s*$", block, re.MULTILINE) + effort = re.search(r"^\s*effort:\s*(\S+)\s*$", block, re.MULTILINE) + assert model and effort, "the Run Codex step must set `model:` and `effort:`" + assert model.group(1) == review_mod.DEFAULT_MODEL + root = self._repo_root() + skill_py = root / ".claude" / "skills" / "plan-review" / "codex_review.py" + if not skill_py.exists(): + pytest.skip("plan-review skill not present") + import importlib.util as _ilu + + spec = _ilu.spec_from_file_location("plan_review_codex_review", skill_py) + mod = _ilu.module_from_spec(spec) + spec.loader.exec_module(mod) + assert mod.CODEX_MODEL == review_mod.DEFAULT_MODEL + controls = [a for a in self._configs()["arms"] if a.get("role") == "control"] + assert len(controls) == 1 + assert controls[0]["model"] == review_mod.DEFAULT_MODEL + assert controls[0]["effort"] == effort.group(1) + exempt = ("measured", "run with", "historical", "previous", "pre-2026-10") + for rel in ("CLAUDE.md", ".claude/skills/plan-review/SKILL.md"): + text = (root / rel).read_text() + for line in text.splitlines(): + if any(w in line for w in exempt): + continue + for tok in re.findall(r"gpt-\d[\w.-]*", line): + assert ( + tok.rstrip(".,;:") == review_mod.DEFAULT_MODEL + ), f"{rel}: prose pin {tok!r} != DEFAULT_MODEL on a non-attribution line: {line!r}" + + def test_attribution_exemption_words_are_exactly_the_registered_set(self): + """'Campaign 1' is deliberately NOT an exemption word (CLAUDE.md carries + the LIVE pin on a 'Campaign 1' line); 'run with' is.""" + exempt = ("measured", "run with", "historical", "previous", "pre-2026-10") + checked = "codex `gpt-5.6-sol`, then merge/verify; Campaign 1 was exploratory" + skipped = "Campaign 1, run with gpt-5.6-sol" + assert not any(w in checked for w in exempt) + assert any(w in skipped for w in exempt) + # --- final-message output wiring (post-comment ref must match Codex id) --- def test_final_message_output_wired_to_codex_step(self, workflow_text): @@ -2983,6 +3073,181 @@ def test_effort_passthrough_e2e(self, review_mod, fake_subprocess): review_mod.call_codex("p", "gpt-5.6-sol", "/r", effort="max") assert "model_reasoning_effort=max" in fake_subprocess["cmd"] + # --- session-event parser (reviewer-eval audit input) --- + + @staticmethod + def _session(review_mod, fake_subprocess, stderr_text): + fake_subprocess["stderr_text"] = stderr_text + _, usage = review_mod.call_codex("p", "gpt-5.6-sol", "/r") + return usage + + def test_session_commands_two_blocks_plus_noise(self, review_mod, fake_subprocess): + stderr = ( + "user\nReview this\ncodex\nI will look.\nexec\n" + "/bin/zsh -lc 'ls -1 | head -3' in /tmp/wt\n succeeded in 0ms:\na.py\nb.py\n\n" + "codex\nNext.\nexec\n" + "/bin/zsh -lc \"sed -n '1,80p' x.py\" in /tmp/x\n succeeded in 3ms:\nline1\nline2\n" + "codex\nDone\ntokens used\n1,234\n" + ) + u = self._session(review_mod, fake_subprocess, stderr) + assert u["session_commands"] == [ + "/bin/zsh -lc 'ls -1 | head -3' in /tmp/wt", + "/bin/zsh -lc \"sed -n '1,80p' x.py\" in /tmp/x", + ] + assert u["session_commands_start_lines"] == 2 + assert u["session_commands_exec_lines"] == 2 + assert u["session_other_events"] == 0 + assert u["session_commands_truncated"] is False + + def test_session_commands_empty_when_no_exec(self, review_mod, fake_subprocess): + u = self._session(review_mod, fake_subprocess, "user\nhi\ncodex\nok\ntokens used\n12\n") + assert u["session_commands"] == [] and u["session_commands_start_lines"] == 0 + assert u["session_commands_exec_lines"] == 0 + + def test_session_commands_multiline_and_status_terminators(self, review_mod, fake_subprocess): + heredoc = "/bin/zsh -lc \"python - <<'EOF'\nprint(1)\nx = \"a' in /x\"\ny = 'b\\\" in /x'\nEOF\" in /tmp/x" + stderr = ( + f"exec\n{heredoc}\n exited 1 in 12ms:\nTraceback\n" + "exec\n/bin/zsh -lc 'false' in /tmp/x\n failed in 5ms:\n\n" + "exec\n/bin/zsh -lc 'sleep 9' in /tmp/x\n timed out in 1000ms:\n" + ) + u = self._session(review_mod, fake_subprocess, stderr) + assert u["session_commands"][0] == heredoc, "quote-ending body lines must not end the entry" + assert u["session_commands"][1] == "/bin/zsh -lc 'false' in /tmp/x" + assert u["session_commands"][2] == "/bin/zsh -lc 'sleep 9' in /tmp/x" + assert "Traceback" not in "".join(u["session_commands"]) + assert u["session_commands_exec_lines"] == 3 == u["session_commands_start_lines"] + + def test_session_commands_unterminated_closed_by_next_start(self, review_mod, fake_subprocess): + stderr = "exec\n/bin/zsh -lc 'a' in /x\noutput1\noutput2\nexec\n/bin/sh -lc 'b' in /x\n succeeded in 1ms:\n" + u = self._session(review_mod, fake_subprocess, stderr) + assert u["session_commands"] == [ + "/bin/zsh -lc 'a' in /x\noutput1\noutput2", + "/bin/sh -lc 'b' in /x", + ] + assert not u["session_commands"][0].endswith("[truncated]") + + def test_session_commands_cap_and_eos(self, review_mod, fake_subprocess): + body = "\n".join(f"l{i}" for i in range(300)) + u = self._session(review_mod, fake_subprocess, f"exec\n/bin/zsh -lc 'a' in /x\n{body}\n") + assert u["session_commands"][0].endswith("[truncated]") + assert "l250" not in u["session_commands"][0] and "l150" in u["session_commands"][0] + # A terminated 150-line heredoc is kept whole, no marker. + heredoc = ( + "/bin/zsh -lc 'cat < 0 and n == u["session_commands_start_lines"] == u["session_commands_exec_lines"] + assert u["session_other_events"] == expected_other + assert not any(c.endswith("[truncated]") for c in u["session_commands"]) + + def test_session_truncated_flag_when_stderr_drain_still_alive( + self, review_mod, fake_subprocess, monkeypatch + ): + import io as _io + import threading as _threading + import types as _types + + class AliveStderrThread(_threading.Thread): + def __init__(self, *a, **k): + super().__init__(*a, **k) + # Thread drops _args after run(); keep our own copy for is_alive(). + self._probe_args = tuple(k.get("args", ())) + + def is_alive(self): + return any(isinstance(a, _io.StringIO) for a in self._probe_args) + + monkeypatch.setattr( + review_mod, "threading", _types.SimpleNamespace(Thread=AliveStderrThread) + ) + u = self._session( + review_mod, fake_subprocess, "exec\n/bin/zsh -lc 'ls' in /x\n succeeded in 0ms:\n" + ) + assert u["session_commands_truncated"] is True + assert u["session_commands"] == ["/bin/zsh -lc 'ls' in /x"] + + def test_summary_line_printed_even_on_timeout(self, review_mod, monkeypatch, capsys): + import io as _io + import subprocess as _sp + + class HangingPopen: + def __init__(self, cmd, **kwargs): + self.returncode = None + self.stdin = _io.StringIO() + self.stdout = _io.StringIO("") + self.stderr = _io.StringIO("exec\n/bin/zsh -lc 'sleep 99' in /x\n") + self._killed = False + + def wait(self, timeout=None): + if timeout is not None and not self._killed: + raise _sp.TimeoutExpired(cmd="codex", timeout=timeout) + self.returncode = -9 + return self.returncode + + def terminate(self): + pass + + def kill(self): + self._killed = True + + monkeypatch.setattr(review_mod.subprocess, "Popen", HangingPopen) + with pytest.raises(RuntimeError, match="timed out after"): + review_mod.call_codex("p", "gpt-6-astra", "/r", timeout_s=1) + err = capsys.readouterr().err + assert ( + "session_commands: entries=1 start_lines=1 exec_lines=1 other_events=0 truncated=False" + in err + ) + def test_timeout_kills_process_and_raises(self, review_mod, monkeypatch): """timeout_s must kill the codex process and raise a RuntimeError (the eval harness turns it into a resumable INFRA_ERROR) — never propagate a @@ -3149,6 +3414,103 @@ def kill(self): review_mod.call_codex("p", "gpt-5.4", "/r") +class TestMainTimeoutForwarding: + """`main()`-level: `--timeout` reaches `call_codex` as `timeout_s` on the codex + backend (0 and omitted -> None; negatives rejected by argparse for both + backends). New scaffolding: no other test drives main() via sys.argv.""" + + @staticmethod + def _run_main(monkeypatch, tmp_path, review_mod, extra_argv): + import sys as _sys + + inputs = { + "--review-criteria": tmp_path / "criteria.md", + "--registry": tmp_path / "REGISTRY.md", + "--diff": tmp_path / "diff.patch", + "--changed-files": tmp_path / "files.txt", + } + inputs["--review-criteria"].write_text("# criteria\n") + inputs["--registry"].write_text("# registry\n") + inputs["--diff"].write_text( + "diff --git a/x.py b/x.py\n--- a/x.py\n+++ b/x.py\n@@ -1 +1 @@\n-a\n+b\n" + ) + inputs["--changed-files"].write_text("M\tx.py\n") + out = tmp_path / "out" / "review.md" + argv = ["openai_review.py"] + for flag, path in inputs.items(): + argv += [flag, str(path)] + argv += ["--output", str(out), "--repo-root", str(tmp_path)] + list(extra_argv) + recorded = {} + + def _fake_call_codex(prompt, model, repo_root, effort="xhigh", timeout_s=None): + recorded["timeout_s"] = timeout_s + recorded["model"] = model + return "## ok", {"backend": "codex"} + + monkeypatch.setattr(review_mod, "call_codex", _fake_call_codex) + monkeypatch.setattr(review_mod, "_detect_backend", lambda requested: "codex") + monkeypatch.setattr(review_mod, "_scan_sensitive_files", lambda root: []) + monkeypatch.setattr(review_mod, "_print_sensitive_notice", lambda *a, **k: None) + monkeypatch.setattr(_sys, "argv", argv) + rc = review_mod.main() + return rc, recorded + + def test_timeout_forwarded_to_codex(self, monkeypatch, tmp_path, review_mod): + rc, rec = self._run_main( + monkeypatch, tmp_path, review_mod, ["--backend", "codex", "--timeout", "7200"] + ) + assert rc in (0, None) and rec["timeout_s"] == 7200 + + def test_omitted_and_zero_mean_no_timeout(self, monkeypatch, tmp_path, review_mod): + _, rec = self._run_main(monkeypatch, tmp_path, review_mod, ["--backend", "codex"]) + assert rec["timeout_s"] is None + _, rec = self._run_main( + monkeypatch, tmp_path, review_mod, ["--backend", "codex", "--timeout", "0"] + ) + assert rec["timeout_s"] is None + + @pytest.mark.parametrize("backend", ["codex", "api"]) + def test_negative_timeout_rejected_before_any_call( + self, monkeypatch, tmp_path, review_mod, backend + ): + with pytest.raises(SystemExit) as exc: + self._run_main( + monkeypatch, tmp_path, review_mod, ["--backend", backend, "--timeout", "-5"] + ) + assert exc.value.code == 2 # argparse usage error + + def test_call_openai_zero_timeout_opens_with_none(self, monkeypatch, review_mod): + import io as _io + import json as _json + + seen = {} + + class _Resp(_io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def _fake_urlopen(req, timeout="unset"): + seen["timeout"] = timeout + body = _json.dumps( + { + "output": [ + {"type": "message", "content": [{"type": "output_text", "text": "ok"}]} + ], + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ) + return _Resp(body.encode()) + + monkeypatch.setattr(review_mod.urllib.request, "urlopen", _fake_urlopen) + review_mod.call_openai("p", "gpt-4.1", "sk-test", timeout=0) + assert seen["timeout"] is None + review_mod.call_openai("p", "gpt-4.1", "sk-test", timeout=30) + assert seen["timeout"] == 30 + + class TestCodexBackendDocConsistency: """The skill doc must enumerate the backend choices that the script actually accepts, and explain the codex install + auth requirement.""" diff --git a/tests/test_plan_review_skill.py b/tests/test_plan_review_skill.py index 305a9635e..8c3b43297 100644 --- a/tests/test_plan_review_skill.py +++ b/tests/test_plan_review_skill.py @@ -146,6 +146,23 @@ def test_skill_md_present_with_frontmatter(): assert re.search(r"^description:\s*\S", fm, re.M) +def test_codex_step_tells_caller_to_run_in_background(): + """The codex call runs 20-40 min; the Bash tool kills foreground calls at + 600 s. Step 4's prose (outside the blockquote and the fence) must say so.""" + text = (_SKILL / "SKILL.md").read_text() + start = text.index("### 4.") + end = text.index("### 5.", start) + section = text[start:end] + sentence = ( + "Run this Bash call with `run_in_background: true` and act on its completion " + "notification by following the exit-code bullets below; the Bash tool caps " + "foreground calls at 600 s, below the codex ceiling." + ) + assert " ".join(sentence.split()) in " ".join(section.split()) + # Outside the fence: the sentence precedes the ```bash block. + assert section.index("Run this Bash call") < section.index("```bash") + + def test_reviewer_invocations_are_pinned(): """Identical templates + a wrong model/effort/timeout would pass the byte-match silently. The codex pins live at the real `call_codex` site diff --git a/tests/test_reviewer_audit.py b/tests/test_reviewer_audit.py new file mode 100644 index 000000000..1fa5f0b30 --- /dev/null +++ b/tests/test_reviewer_audit.py @@ -0,0 +1,554 @@ +"""Tests for tools/reviewer-eval/reviewer_audit.py (the session-log audit). + +Mirrors test_evals_runtime.py's isolation: module-level skip when the harness is +absent, sys.path insert guarded the same way, and NO top-level import of the +module under test (pytestmark applies only after import; a top-level import +would still raise at collection on the isolated /tmp/tests CI legs). +""" + +import functools +import importlib.util +import json +import os +import pathlib +import sys + +import pytest + +_REPO = pathlib.Path(__file__).resolve().parent.parent +_EVAL_ROOT = _REPO / "tools" / "reviewer-eval" + +pytestmark = pytest.mark.skipif( + not _EVAL_ROOT.exists(), + reason="reviewer-eval harness not present (isolated install)", +) + +if _EVAL_ROOT.exists() and str(_EVAL_ROOT) not in sys.path: + sys.path.insert(0, str(_EVAL_ROOT)) +if str(_REPO / "tools") not in sys.path: + sys.path.insert(0, str(_REPO / "tools")) + + +@functools.lru_cache(maxsize=1) +def _audit(): + import reviewer_audit + + return reviewer_audit + + +@functools.lru_cache(maxsize=1) +def _run_eval(): + spec = importlib.util.spec_from_file_location( + "reviewer_eval_run_eval", _EVAL_ROOT / "run_eval.py" + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +WT = "/Users/igerber/diff-diff-codex-upgrade/tools/reviewer-eval/runs/.worktrees/0123456789abcdef0123456789abcdef-0123456789abcdef" +ROOT = "/Users/igerber/diff-diff-codex-upgrade" +MARKERS = { + "repo_branch": "codex-upgrade", + "repo_common_dir": "/Users/igerber/diff-diff/.git", + "repo_worktree_paths": ["/Users/igerber/diff-diff-contributions"], + "repo_ref_names": [ + "contributions", + "eval-pins/pr590-r1", + "origin/main", + "origin", + "v3.12.0", + "codex-upgrade", + ], + "codex_home_path": None, +} + + +def _hits(cmd, markers=MARKERS, root=ROOT, literals=None, wt=WT): + a = _audit() + stripped = a.strip_own_path(cmd, wt, root) + pm = a._path_markers(root, markers) + rr = a._ref_name_regex(markers.get("repo_ref_names") or [], markers.get("repo_branch") or "") + kw = {} if literals is None else {"literals": literals} + return a.scan_entry(stripped, root, markers, pm, rr, **kw) + + +def _wrapped(cmd, quote="'"): + return f"/bin/zsh -lc {quote}{cmd}{quote} in {WT}" + + +# --------------------------------------------------------------------------- # +# Own-path substitution +# --------------------------------------------------------------------------- # + + +def test_strip_own_path_all_forms_and_order(): + a = _audit() + rel = WT[len(ROOT) + 1 :] + leaf = os.path.basename(WT) + for form in (WT, rel, leaf, os.path.realpath(WT)): + out = a.strip_own_path(f"cat {form}/x.py", WT, ROOT) + assert out == "cat /x.py", (form, out) + # ABSOLUTE form leaves no / residue (pins realpath -> absolute -> relative -> leaf). + assert a.strip_own_path(f"grep -rn x {WT}", WT, ROOT) == "grep -rn x " + assert "" in a.strip_own_path( + f"ls ~/{WT[len(os.path.expanduser('~')) + 1:]}", WT, ROOT + ) or not WT.startswith(os.path.expanduser("~")) + # Continuations are joined before scanning. + assert a.strip_own_path("git diff HEAD \\\n main", WT, ROOT) == "git diff HEAD main" + # Nothing else is touched. + assert a.strip_own_path("echo hello", WT, ROOT) == "echo hello" + + +def test_own_worktree_git_commands_still_hit_after_substitution(): + assert "GIT_LOG_RE" in _hits(f"git -C {WT} log -1") + assert "BARE_SHOW_RE" in _hits(f"git -C {WT} show HEAD") + assert "repo_root" not in _hits(f"cat {WT}/diff_diff/x.py") + + +# --------------------------------------------------------------------------- # +# Literals +# --------------------------------------------------------------------------- # + + +def test_every_literal_hits(): + a = _audit() + for lit in a.LEAK_MARKERS: + assert lit in _hits(f"x {lit} y"), lit + + +@pytest.mark.parametrize( + "cmd, marker", + [ + ('grep -rl slug "${PWD%/tools/*}"', "${PWD"), + ('find "$(dirname "$PWD")" -name case.json', "dirname"), + ("grep -rn slug /Users/igerber --include=*.json", "--include=*.json"), + ("g=git; $g show main:x.py", "=git"), + ("git format-patch -1 --stdout", "format-patch"), + ("git shortlog HEAD", "shortlog"), + ("git whatchanged -1", "whatchanged"), + ("git log @~1..@", "@~"), + ("git status --porcelain=v2 --branch", "porcelain=v2"), + ("grep branch.oid out.txt", "branch.oid"), + ("git ls-remote .", "ls-remote"), + ("cd ..", "cd .."), + ("git rev-parse HEAD", "rev-parse"), + ("git diff HEAD campaign-gpt6 -- x.py", "campaign-"), + ("git describe --always", "describe"), + ("git name-rev HEAD", "name-rev"), + ("git log --walk-reflogs", "--walk-reflogs"), + ("git diff HEAD^..HEAD", "HEAD^"), + ("rm -r build", " -r "), + ], +) +def test_literal_examples(cmd, marker): + assert marker in _hits(cmd) + assert marker in _hits(_wrapped(cmd)) and marker in _hits(_wrapped(cmd, '"')) + + +# --------------------------------------------------------------------------- # +# The seven regexes +# --------------------------------------------------------------------------- # + +GIT_LOG_HITS = [ + "git log", + "git log -1", + "git log abc1234..def5678", + "git log 0123456789ab", + "git log -p HEAD~3..", + "git log HEAD@{1}..HEAD", + "git log --oneline base..head", + "git log origin/main..HEAD", + "git log -n 1000000", + "git log --grep='eval:..*'", + "git log -S cafe1234", + "git --no-optional-locks log", + "git log @^..@", + "git log -n 3 --stat && git diff abc1234..def5678", + "git -C /x log -1", + "git --git-dir=/x/.git log", + "git --no-replace-objects log -1", + "git --exec-path=/x --no-advice log", + "git -c core.pager=cat log", +] +GIT_LOG_MISSES = [ + "git diff abc1234..def5678", + "git diff --stat base..head", + "git logs-dir", + "cat changelog.md", + "git diff HEAD..main", +] + + +@pytest.mark.parametrize("cmd", GIT_LOG_HITS) +def test_git_log_re_hits(cmd): + for form in (cmd, _wrapped(cmd), _wrapped(cmd, '"')): + assert "GIT_LOG_RE" in _hits(form), form + + +@pytest.mark.parametrize("cmd", GIT_LOG_MISSES) +def test_git_log_re_misses(cmd): + for form in (cmd, _wrapped(cmd), _wrapped(cmd, '"')): + assert "GIT_LOG_RE" not in _hits(form), form + + +MAIN_HITS = [ + "git diff HEAD main", + "git log -p main", + "git checkout main", + "git -C /x checkout main", + "git --no-pager diff main", + "git diff HEAD..main", + "git diff main...HEAD", + "git log -p HEAD..main", + "git log master..HEAD", + "git show main:diff_diff/x.py", + "git diff HEAD heads/main", + "git diff HEAD refs/heads/main", + "git diff HEAD remotes/origin/main", + "git --no-replace-objects diff HEAD..main", +] +MAIN_MISSES = [ + "git diff HEAD~1", + "ls domain.py", + "git diff -- main.py", + "git log -- src/main/", + "ls maintenance.md", + "git diff HEAD", +] + + +@pytest.mark.parametrize("cmd", MAIN_HITS) +def test_main_ref_re_hits(cmd): + for form in (cmd, _wrapped(cmd), _wrapped(cmd, '"')): + assert "MAIN_REF_RE" in _hits(form), form + + +@pytest.mark.parametrize("cmd", MAIN_MISSES) +def test_main_ref_re_misses(cmd): + for form in (cmd, _wrapped(cmd), _wrapped(cmd, '"')): + assert "MAIN_REF_RE" not in _hits(form), form + + +def test_main_ref_two_line_entry_only_with_continuation(): + assert "MAIN_REF_RE" not in _hits("git diff HEAD\nmain") + assert "MAIN_REF_RE" in _hits("git diff HEAD \\\n main") + + +@pytest.mark.parametrize( + "cmd", + [ + "git show", + "git show --stat", + "git -C /x show", + "git --no-pager show", + "git --no-replace-objects show HEAD", + "git --git-dir /x show HEAD", + ], +) +def test_bare_show_re_hits(cmd): + assert "BARE_SHOW_RE" in _hits(cmd) and "BARE_SHOW_RE" in _hits(_wrapped(cmd)) + + +@pytest.mark.parametrize("cmd", ["git showcase", "github show", "git status"]) +def test_bare_show_re_misses(cmd): + assert "BARE_SHOW_RE" not in _hits(cmd) + + +@pytest.mark.parametrize( + "cmd", + [ + "git tag", + "git tag -l", + "git branch --show-current", + f"git -C {WT} branch -a", + "git --no-pager tag", + "git --git-dir=/x/.git branch", + ], +) +def test_ref_list_re_hits(cmd): + assert "REF_LIST_RE" in _hits(cmd) and "REF_LIST_RE" in _hits(_wrapped(cmd)) + + +@pytest.mark.parametrize("cmd", ["git tagger", "cat branch.py"]) +def test_ref_list_re_misses(cmd): + assert "REF_LIST_RE" not in _hits(cmd) + + +@pytest.mark.parametrize( + "cmd", ["rg secret ..", "ls ..", "find .. -name '*.json'", "diff -r diff_diff ..", "cd ..; ls"] +) +def test_dotdot_re_hits(cmd): + assert "DOTDOT_RE" in _hits(cmd) and "DOTDOT_RE" in _hits(_wrapped(cmd, '"')) + + +@pytest.mark.parametrize( + "cmd", + [ + "git diff base..head", + "git log HEAD~3..", + "git diff main...HEAD", + "git log --grep='eval:..*'", + "cat ../x", + ], +) +def test_dotdot_re_misses(cmd): + assert "DOTDOT_RE" not in _hits(cmd) + + +@pytest.mark.parametrize( + "cmd", ["cat .git", "p=$(cut -d' ' -f2 .git)", "cat .git/HEAD", "cat ../.git/worktrees/x/HEAD"] +) +def test_git_dir_re_hits(cmd): + assert "GIT_DIR_RE" in _hits(cmd) and "GIT_DIR_RE" in _hits(_wrapped(cmd, '"')) + + +@pytest.mark.parametrize("cmd", ["cat .gitignore", "ls .github", "git status"]) +def test_git_dir_re_misses(cmd): + assert "GIT_DIR_RE" not in _hits(cmd) + + +@pytest.mark.parametrize( + "cmd", + [ + "git diff HEAD v3.12.0 -- diff_diff/x.py", + "git grep pat v3.12.0", + "git ls-tree v3.12.0", + "git -C /x show v3.9.1:x.py", + "git diff HEAD tags/v3.12.0", + "git diff HEAD refs/tags/v3.12.0 -- f", + "git diff HEAD..v3.12.0", + ], +) +def test_tag_ref_re_hits(cmd): + assert "TAG_REF_RE" in _hits(cmd) and "TAG_REF_RE" in _hits(_wrapped(cmd)) + + +@pytest.mark.parametrize( + "cmd", ["git diff HEAD~1", "pip install diff-diff==3.12.0", "cat v3.12.0.md"] +) +def test_tag_ref_re_misses(cmd): + assert "TAG_REF_RE" not in _hits(cmd) + + +# --------------------------------------------------------------------------- # +# Computed markers +# --------------------------------------------------------------------------- # + + +def test_computed_path_markers_and_longest_attribution(): + assert _hits("cat /Users/igerber/diff-diff/.git/worktrees/x/HEAD") == [ + "GIT_DIR_RE", + "worktrees/", + "repo_common_dir", + ] or "repo_common_dir" in _hits("cat /Users/igerber/diff-diff/.git/worktrees/x/HEAD") + assert "repo_main_checkout" in _hits("sed -n 1,80p /Users/igerber/diff-diff/diff_diff/utils.py") + h = _hits("cat /Users/igerber/diff-diff-contributions/x") + assert "repo_worktree_path" in h and "repo_main_checkout" not in h, h + assert "repo_root" in _hits(f"git -C {ROOT} grep slug") + assert "repo_root" in _hits(f"grep -rn slug {ROOT}/tests") + assert "repo_root" not in _hits(f"cat {WT}/x.py") + # Boundary: the primary checkout path is a prefix of repo_root's path but must not match inside it. + assert "repo_main_checkout" not in _hits(f"cat {ROOT}/x.py") + + +def test_home_relative_forms_derive_from_recorded_paths(monkeypatch): + monkeypatch.setenv("HOME", "/Users/igerber") + import importlib + + importlib.reload(_audit()) + assert "repo_main_checkout" in _hits("sed -n 1,80p ~/diff-diff/diff_diff/utils.py") + assert "repo_worktree_path" in _hits("ls $HOME/diff-diff-contributions") + + +def test_union_worktree_marker_from_one_artifact_only(tmp_path): + """A path present only in ONE artifact's repo_worktree_paths is a marker.""" + from eval_core.models import RunResult + from eval_core.store import RunStore + + a = _audit() + store = RunStore(str(tmp_path / "s")) + u1 = _usage( + ["/bin/zsh -lc 'ls /tmp/other-wt/x' in " + WT], + extra={"repo_worktree_paths": ["/tmp/other-wt"]}, + ) + u2 = _usage([], start=0, execs=0) + r1 = RunResult(case_id="c", config_id="B", repeat_idx=0, usage=u1, run_id="r1") + r2 = RunResult(case_id="c", config_id="E", repeat_idx=0, usage=u2, run_id="r2") + store.save("r1", r1) + store.save("r2", r2) + sealed = a.artifacts_sha(store.root, ["r1", "r2"]) + out = a.audit( + [r1, r2], store.root, ROOT, {**MARKERS, "repo_worktree_paths": []}, "rsha", sealed + ) + assert any("repo_worktree_path" in h["markers"] for h in out["hits"]) + assert out["coverage"] == {"covered": 1, "no_commands": 1, "unaudited": 0, "total": 2} + assert ( + out["vacuous"] is False + and out["run_ids_sha"] == "rsha" + and out["branch"] == "codex-upgrade" + ) + + +def test_ref_name_marker_is_git_scoped(): + for cmd in ( + "git diff HEAD contributions -- f", + "git log eval-pins/pr590-r1 -1", + "git diff HEAD..contributions", + ): + assert "repo_ref_name" in _hits(cmd), cmd + for cmd in ("rg contributions src/", "echo eval-pins/pr590-r1", "git diff HEAD origin"): + assert "repo_ref_name" not in _hits(cmd), cmd + + +def test_branch_marker_is_git_scoped(): + assert "repo_branch" in _hits("git diff HEAD codex-upgrade") + assert _hits("git diff HEAD") == [] + assert "repo_branch" not in _hits("echo codex-upgrade") + + +def test_codex_home_marker_without_literal(): + a = _audit() + literals = tuple(m for m in a.LEAK_MARKERS if m != "codex-home-") + markers = {**MARKERS, "codex_home_path": "/tmp/codex-home-abc"} + assert _hits("cat /tmp/codex-home-abc/history.jsonl", markers=markers, literals=literals) == [ + "codex_home_path" + ] + + +# --------------------------------------------------------------------------- # +# Coverage classes + orientation + rubric rendering +# --------------------------------------------------------------------------- # + + +def _usage(cmds, start=None, execs=None, other=0, truncated=False, isolated=True, extra=None): + u = { + "session_commands": cmds, + "session_commands_start_lines": len(cmds) if start is None else start, + "session_commands_exec_lines": len(cmds) if execs is None else execs, + "session_other_events": other, + "session_commands_truncated": truncated, + "codex_home_isolated": isolated, + "worktree_dir": WT, + } + if extra: + u.update(extra) + return u + + +def test_coverage_classes_exact_definition(): + cc = _audit().coverage_class + assert cc({}) == "unaudited" # absent key + assert cc(_usage([])) == "no_commands" + assert cc(_usage([], start=1)) == "unaudited" + assert cc(_usage([], execs=1)) == "unaudited" + u = _usage([]) + del u["session_commands_exec_lines"] + assert cc(u) == "unaudited" + assert cc(_usage(["/bin/zsh -lc 'ls' in /x"])) == "covered" + assert cc(_usage(["/bin/zsh -lc 'ls' in /x"], isolated=False)) == "unaudited" + u = _usage(["/bin/zsh -lc 'ls' in /x"]) + del u["codex_home_isolated"] + assert cc(u) == "unaudited" + assert cc(_usage(["/bin/zsh -lc 'ls' in /x"], other=2)) == "unaudited" + assert cc(_usage(["/bin/zsh -lc 'ls' in /x\n[truncated]"])) == "unaudited" + assert cc(_usage(["/bin/zsh -lc 'ls' in /x"], truncated=True)) == "unaudited" + # Truncation state must be an explicit False: missing / None / 1 are unknown -> unaudited. + u = _usage(["/bin/zsh -lc 'ls' in /x"]) + del u["session_commands_truncated"] + assert cc(u) == "unaudited" + assert cc(_usage(["/bin/zsh -lc 'ls' in /x"], truncated=None)) == "unaudited" + assert cc(_usage(["/bin/zsh -lc 'ls' in /x"], truncated=1)) == "unaudited" + assert cc(_usage([], truncated=None)) == "unaudited" + # exec_lines == 2, start_lines == 1 (a command in a format the start regex missed). + assert cc(_usage(["/bin/zsh -lc 'ls' in /x"], execs=2)) == "unaudited" + + +def test_orientation_only_definition(): + a = _audit() + assert a.ORIENTATION_MARKERS <= set(a.LEAK_MARKERS) | {n for n, _ in a.REGEX_RULES} + assert set(_hits("git log -1")) <= a.ORIENTATION_MARKERS + assert set(_hits(f"git -C {WT} log -1")) <= a.ORIENTATION_MARKERS + assert not set(_hits("git log --all")) <= a.ORIENTATION_MARKERS + assert not set(_hits("git log main")) <= a.ORIENTATION_MARKERS + assert not set(_hits("cat ../x")) <= a.ORIENTATION_MARKERS + + +def test_audit_vacuous_and_failed_manifest_refusal(tmp_path, monkeypatch): + from eval_core.models import RunResult + from eval_core.store import RunStore + + a = _audit() + store = RunStore(str(tmp_path / "s")) + r = RunResult( + case_id="c", + config_id="B", + repeat_idx=0, + usage=_usage(["/bin/zsh -lc 'ls' in /x\n[truncated]"]), + run_id="r", + ) + store.save("r", r) + sealed = a.artifacts_sha(store.root, ["r"]) + out = a.audit([r], store.root, ROOT, MARKERS, "rsha", sealed) + assert out["vacuous"] is True and out["coverage"]["unaudited"] == 1 + assert out["unaudited"] == [{"case": "c", "arm": "B", "repeat": 0}] + with pytest.raises(ValueError, match="artifacts changed"): + a.audit([r], store.root, ROOT, MARKERS, "rsha", "0" * 64) + with pytest.raises(FileNotFoundError): + a.artifacts_sha(store.root, ["missing"]) + # The CLI refuses a failed manifest (traversal refused too). + run_eval = _run_eval() + monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) + (tmp_path / "runs").mkdir() + (tmp_path / "runs" / "f-manifest.json").write_text(json.dumps({"failed": True})) + with pytest.raises(SystemExit, match="failed/incomplete"): + run_eval.cmd_audit(type("N", (), {"subdir": "f", "output": str(tmp_path / "h.json")})()) + assert ( + run_eval.cmd_audit(type("N", (), {"subdir": "../x", "output": str(tmp_path / "h.json")})()) + == 1 + ) + + +def test_rubric_renders_exactly_leak_rules(): + a = _audit() + rubric = (_EVAL_ROOT / "GRADING_RUBRIC.md").read_text() + begin, end = "", "" + assert begin in rubric and end in rubric + rendered = rubric.split(begin, 1)[1].split(end, 1)[0].strip("\n") + assert rendered == a.render_rules_for_rubric().strip("\n") + # One source, two renderings: every literal and every regex source appears, in order. + pos = -1 + for kind, src in a.LEAK_RULES: + needle = f"`{src}`" if kind == "literal" else f"`{kind}` = `{src}`" + nxt = rendered.find(needle, pos + 1) + assert nxt > pos, (kind, src) + pos = nxt + + +def test_rubric_hygiene_no_arm_identities(): + import re + + rubric = (_EVAL_ROOT / "GRADING_RUBRIC.md").read_text() + low = rubric.lower() + cfg = json.loads((_EVAL_ROOT / "config" / "configs.json").read_text()) + for arm in cfg["arms"]: + assert arm["model"].lower() not in low + for word in ( + "gpt-", + "astra", + "sol", + "terra", + "luna", + "no-go", + "parity", + "2026-07", + "arm b", + "arm e", + ): + assert not re.search(r"\b" + re.escape(word) + r"\b", low), word + for sentence in ( + 'The "_optional_ (must_catch=false)" tag in the bundle is a legacy label with no grading meaning: grade every listed bug in full.', + "This rubric's one-row-per-(case, bug, arm label, repeat) table overrides the bundle's \"How to grade\" row format.", + "On a negative-control case, list EVERY finding whose severity is above the allowed severities, including findings on a known-FP topic; the engine, not the grader, decides what counts.", + ): + assert " ".join(sentence.split()) in " ".join(rubric.split()), sentence diff --git a/tests/test_reviewer_verdict.py b/tests/test_reviewer_verdict.py new file mode 100644 index 000000000..84a4b442b --- /dev/null +++ b/tests/test_reviewer_verdict.py @@ -0,0 +1,1090 @@ +"""Tests for tools/reviewer-eval/reviewer_verdict.py and the `verdict` CLI. + +Isolation pattern copied from test_evals_runtime.py (module-level skip, guarded +sys.path insert, no top-level import of the module under test). +""" + +import functools +import importlib.util +import json +import pathlib +import sys + +import pytest + +_REPO = pathlib.Path(__file__).resolve().parent.parent +_EVAL_ROOT = _REPO / "tools" / "reviewer-eval" + +pytestmark = pytest.mark.skipif( + not _EVAL_ROOT.exists(), + reason="reviewer-eval harness not present (isolated install)", +) + +if _EVAL_ROOT.exists() and str(_EVAL_ROOT) not in sys.path: + sys.path.insert(0, str(_EVAL_ROOT)) +if str(_REPO / "tools") not in sys.path: + sys.path.insert(0, str(_REPO / "tools")) + + +@functools.lru_cache(maxsize=1) +def _v(): + import reviewer_verdict + + return reviewer_verdict + + +@functools.lru_cache(maxsize=1) +def _run_eval(): + spec = importlib.util.spec_from_file_location( + "reviewer_eval_run_eval", _EVAL_ROOT / "run_eval.py" + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +# --------------------------------------------------------------------------- # +# Synthetic k=2 tables +# --------------------------------------------------------------------------- # + + +def _run( + case, arm, rep, stratum="s1_synthetic", bugs=("b1",), allow=("P3",), must=True, infra=None +): + return { + "case_id": case, + "config_id": arm, + "repeat_idx": rep, + "infra_error": infra, + "run_id": f"{case}.{arm}.r{rep}", + "case_snapshot": { + "stratum": stratum, + "ground_truth": [{"id": b, "must_catch": must} for b in bugs], + "allow_severities": list(allow), + }, + } + + +def _runs(cases=(("c1", ("b1",)),), s3=("s3",), k=2, arms=("B", "E")): + out = [] + for case, bugs in cases: + for a in arms: + for r in range(k): + out.append(_run(case, a, r, bugs=bugs)) + for case in s3: + for a in arms: + for r in range(k): + out.append(_run(case, a, r, stratum="s3_negative", bugs=())) + return out + + +def _row(case, bug, arm, rep, verd, leak="none", flag="none", ev="", notes=""): + return { + "case": case, + "bug_id": bug, + "arm": arm, + "repeat": rep, + "verdict": verd, + "evidence": "quote" if verd == "caught" else "", + "leak": leak, + "grader_flag": flag, + "leak_evidence": ev, + "notes": notes, + } + + +def _neg(case, arm, rep, findings=(), leak="none", flag="none", ev="", notes=""): + return { + "case": case, + "arm": arm, + "repeat": rep, + "findings": list(findings), + "leak": leak, + "grader_flag": flag, + "leak_evidence": ev, + "notes": notes, + } + + +def _table( + b=("caught", "caught"), + e=("caught", "caught"), + case="c1", + bug="b1", + s3=("s3",), + b_fp=(), + e_fp=(), +): + rows = [_row(case, bug, "B", i, v) for i, v in enumerate(b)] + [ + _row(case, bug, "E", i, v) for i, v in enumerate(e) + ] + negs = [] + for c in s3: + for r in (0, 1): + negs.append(_neg(c, "B", r, b_fp if r == 0 else ())) + negs.append(_neg(c, "E", r, e_fp if r == 0 else ())) + return {"rows": rows, "negative_assessments": negs} + + +def _gates(table, runs=None, audit=None): + return _v().gates(table, runs or _runs(), "B", "E", audit) + + +# --------------------------------------------------------------------------- # +# Gates +# --------------------------------------------------------------------------- # + + +def test_labels_and_precedence(): + v = _v() + assert _gates(_table(b=("caught", "caught"), e=("missed", "missed")))["verdict"] == v.NO_GO + # Regression on a must_catch=false bug is still NO-GO. + runs = [dict(r) for r in _runs()] + for r in runs: + if r["case_id"] == "c1": + r["case_snapshot"] = { + **r["case_snapshot"], + "ground_truth": [{"id": "b1", "must_catch": False}], + } + assert _v().gates(_table(e=("missed", "missed")), runs, "B", "E")["verdict"] == v.NO_GO + # partial == missed: B partial both + E caught both -> improvement -> GO. + assert _gates(_table(b=("partial", "partial")))["verdict"] == v.GO + # B caught both + E partial both -> regression. + assert _gates(_table(e=("partial", "partial")))["verdict"] == v.NO_GO + # B caught both + E caught/partial -> one downgrade -> PARITY (no improvement). + r = _gates(_table(e=("caught", "partial"))) + assert r["verdict"] == v.PARITY and r["downgrades"] == [{"case": "c1", "bug_id": "b1"}] + # FP excess -> NO-GO; P3 on an allow-P3 case is not an FP, P2 is. + assert _gates(_table(e_fp=[{"severity": "P3", "summary": "x"}]))["verdict"] == v.PARITY + assert _gates(_table(e_fp=[{"severity": "P2", "summary": "x"}]))["verdict"] == v.NO_GO + # GO via fewer FPs. + assert _gates(_table(b_fp=[{"severity": "P1", "summary": "x"}]))["verdict"] == v.GO + # 2 downgrades + 1 improvement -> PARITY; regression + 2 downgrades -> NO-GO. + runs3 = _runs(cases=(("c1", ("b1",)), ("c2", ("b1",)), ("c3", ("b1",)))) + t = {"rows": [], "negative_assessments": _table()["negative_assessments"]} + t["rows"] += [_row("c1", "b1", "B", i, "caught") for i in (0, 1)] + [ + _row("c1", "b1", "E", 0, "caught"), + _row("c1", "b1", "E", 1, "missed"), + ] + t["rows"] += [_row("c2", "b1", "B", i, "caught") for i in (0, 1)] + [ + _row("c2", "b1", "E", 0, "caught"), + _row("c2", "b1", "E", 1, "missed"), + ] + t["rows"] += [_row("c3", "b1", "B", i, "missed") for i in (0, 1)] + [ + _row("c3", "b1", "E", i, "caught") for i in (0, 1) + ] + res = v.gates(t, runs3, "B", "E") + assert ( + res["verdict"] == v.PARITY and len(res["downgrades"]) == 2 and len(res["improvements"]) == 1 + ) + # Turn c3 into a regression (B caught both, E missed both): regression + 2 downgrades -> NO-GO. + t["rows"] = [r for r in t["rows"] if r["case"] != "c3"] + t["rows"] += [_row("c3", "b1", "B", i, "caught") for i in (0, 1)] + [ + _row("c3", "b1", "E", i, "missed") for i in (0, 1) + ] + assert v.gates(t, runs3, "B", "E")["verdict"] == v.NO_GO + + +def test_judgment_flag_b_unstable_e_missed(): + r = _gates(_table(b=("caught", "missed"), e=("missed", "missed"))) + assert r["judgment_flags"] == [{"bug": "c1|b1", "kind": "b_unstable_e_missed"}] + assert r["verdict"] == _v().PARITY + assert _gates(_table(b=("caught", "caught"), e=("missed", "missed")))["judgment_flags"] == [] + + +def test_undetermined_on_infra_short_exposure(): + runs = _runs() + runs[1] = _run("c1", "B", 1, infra="boom") + t = _table(b=("caught",)) + t["rows"] = [r for r in t["rows"] if not (r["arm"] == "B" and r["repeat"] == 1)] + assert _v().gates(t, runs, "B", "E")["verdict"] == _v().UNDETERMINED + + +def test_void_cells_contest_and_statuses(): + v = _v() + # One void B repeat on a bug case -> B at most UNSTABLE + contested, not UNDETERMINED. + t = _table(b=("caught", "caught"), e=("missed", "missed")) + t["rows"][0]["leak"] = "confirmed" + t["rows"][0]["leak_evidence"] = "git log --all" + r = _gates(t) + assert r["catch_status"]["c1|b1|B"] == v.UNSTABLE + assert r["verdict"] != v.UNDETERMINED + assert sorted(r["contested_gates"]) == ["downgrade", "improvement", "regression"] + assert r["voided_cells"] == [{"case": "c1", "arm": "B", "repeat": 0}] + # Both repeats void -> UNGRADEABLE + contested. + t["rows"][1]["leak"] = "confirmed" + t["rows"][1]["leak_evidence"] = "git log --all" + r = _gates(t) + assert r["catch_status"]["c1|b1|B"] == v.UNGRADEABLE + assert any("ungradeable" in c["reason"] for c in r["contested"]) + # Confirmed leak on an S3 repeat -> its FP list dropped + gate 2 contested. + t = _table(e_fp=[{"severity": "P1", "summary": "bad"}]) + cell = next(c for c in t["negative_assessments"] if c["arm"] == "E" and c["repeat"] == 0) + cell["leak"] = "confirmed" + cell["leak_evidence"] = "git show main:x" + r = _gates(t) + assert r["fp_candidate"] == 0 and "fp" in r["contested_gates"] + assert r["informational"]["E"]["leaks"] == 1 + # leak confirmed with grader_flag none + evidence -> accepted and voids. + t = _table(b=("caught", "caught"), e=("caught", "caught")) + t["rows"][2]["leak"] = "confirmed" + t["rows"][2]["leak_evidence"] = "cites the commit message" + assert _gates(t)["catch_status"]["c1|b1|E"] == v.UNSTABLE + + +def test_distinct_cell_counting(): + runs = _runs(cases=(("c1", ("b1", "b2")),)) + t = _table() + t["rows"] = [] + for bug in ("b1", "b2"): + for arm in ("B", "E"): + for rep in (0, 1): + t["rows"].append( + _row( + "c1", + bug, + arm, + rep, + "caught", + flag="flag" if (arm == "B" and rep == 0) else "none", + ev="x" if (arm == "B" and rep == 0) else "", + notes="halluc" if (arm == "E" and rep == 1) else "", + ) + ) + r = _v().gates(t, runs, "B", "E") + assert r["informational"]["B"] == {"leaks": 0, "unconfirmed_flags": 1, "hallucinations": 0} + assert r["informational"]["E"] == {"leaks": 0, "unconfirmed_flags": 0, "hallucinations": 1} + # A flagged cell the verifier confirms counts one leak and zero unconfirmed flags. + t["rows"][0]["leak"] = "confirmed" + r = _v().gates(t, runs, "B", "E") + assert ( + r["informational"]["B"]["leaks"] == 1 and r["informational"]["B"]["unconfirmed_flags"] == 0 + ) + + +# --------------------------------------------------------------------------- # +# Validation +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "mutate, msg", + [ + (lambda t: t["negative_assessments"].pop(), "missing negative-control assessment"), + (lambda t: t["rows"][0].__setitem__("leak", "flag"), "leak 'flag'"), + (lambda t: t["rows"][0].pop("leak"), "leak None"), + (lambda t: t["rows"][0].pop("grader_flag"), "grader_flag None"), + (lambda t: t["rows"][0].update(leak="confirmed"), "leak_evidence must be non-empty"), + (lambda t: t["rows"][0].update(grader_flag="flag"), "leak_evidence must be non-empty"), + (lambda t: t["rows"][0].update(leak_evidence="stray"), "none/none cell"), + (lambda t: t["negative_assessments"][0].update(leak="flag"), "leak 'flag'"), + ( + lambda t: t["negative_assessments"][0].update( + findings=[ + {"severity": "P1", "summary": "Same Thing"}, + {"severity": "P2", "summary": "same thing"}, + ] + ), + "duplicate finding", + ), + ( + lambda t: t["negative_assessments"][0].update( + findings=[{"severity": "blocker", "summary": "x"}] + ), + "severity 'blocker'", + ), + (lambda t: t["rows"].pop(), "missing grading row"), + (lambda t: t["rows"].append(dict(t["rows"][0])), "duplicate grading row"), + ], +) +def test_validation_errors(mutate, msg): + t = _table() + mutate(t) + with pytest.raises(ValueError, match=msg): + _v().validate_grades(t, _runs()) + + +def test_unblind_rejects_unknown_labels(): + t = _table() + t["rows"][0]["arm"] = "M9" + with pytest.raises(ValueError, match="M9"): + _v().unblind(t, {"B": "M1", "E": "M2"}) + + +# --------------------------------------------------------------------------- # +# Audit attachment (informational) +# --------------------------------------------------------------------------- # + + +def _audit(hits=(), unaudited=(), no_commands=(), vacuous=False, total=8): + return { + "hits": list(hits), + "unaudited": list(unaudited), + "no_commands": list(no_commands), + "coverage": { + "covered": total - len(unaudited) - len(no_commands), + "no_commands": len(no_commands), + "unaudited": len(unaudited), + "total": total, + }, + "vacuous": vacuous, + } + + +def test_audit_hits_attach_per_gate_without_changing_label(): + v = _v() + base = _table( + b=("caught", "missed"), e=("missed", "missed") + ) # B unstable blocks nothing; label PARITY + hit_b = { + "case": "c1", + "arm": "B", + "repeat": 0, + "markers": ["--all"], + "line": "git log --all", + "orientation_only": False, + } + hit_s3 = { + "case": "s3", + "arm": "E", + "repeat": 1, + "markers": ["git log -1", "GIT_LOG_RE"], + "line": "git log -1", + "orientation_only": True, + } + plain = _gates(base) + assert plain["definitive"] is False and "PROVISIONAL" in plain["label"] + r = _gates( + base, + audit=_audit(hits=[hit_b, hit_s3], unaudited=[{"case": "c1", "arm": "E", "repeat": 0}]), + ) + assert ( + r["verdict"] == plain["verdict"] and r["definitive"] is True and "DEFINITIVE" in r["label"] + ) + assert r["contested"] == [] and r["contested_gates"] == [] + reg = {(c["case"], c["arm"], c["repeat"]): c for c in r["gate_cells"]["regression"]} + assert reg[("c1", "B", 0)]["hits"][0]["markers"] == ["--all"] + assert reg[("c1", "E", 0)]["coverage"] == "unaudited" + fp = {(c["case"], c["arm"], c["repeat"]): c for c in r["gate_cells"]["fp"]} + assert fp[("s3", "E", 1)]["orientation_only"] is True + assert ("c1", "B", 0) not in fp + assert ( + r["audit_summary"]["cells_with_hits"] == 2 + and r["audit_summary"]["orientation_only_cells"] == 1 + ) + # Vacuous audit sets the flag, label unchanged, no contests. + r2 = _gates( + base, + audit=_audit( + unaudited=[ + {"case": c, "arm": a, "repeat": k} + for c in ("c1", "s3") + for a in "BE" + for k in (0, 1) + ], + vacuous=True, + ), + ) + assert ( + r2["audit_vacuous"] is True and r2["verdict"] == plain["verdict"] and r2["contested"] == [] + ) + # An S3 hit is listed under gate 2 even when the label is already NO-GO on a regression. + nogo = _gates(_table(e=("missed", "missed")), audit=_audit(hits=[hit_s3])) + assert nogo["verdict"] == v.NO_GO and any(c["hits"] for c in nogo["gate_cells"]["fp"]) + + +# --------------------------------------------------------------------------- # +# CLI: binding order (a)-(h) with synthetic sealed artifacts +# --------------------------------------------------------------------------- # + + +def _write_campaign(tmp_path, monkeypatch, cases=None, k=2, arms=("B", "E"), tamper=None): + """A synthetic sealed campaign under a tmp RUNS_DIR: artifacts, manifest, + blinding.json and a bundle whose id hashes. Returns (run_eval, paths).""" + + import reviewer_audit + from eval_core.models import RunResult + from eval_core.store import RunStore, write_json + + run_eval = _run_eval() + runs_dir = tmp_path / "runs" + monkeypatch.setattr(run_eval, "RUNS_DIR", str(runs_dir)) + monkeypatch.setattr(run_eval, "_assert_tree_identity", lambda *a, **kw: None) + # Campaign ledger: this synthetic campaign is the contrast's first dispatch on + # head h1 (tests that need another ledger shape rewrite the file afterwards); + # the registered-surface diff is stubbed (fake heads have no git history). + monkeypatch.setattr(run_eval, "_registered_surface_diff", lambda root, a, b: ("", False)) + monkeypatch.setattr(run_eval, "_read_frozen_ref", lambda root, contrast: "h1") + run_eval._append_ledger( + { + "kind": "dispatch", + "contrast": ["B", "E"], + "head": "h1", + "tree": "t1", + "subdir": "camp", + "at": "t", + "ledger_reset": False, + } + ) + loader_cases = cases if cases is not None else _corpus_cases() + monkeypatch.setattr( + run_eval.CorpusLoader, "load_cases", lambda self, strata: loader_cases, raising=True + ) + store = RunStore(str(runs_dir / "camp")) + results = [] + for case in loader_cases: + for arm in arms: + for rep in range(k): + snap = { + "stratum": case.stratum, + "ground_truth": ( + [{"id": f"{case.id}:b1", "must_catch": True}] + if case.stratum != "s3_negative" + else [] + ), + "allow_severities": ["P3"], + "title": case.id, + } + usage = { + "repo_head_sha": "h1", + "repo_dirty": False, + "repo_branch": "trunk", + "repo_common_dir": "/r/.git", + "codex_home_isolated": True, + "repo_ref_names": ["trunk"], + "repo_worktree_paths": [], + "session_commands": [], + "session_commands_start_lines": 0, + "session_commands_exec_lines": 0, + "session_other_events": 0, + "session_commands_truncated": False, + "worktree_dir": "/x/wt", + } + rr = RunResult( + case_id=case.id, + config_id=arm, + repeat_idx=rep, + review_markdown=f"review {case.id} {arm} {rep}", + model="m", + usage=usage, + run_id=f"{case.id}.{arm}.r{rep}", + case_snapshot=snap, + ) + store.save(rr.run_id, rr) + results.append(rr) + run_ids = [r.run_id for r in results] + prov = { + "repo_head_sha": "h1", + "repo_tree_sha": "t1", + "repo_dirty": False, + "repo_branch": "trunk", + "repo_common_dir": "/r/.git", + "repo_worktree_paths": [], + "repo_ref_names": ["trunk"], + } + manifest = { + "run_ids": run_ids, + "configs": list(arms), + "k": k, + "k_per": {}, + "codex_home_path": "/tmp/codex-home-x", + "artifacts_sha": reviewer_audit.artifacts_sha(store.root, run_ids), + **prov, + **run_eval._provenance_aggregates(results, prov), + } + if tamper: + tamper(manifest, store) + manifest["artifacts_sha"] = manifest.get("artifacts_sha") or reviewer_audit.artifacts_sha( + store.root, run_ids + ) + write_json(str(runs_dir / "camp-manifest.json"), manifest) + monkeypatch.setattr(run_eval, "find_repo_root", lambda: str(_REPO)) + # Build the blinded bundle through the real compare path. + rc = run_eval.cmd_compare( + type("N", (), {"subdir": "camp", "allow_mixed": False, "blinded": True})() + ) + assert rc == 0 + blinding = json.loads((runs_dir / "camp" / "blinding.json").read_text()) + return run_eval, runs_dir, manifest, blinding, results + + +def _corpus_cases(): + from eval_core.models import Case + + cases = [Case(id=f"s1-c{i}", stratum="s1_synthetic") for i in range(2)] + cases += [Case(id=f"s2-c{i}", stratum="s2_historical") for i in range(4)] + cases += [Case(id=f"s3-c{i}", stratum="s3_negative") for i in range(2)] + cases += [Case(id=f"s4-c{i}", stratum="s4_missed") for i in range(3)] + return cases + + +def _full_grades( + cases, blinding, results, b_catch=("caught", "caught"), e_catch=("caught", "caught") +): + rows, negs = [], [] + for case in cases: + for arm, verds in (("B", b_catch), ("E", e_catch)): + label = blinding["mapping"][arm] + for rep in (0, 1): + if case.stratum == "s3_negative": + negs.append(_neg(case.id, label, rep)) + else: + rows.append(_row(case.id, f"{case.id}:b1", label, rep, verds[rep])) + return {"bundle_id": blinding["bundle_id"], "rows": rows, "negative_assessments": negs} + + +def _ns(**kw): + base = { + "subdir": "camp", + "control": "B", + "candidate": "E", + "audit_hits": None, + "expect_head": None, + } + base.update(kw) + return type("N", (), base)() + + +def test_verdict_cli_happy_path_and_audit(tmp_path, monkeypatch, capsys): + run_eval, runs_dir, manifest, blinding, results = _write_campaign(tmp_path, monkeypatch) + cases = _corpus_cases() + grades = tmp_path / "g.json" + grades.write_text( + json.dumps(_full_grades(cases, blinding, results, b_catch=("missed", "missed"))) + ) + assert run_eval.cmd_verdict(_ns(grades=str(grades))) == 0 + out = capsys.readouterr().out + assert "PROVISIONAL" in out and "GO" in out + hits = tmp_path / "hits.json" + assert run_eval.cmd_audit(type("N", (), {"subdir": "camp", "output": str(hits)})()) == 0 + h = json.loads(hits.read_text()) + assert ( + h["coverage"]["no_commands"] == len(results) + and h["vacuous"] is False + and h["branch"] == "trunk" + ) + assert ( + run_eval.cmd_verdict(_ns(grades=str(grades), audit_hits=str(hits), expect_head="h1")) == 0 + ) + assert "DEFINITIVE" in capsys.readouterr().out + + +@pytest.mark.parametrize( + "control, candidate, msg", + [ + ("E", "B", "registered contrast"), + ("B", "B", "must differ"), + ("A", "B", "registered contrast"), + ], +) +def test_verdict_cli_argument_checks(tmp_path, monkeypatch, control, candidate, msg): + run_eval = _run_eval() + with pytest.raises(SystemExit, match=msg): + run_eval.cmd_verdict(_ns(control=control, candidate=candidate, grades="/nonexistent")) + + +def test_verdict_cli_binding_refusals(tmp_path, monkeypatch): + run_eval, runs_dir, manifest, blinding, results = _write_campaign(tmp_path, monkeypatch) + cases = _corpus_cases() + grades = tmp_path / "g.json" + grades.write_text(json.dumps(_full_grades(cases, blinding, results))) + bpath = runs_dir / "camp" / "blinding.json" + good = bpath.read_text() + # (c) mapping swapped. + b = json.loads(good) + b["mapping"] = {k: v for k, v in zip(b["mapping"], reversed(list(b["mapping"].values())))} + bpath.write_text(json.dumps(b)) + with pytest.raises(SystemExit, match=r"(?s)\(c\).*mapping"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + bpath.write_text(good) + # (c) run_ids_sha. + b = json.loads(good) + b["run_ids_sha"] = "0" * 64 + bpath.write_text(json.dumps(b)) + with pytest.raises(SystemExit, match=r"(?s)\(c\).*run_ids_sha"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + bpath.write_text(good) + # (c) rubric sha. + b = json.loads(good) + b["rubric_sha256"] = "0" * 64 + bpath.write_text(json.dumps(b)) + with pytest.raises(SystemExit, match=r"(?s)\(c\).*rubric"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + bpath.write_text(good) + # (c) bundle edited after compare. + bundle = runs_dir / "camp" / "comparison.blinded.md" + orig = bundle.read_text() + bundle.write_text(orig + "x") + with pytest.raises(SystemExit, match=r"(?s)\(c\).*(bundle_id|regenerated)"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + bundle.write_text(orig) + # (d) wrong bundle_id in grades. + g = json.loads(grades.read_text()) + g["bundle_id"] = "deadbeefdeadbeef" + bad = tmp_path / "bad.json" + bad.write_text(json.dumps(g)) + with pytest.raises(SystemExit, match=r"(?s)\(d\).*bundle_id"): + run_eval.cmd_verdict(_ns(grades=str(bad))) + # (d) validation error (omitted S3 cell). + g = json.loads(grades.read_text()) + g["negative_assessments"].pop() + bad.write_text(json.dumps(g)) + with pytest.raises(SystemExit, match=r"(?s)\(d\).*missing negative-control"): + run_eval.cmd_verdict(_ns(grades=str(bad))) + # (c) artifact edited after sealing -> both the manifest's and blinding.json's seal mismatch. + art = runs_dir / "camp" / f"{results[0].run_id}.json" + d = json.loads(art.read_text()) + d["usage"]["session_commands"] = ["/bin/zsh -lc 'git log --all' in /x"] + art_orig = art.read_text() + art.write_text(json.dumps(d)) + with pytest.raises(SystemExit, match=r"(?s)\(c\).*artifacts_sha"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + art.write_text(art_orig) + + +def test_verdict_cli_audit_hits_refusals(tmp_path, monkeypatch): + run_eval, runs_dir, manifest, blinding, results = _write_campaign(tmp_path, monkeypatch) + grades = tmp_path / "g.json" + grades.write_text(json.dumps(_full_grades(_corpus_cases(), blinding, results))) + hits = tmp_path / "hits.json" + assert run_eval.cmd_audit(type("N", (), {"subdir": "camp", "output": str(hits)})()) == 0 + good = json.loads(hits.read_text()) + for key, val, msg in ( + ("run_ids_sha", "0" * 64, "run_ids_sha"), + ("artifacts_sha", "0" * 64, "artifacts_sha"), + ("branch", "other", "branch"), + ("vacuous", True, "'vacuous'"), + ("no_commands", [], "'no_commands'"), + ): + h = dict(good) + h[key] = val + (tmp_path / "h2.json").write_text(json.dumps(h)) + with pytest.raises(SystemExit, match=r"(?s)\(e\).*" + msg): + run_eval.cmd_verdict(_ns(grades=str(grades), audit_hits=str(tmp_path / "h2.json"))) + # Hashes intact but one hit removed -> recomputed audit differs. (Add a hit first.) + art = runs_dir / "camp" / f"{results[0].run_id}.json" + d = json.loads(art.read_text()) + d["usage"]["session_commands"] = ["/bin/zsh -lc 'git log --all' in /x"] + d["usage"]["session_commands_start_lines"] = 1 + d["usage"]["session_commands_exec_lines"] = 1 + art.write_text(json.dumps(d)) + import reviewer_audit + + manifest_path = runs_dir / "camp-manifest.json" + m = json.loads(manifest_path.read_text()) + m["artifacts_sha"] = reviewer_audit.artifacts_sha(str(runs_dir / "camp"), m["run_ids"]) + manifest_path.write_text(json.dumps(m)) + assert ( + run_eval.cmd_compare( + type("N", (), {"subdir": "camp", "allow_mixed": False, "blinded": True})() + ) + == 0 + ) + blinding = json.loads((runs_dir / "camp" / "blinding.json").read_text()) + grades.write_text(json.dumps(_full_grades(_corpus_cases(), blinding, results))) + assert run_eval.cmd_audit(type("N", (), {"subdir": "camp", "output": str(hits)})()) == 0 + h = json.loads(hits.read_text()) + assert len(h["hits"]) == 1 + h["hits"] = [] + (tmp_path / "h3.json").write_text(json.dumps(h)) + with pytest.raises(SystemExit, match=r"\(e\).*'hits'"): + run_eval.cmd_verdict(_ns(grades=str(grades), audit_hits=str(tmp_path / "h3.json"))) + + +def test_verdict_cli_protocol_guards(tmp_path, monkeypatch): + # k=1 realized grid -> refused at (f) AFTER (c)-(d) passed. + run_eval, runs_dir, manifest, blinding, results = _write_campaign(tmp_path, monkeypatch, k=1) + rows, negs = [], [] + for case in _corpus_cases(): + for arm in ("B", "E"): + label = blinding["mapping"][arm] + if case.stratum == "s3_negative": + negs.append(_neg(case.id, label, 0)) + else: + rows.append(_row(case.id, f"{case.id}:b1", label, 0, "caught")) + grades = tmp_path / "g.json" + grades.write_text( + json.dumps({"bundle_id": blinding["bundle_id"], "rows": rows, "negative_assessments": negs}) + ) + with pytest.raises(SystemExit, match=r"(?s)\(f\).*realized grid"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + + +def test_verdict_cli_corpus_floor_and_filter(tmp_path, monkeypatch): + seven = _corpus_cases()[:7] + run_eval, runs_dir, manifest, blinding, results = _write_campaign( + tmp_path, monkeypatch, cases=seven + ) + grades = tmp_path / "g.json" + grades.write_text(json.dumps(_full_grades(seven, blinding, results))) + with pytest.raises(SystemExit, match=r"(?s)\(f\).*below the floor"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + # A filtered run (manifest covers fewer cases than the corpus) is refused too. + run_eval, runs_dir, manifest, blinding, results = _write_campaign(tmp_path, monkeypatch) + monkeypatch.setattr( + run_eval.CorpusLoader, + "load_cases", + lambda self, strata: _corpus_cases() + + [_corpus_cases()[0].__class__(id="s4-extra", stratum="s4_missed")], + raising=True, + ) + grades.write_text(json.dumps(_full_grades(_corpus_cases(), blinding, results))) + with pytest.raises(SystemExit, match=r"(?s)\(f\).*OK runs cover"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + + +def test_verdict_cli_provenance_guards(tmp_path, monkeypatch): + def dirty(manifest, store): + manifest["run_dirty_any"] = False # stored says clean ... + import glob + import os as _os + + f = sorted( + x + for x in glob.glob(store.root + "/*.json") + if not _os.path.basename(x).startswith("blinding") + )[0] + d = json.loads(open(f).read()) + d["usage"]["repo_dirty"] = True # ... but an artifact is dirty + open(f, "w").write(json.dumps(d)) + manifest["artifacts_sha"] = None # re-sealed by the helper after the tamper + + run_eval, runs_dir, manifest, blinding, results = _write_campaign( + tmp_path, monkeypatch, tamper=dirty + ) + grades = tmp_path / "g.json" + grades.write_text(json.dumps(_full_grades(_corpus_cases(), blinding, results))) + with pytest.raises(SystemExit, match=r"(?s)\(f\).*aggregates"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + # wrong --expect-head + run_eval, runs_dir, manifest, blinding, results = _write_campaign(tmp_path, monkeypatch) + grades.write_text(json.dumps(_full_grades(_corpus_cases(), blinding, results))) + with pytest.raises(SystemExit, match=r"(?s)\(f\).*expect-head"): + run_eval.cmd_verdict(_ns(grades=str(grades), expect_head="h2")) + + # a second head sha hidden by editing run_head_shas -> recomputed differs + def two_heads(manifest, store): + import glob + import os as _os + + f = sorted( + x + for x in glob.glob(store.root + "/*.json") + if not _os.path.basename(x).startswith("blinding") + )[0] + d = json.loads(open(f).read()) + d["usage"]["repo_head_sha"] = "h2" + open(f, "w").write(json.dumps(d)) + manifest["run_head_shas"] = ["h1"] + manifest["artifacts_sha"] = None + + run_eval, runs_dir, manifest, blinding, results = _write_campaign( + tmp_path, monkeypatch, tamper=two_heads + ) + grades.write_text(json.dumps(_full_grades(_corpus_cases(), blinding, results))) + with pytest.raises(SystemExit, match=r"(?s)\(f\).*aggregates"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + + +def test_verdict_cli_order_c_before_f(tmp_path, monkeypatch): + """A manifest violating (c) AND (f) at once is refused at (c).""" + run_eval, runs_dir, manifest, blinding, results = _write_campaign(tmp_path, monkeypatch, k=1) + bpath = runs_dir / "camp" / "blinding.json" + b = json.loads(bpath.read_text()) + b["run_ids_sha"] = "0" * 64 + bpath.write_text(json.dumps(b)) + grades = tmp_path / "g.json" + grades.write_text( + json.dumps({"bundle_id": b["bundle_id"], "rows": [], "negative_assessments": []}) + ) + with pytest.raises(SystemExit, match=r"verdict \(c\)"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + + +def test_verdict_refuses_failed_manifest_and_traversal(tmp_path, monkeypatch): + run_eval = _run_eval() + monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) + (tmp_path / "runs").mkdir() + (tmp_path / "runs" / "camp-manifest.json").write_text(json.dumps({"failed": True})) + with pytest.raises(SystemExit, match="failed/incomplete"): + run_eval.cmd_verdict(_ns(grades="/nonexistent")) + assert run_eval.cmd_verdict(_ns(subdir="../x", grades="/nonexistent")) == 1 + + +@pytest.mark.parametrize("field", ["repo_ref_names", "repo_worktree_paths"]) +def test_verdict_refuses_incomplete_manifest_provenance(tmp_path, monkeypatch, field): + def blank(manifest, store): + manifest[field] = None + + run_eval, runs_dir, manifest, blinding, results = _write_campaign( + tmp_path, monkeypatch, tamper=blank + ) + grades = tmp_path / "g.json" + grades.write_text(json.dumps(_full_grades(_corpus_cases(), blinding, results))) + with pytest.raises(SystemExit, match=r"(?s)\(f\).*write-time provenance incomplete"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + + +def test_campaign_reads_latency_flag_and_cost(): + v = _v() + runs = _runs() + for rr in runs: + rr["latency_s"] = 100.0 if rr["config_id"] == "B" else 250.0 + rr["model"] = "gpt-5.6-sol" if rr["config_id"] == "B" else "gpt-6-astra" + rr["usage"] = {"prompt_tokens_est": 1000} + reads = v.campaign_reads(runs, "B", "E", {"gpt-5.6-sol": 4.0, "gpt-6-astra": 10.0}) + assert reads["latency"]["B"]["n"] == 4 and reads["latency"]["B"]["p95"] == 100.0 + assert reads["latency_flag_candidate_p95_over_2x_control"] is True # 250 > 200 + assert reads["cost_estimate_usd"]["E"]["prompt_tokens_est"] == 4000 + assert reads["cost_estimate_usd"]["E"]["usd"] == pytest.approx(4000 * 10.0 / 1e6) + assert reads["cost_estimate_usd"]["B"]["usd"] == pytest.approx(4000 * 4.0 / 1e6) + for rr in runs: + if rr["config_id"] == "E": + rr["latency_s"] = 199.0 + assert v.campaign_reads(runs, "B", "E")["latency_flag_candidate_p95_over_2x_control"] is False + assert ( + v.campaign_reads(runs, "B", "E")["cost_estimate_usd"]["E"]["usd"] is None + ) # no price given + assert v.campaign_reads(runs, "B", "E")["vendor_lifecycle"] == "operator_supplied" + + +def test_verdict_cli_output_carries_campaign_reads(tmp_path, monkeypatch, capsys): + run_eval, runs_dir, manifest, blinding, results = _write_campaign(tmp_path, monkeypatch) + grades = tmp_path / "g.json" + grades.write_text(json.dumps(_full_grades(_corpus_cases(), blinding, results))) + capsys.readouterr() # drop the compare/setup output + assert run_eval.cmd_verdict(_ns(grades=str(grades))) == 0 + out = capsys.readouterr().out + body = json.loads(out[out.index("{") : out.index("\nVERDICT:")]) + assert "latency" in body["campaign_reads"] and "cost_estimate_usd" in body["campaign_reads"] + assert body["campaign_reads"]["cost_estimate_usd"]["B"]["model"] == "m" + + +def _ledger_write(run_eval, entries): + import os as _os + + path = run_eval._ledger_path() + if _os.path.exists(path): + _os.remove(path) + for e in entries: + run_eval._append_ledger(e) + + +def _dispatch(head, subdir): + return { + "kind": "dispatch", + "contrast": ["B", "E"], + "head": head, + "tree": "t", + "subdir": subdir, + "at": "t", + } + + +def _verdict_body(run_eval, grades, capsys, **kw): + capsys.readouterr() + assert run_eval.cmd_verdict(_ns(grades=str(grades), **kw)) == 0 + out = capsys.readouterr().out + return json.loads(out[out.index("{") : out.index("\nVERDICT:")]) + + +def test_verdict_requires_ledger_record(tmp_path, monkeypatch): + run_eval, runs_dir, manifest, blinding, results = _write_campaign(tmp_path, monkeypatch) + grades = tmp_path / "g.json" + grades.write_text(json.dumps(_full_grades(_corpus_cases(), blinding, results))) + _ledger_write(run_eval, []) + with pytest.raises(SystemExit, match=r"(?s)\(f\).*no campaign-ledger"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + + +def test_verdict_exposure_status_cases(tmp_path, monkeypatch, capsys): + run_eval, runs_dir, manifest, blinding, results = _write_campaign(tmp_path, monkeypatch) + grades = tmp_path / "g.json" + grades.write_text(json.dumps(_full_grades(_corpus_cases(), blinding, results))) + # First dispatch on the frozen head -> mechanical. + body = _verdict_body(run_eval, grades, capsys) + assert body["exposure"]["mechanical"] is True + assert "RE-REGISTERED" not in body["label"] and "PROVISIONAL" in body["label"] + # Re-exposure: an earlier subdir on the SAME head -> non-mechanical. + _ledger_write(run_eval, [_dispatch("h1", "first"), _dispatch("h1", "camp")]) + body = _verdict_body(run_eval, grades, capsys) + assert body["exposure"]["mechanical"] is False and body["definitive"] is False + assert "RE-REGISTERED" in body["label"] and "re-exposure" in body["exposure"]["reason"] + assert body["exposure"]["subdirs"] == ["camp", "first"] + # The FIRST subdir reused after a HEAD change -> non-mechanical (fresh subdir required). + _ledger_write(run_eval, [_dispatch("h0", "camp"), _dispatch("h1", "camp")]) + monkeypatch.setattr(run_eval, "_read_frozen_ref", lambda root, contrast: "h0") + body = _verdict_body(run_eval, grades, capsys) + assert body["exposure"]["mechanical"] is False and "FRESH subdir" in body["exposure"]["reason"] + # Re-run on a NEW head (frozen h0, anchored by the ref): registered surfaces untouched -> mechanical. + _ledger_write(run_eval, [_dispatch("h0", "first"), _dispatch("h1", "camp")]) + monkeypatch.setattr(run_eval, "_read_frozen_ref", lambda root, contrast: "h0") + monkeypatch.setattr(run_eval, "_registered_surface_diff", lambda root, a, b: ("", False)) + body = _verdict_body(run_eval, grades, capsys) + assert body["exposure"]["mechanical"] is True and body["exposure"]["head_changed"] is True + # ... the CALL_TIMEOUT_S-only exception -> mechanical with the note. + monkeypatch.setattr( + run_eval, + "_registered_surface_diff", + lambda root, a, b: (" tools/reviewer-eval/adapters/codex_reviewer.py | 2 +-", True), + ) + body = _verdict_body(run_eval, grades, capsys) + assert body["exposure"]["mechanical"] is True and body["exposure"]["timeout_only"] is True + # ... any other registered-surface change -> RE-REGISTERED, non-mechanical. + monkeypatch.setattr( + run_eval, + "_registered_surface_diff", + lambda root, a, b: (" tools/reviewer-eval/reviewer_verdict.py | 4 ++--", False), + ) + body = _verdict_body(run_eval, grades, capsys) + assert body["exposure"]["mechanical"] is False and body["definitive"] is False + assert ( + "re-registered" in body["exposure"]["reason"] + and "reviewer_verdict.py" in body["exposure"]["surface_diff_stat"] + ) + + +def test_registered_surface_diff_detects_timeout_only_change(tmp_path): + """Real git: a CALL_TIMEOUT_S-only commit is the exception; any other change is not.""" + import subprocess + + run_eval = _run_eval() + repo = tmp_path / "r" + repo.mkdir() + + def git(*a): + return subprocess.run( + ["git", "-c", "user.name=t", "-c", "user.email=t@l", *a], + cwd=str(repo), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + git("init", "-q", "-b", "trunk") + adapter = repo / "tools" / "reviewer-eval" / "adapters" / "codex_reviewer.py" + adapter.parent.mkdir(parents=True) + adapter.write_text("class X:\n CALL_TIMEOUT_S = 3600.0\n") + git("add", "-A") + git("commit", "-q", "--no-verify", "-m", "a") + frozen = git("rev-parse", "HEAD") + adapter.write_text("class X:\n CALL_TIMEOUT_S = 7200.0\n") + git("commit", "-q", "-a", "--no-verify", "-m", "b") + stat, timeout_only = run_eval._registered_surface_diff( + str(repo), frozen, git("rev-parse", "HEAD") + ) + assert "codex_reviewer.py" in stat and timeout_only is True + (repo / "tools" / "reviewer-eval" / "x.py").write_text("y = 1\n") + git("add", "-A") + git("commit", "-q", "--no-verify", "-m", "c") + stat, timeout_only = run_eval._registered_surface_diff( + str(repo), frozen, git("rev-parse", "HEAD") + ) + assert "x.py" in stat and timeout_only is False + assert run_eval._registered_surface_diff(str(repo), frozen, frozen) == ("", False) + + +def test_verdict_refuses_when_frozen_ref_absent_or_conflicting(tmp_path, monkeypatch): + """The three frozen-provenance integrity failures are protocol-guard REFUSALS + (DECISION_RULE.md), distinct from the documented non-mechanical labels.""" + run_eval, runs_dir, manifest, blinding, results = _write_campaign(tmp_path, monkeypatch) + grades = tmp_path / "g.json" + grades.write_text(json.dumps(_full_grades(_corpus_cases(), blinding, results))) + monkeypatch.setattr(run_eval, "_read_frozen_ref", lambda root, contrast: None) + with pytest.raises(SystemExit, match=r"(?s)\(f\).*frozen ref .* is absent"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + monkeypatch.setattr(run_eval, "_read_frozen_ref", lambda root, contrast: "h0") + with pytest.raises(SystemExit, match=r"(?s)\(f\).*!= frozen ref"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + monkeypatch.setattr(run_eval, "_read_frozen_ref", lambda root, contrast: "h1") + _ledger_write(run_eval, [{**_dispatch("h1", "camp"), "ledger_reset": True}]) + with pytest.raises(SystemExit, match=r"(?s)\(f\).*ledger was reset"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + + +def test_integrity_logs_fail_closed_on_malformed_records(tmp_path, monkeypatch): + run_eval = _run_eval() + monkeypatch.setattr(run_eval, "RUNS_DIR", str(tmp_path / "runs")) + (tmp_path / "runs").mkdir() + good = json.dumps( + { + "kind": "dispatch", + "contrast": ["B", "E"], + "head": "h", + "tree": "t", + "subdir": "s", + "at": "a", + } + ) + (tmp_path / "runs" / "campaign-ledger.jsonl").write_text(good + "\n" + good[:20] + "\n") + with pytest.raises(SystemExit, match=r"campaign-ledger.jsonl:2: malformed"): + run_eval._read_ledger() + (tmp_path / "runs" / "campaign-ledger.jsonl").write_text( + json.dumps({"kind": "dispatch", "head": "h"}) + "\n" + ) + with pytest.raises(SystemExit, match=r":1: 'dispatch' record missing 'contrast'"): + run_eval._read_ledger() + (tmp_path / "runs" / "campaign-ledger.jsonl").write_text(good + "\n\n" + good + "\n") + assert len(run_eval._read_ledger()) == 2 # blank lines are fine + (tmp_path / "runs" / "x-attempts.jsonl").write_text( + json.dumps({"kind": "end", "failures": []}) + "\n" + ) + with pytest.raises(SystemExit, match=r"x-attempts.jsonl:1: 'end' record missing 'started_at'"): + run_eval._read_attempts("x") + (tmp_path / "runs" / "x-attempts.jsonl").write_text(json.dumps({"kind": "weird"}) + "\n") + with pytest.raises(SystemExit, match=r"unknown record kind"): + run_eval._read_attempts("x") + + +def test_verdict_refuses_self_consistent_bundle_tamper(tmp_path, monkeypatch): + """Editing the bundle, recomputing its id and updating blinding.json + the grades + stays self-consistent but is NOT the bundle regenerated from the sealed artifacts.""" + run_eval, runs_dir, manifest, blinding, results = _write_campaign(tmp_path, monkeypatch) + bundle = runs_dir / "camp" / "comparison.blinded.md" + text = bundle.read_text() + old_id = blinding["bundle_id"] + edited_with_token = ( + text.replace(old_id, run_eval._BUNDLE_ID_TOKEN) + "\n(an extra grader-visible line)\n" + ) + new_id = run_eval._bundle_id_of(edited_with_token) + bundle.write_text(edited_with_token.replace(run_eval._BUNDLE_ID_TOKEN, new_id)) + b = dict(blinding) + b["bundle_id"] = new_id + (runs_dir / "camp" / "blinding.json").write_text(json.dumps(b)) + grades = tmp_path / "g.json" + grades.write_text(json.dumps(_full_grades(_corpus_cases(), b, results))) + with pytest.raises(SystemExit, match=r"(?s)\(c\).*regenerated from the sealed artifacts"): + run_eval.cmd_verdict(_ns(grades=str(grades))) + + +def test_registered_surface_diff_rejects_timeout_decrease(tmp_path): + import subprocess + + run_eval = _run_eval() + repo = tmp_path / "r2" + repo.mkdir() + + def git(*a): + return subprocess.run( + ["git", "-c", "user.name=t", "-c", "user.email=t@l", *a], + cwd=str(repo), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + git("init", "-q", "-b", "trunk") + adapter = repo / "tools" / "reviewer-eval" / "adapters" / "codex_reviewer.py" + adapter.parent.mkdir(parents=True) + adapter.write_text("class X:\n CALL_TIMEOUT_S = 7200.0\n") + git("add", "-A") + git("commit", "-q", "--no-verify", "-m", "a") + frozen = git("rev-parse", "HEAD") + adapter.write_text("class X:\n CALL_TIMEOUT_S = 60.0\n") + git("commit", "-q", "-a", "--no-verify", "-m", "b") + stat, timeout_only = run_eval._registered_surface_diff( + str(repo), frozen, git("rev-parse", "HEAD") + ) + assert ( + "codex_reviewer.py" in stat and timeout_only is False + ), "lowering the cap is NOT the exception" diff --git a/tools/eval_core/compare.py b/tools/eval_core/compare.py index 517cf9c0f..438a76191 100644 --- a/tools/eval_core/compare.py +++ b/tools/eval_core/compare.py @@ -26,6 +26,13 @@ } +def render_default_header(config_ids: list[str], has_repeats: bool) -> str: + """Public alias of the default grading header (``_render_header``), so a + harness building a BLINDED bundle can render the standard "How to grade" + section with the blind labels as columns and append its own binding line.""" + return _render_header(config_ids, has_repeats) + + def _render_header(config_ids: list[str], has_repeats: bool) -> str: """Grading header + table, sized to the configs (and repeats) actually present. @@ -174,13 +181,19 @@ def _render_review(rr: RunResult, redact_meta: bool = False) -> str: # --------------------------------------------------------------------------- # # Family patterns scrubbed from blinded text in ADDITION to the configured model -# names: any gpt-5.x-style token, the GPT-5.6 tier names as standalone words, and -# bare major.minor version shorthand ("5.5", "5.6"). Over-redaction is safe (a -# stray [model-redacted] doesn't change whether a defect was named at a location); -# under-redaction breaks the blind. +# names: any gpt-5.x / gpt-6.x-style token (the separator class includes the +# Unicode hyphens/dashes codex output can contain: U+2010..U+2014, so "GPT‑6" +# with a non-breaking hyphen and "GPT–6" with an en dash are scrubbed too), the +# GPT-5.6 tier names and "astra" as standalone words, and bare major.minor +# version shorthand for the 5.x family ("5.5", "5.6"). There is deliberately NO +# bare "6.x" numeric pattern (the 5.x one is grandfathered; a bare "6.2" is too +# common in ordinary text). Over-redaction is safe (a stray [model-redacted] +# doesn't change whether a defect was named at a location); under-redaction +# breaks the blind. _MODEL_FAMILY_PATTERNS = ( - r"gpt[\s._-]?5(?:\.\d+)?(?:-[a-z0-9]+)*", - r"\b(?:sol|terra|luna)\b", + r"gpt[\s._\-\u2010\u2011\u2012\u2013\u2014]?5(?:\.\d+)?(?:-[a-z0-9]+)*", + r"gpt[\s._\-\u2010\u2011\u2012\u2013\u2014]?6(?:\.\d+)?(?:-[a-z0-9]+)*", + r"\b(?:sol|terra|luna|astra)\b", r"\b5\.[0-9]\b", # Claude-family MODEL tokens only: "claude" followed by a version/family # tail ("claude-fable-5", "Claude 3.5", "claude_sonnet"). A bare "claude" @@ -340,4 +353,10 @@ def _sort_key(gkey: tuple[str, str]): return "\n".join(parts).rstrip() + "\n" -__all__ = ["build_bundle", "derive_blind_mapping", "sanitize_model_refs", "apply_blinding"] +__all__ = [ + "build_bundle", + "derive_blind_mapping", + "sanitize_model_refs", + "apply_blinding", + "render_default_header", +] diff --git a/tools/eval_core/models.py b/tools/eval_core/models.py index 8b8e817e1..432f5d50e 100644 --- a/tools/eval_core/models.py +++ b/tools/eval_core/models.py @@ -111,7 +111,9 @@ class Case: fixture: dict[str, Any] = field(default_factory=dict) ground_truth: list[GroundTruthBug] = field(default_factory=list) # Negative-control / precision controls: - expect_no_blockers: bool = False # True ⇒ any P0/P1 is a false positive + expect_no_blockers: bool = ( + False # True ⇒ any finding above `allow_severities` is a false positive + ) allow_severities: list[str] = field(default_factory=lambda: ["P2", "P3"]) known_fp_topics: list[dict[str, Any]] = field(default_factory=list) # Exact post-diff path contract. When set, verify() requires the diff to touch diff --git a/tools/eval_core/runner.py b/tools/eval_core/runner.py index 3876d9ef7..424b67266 100644 --- a/tools/eval_core/runner.py +++ b/tools/eval_core/runner.py @@ -92,9 +92,23 @@ def run_matrix( progress: Optional[Callable[[str], None]] = None, treatment_fields: tuple = ("model",), k_overrides: Optional[dict] = None, + on_dispatch: Optional[Callable[[], None]] = None, + on_plan: Optional[Callable[[list[str]], None]] = None, ) -> list[RunResult]: """Execute the full matrix, resuming completed runs, returning all results. + ``on_dispatch`` (if given) is called ONCE, immediately before the first + pending review is submitted to the pool - i.e. before the first reviewer + call can start. Everything before it (CLI pin check, confound check, + experiment/case tags, resume verification) is planning; a failure there has + made no reviewer call. Not called when every job resumed from cache. + + ``on_plan`` (if given) is called ONCE with the run key of EVERY job of this + matrix (cached and pending alike), before resume loading - so a caller + interrupted mid-matrix (where this function never returns) can read the + store for exactly those keys and learn which cells it holds on disk, + including an artifact persisted an instant before the interrupt. + ``treatment_fields`` declares which Config fields are the experimental treatment (default: model only — the classic A/B); everything else in ``CONFOUND_FIELDS`` stays a held-constant confound. ``k_overrides`` maps @@ -236,6 +250,13 @@ def _prompt_matches(case, cached) -> bool: log(f"WARNING: prompt re-verify failed for {case.id}: {exc}; will rerun") return False + if on_plan is not None: + on_plan( + [ + run_key(case.id, config.id, r, config_tags[config.id], case_tags[case.id]) + for case, config, r in jobs + ] + ) # Resume: load any already-completed runs, keyed on the experiment identity, # AND verified to match the prompt the reviewer would build now. for case, config, r in jobs: @@ -286,19 +307,24 @@ def _execute(job: tuple[Case, Config, int]) -> RunResult: case_snapshot=snap, infra_error=f"{type(exc).__name__}: {exc}", ) + # Persist IN THE WORKER, before the result is handed back: a coordinator + # interrupted while consuming futures still leaves every finished review + # on disk (the pool waits for running workers), so an aborted attempt's + # journal - rebuilt from the store - never omits a completed cell. + # Successes and infra-errors alike (infra-errors are surfaced but excluded + # from the bundle; persisting avoids re-run churn while staying visibly + # non-ok). + store.save(key, rr) return rr if pending: + if on_dispatch is not None: + on_dispatch() with ThreadPoolExecutor(max_workers=max(1, max_parallel)) as pool: futures = {pool.submit(_execute, job): job for job in pending} for fut in as_completed(futures): case, config, r = futures[fut] - rr = fut.result() - key = run_key(case.id, config.id, r, config_tags[config.id], case_tags[case.id]) - # Persist successes and infra-errors alike (infra-errors are - # surfaced but excluded from the bundle; persisting avoids - # re-run churn while still being visibly non-ok). - store.save(key, rr) + rr = fut.result() # already persisted by the worker results.append(rr) status = INFRA_ERROR if not rr.ok else f"{rr.latency_s:.0f}s" log(f"done: {case.id} {config.id} r{r} [{status}]") diff --git a/tools/reviewer-eval/DECISION_RULE.md b/tools/reviewer-eval/DECISION_RULE.md index d9038a40c..1a9c8ec0c 100644 --- a/tools/reviewer-eval/DECISION_RULE.md +++ b/tools/reviewer-eval/DECISION_RULE.md @@ -1,87 +1,290 @@ -# Pre-Registered Decision Rule: gpt-5.5 -> gpt-5.6 Reviewer Evaluation +# Pre-Registered Decision Rule: gpt-5.6-sol -> gpt-6-astra Reviewer Evaluation -Committed BEFORE any campaign run (the PR that adds this file timestamps the -pre-registration). The verdict is computed MECHANICALLY from the graded tables -against these rules; the rules may not be edited after the campaign starts. +Date: 2026-10-16. Pre-registered in PR #849 (frozen at that PR's labeled, CI-green HEAD). The +candidate arm is NEVER run on the corpus before the campaign: its first corpus +run is the campaign itself, so pre-registration holds by construction. The +verdict is computed MECHANICALLY by `run_eval.py verdict` (`reviewer_verdict.py`) +from the reconciled grading table; this file may not be edited after the +campaign starts (the campaign's frozen PR-1 HEAD pins it). + +This file is NOT grader-facing. Graders receive `GRADING_RUBRIC.md` only. ## The experiment | Arm | Model | Effort | Repeats | Role | |-----|-------|--------|---------|------| -| A | gpt-5.5 | xhigh | k=2 | control (current production) | -| B | gpt-5.6-sol | xhigh | k=2 | primary candidate | -| C | gpt-5.6-terra | xhigh | k=1 | cost-fallback probe (informational) | -| D | gpt-5.6-sol | max | k=1 | effort probe (informational) | +| B | gpt-5.6-sol | xhigh | k=2 | control (current production) | +| E | gpt-6-astra | xhigh | k=2 | candidate | -The production decision is A vs B. C and D never gate the swap. +No probe arms. CLI pinned to codex-cli 0.162.1 on every arm (the runner refuses a +live `codex --version` mismatch); CI pins the same version. Every harness codex +call runs under an isolated `CODEX_HOME` that mirrors CI's codex-action home +(credential + `sandbox_mode = "read-only"` + `approval_policy = "never"`; no MCP +servers, hooks or memories; codex's DEFAULT features otherwise, web search +included). ## Definitions -- **Reliably caught** (per arm, per ground-truth bug): named the same defect at - the same location/symbol in EVERY repeat of that arm. -- **Unstably caught**: caught in some but not all repeats. -- **Missed**: caught in no repeat. -- **False positive (FP)**: on an `expect_no_blockers` case, any finding whose - severity is ABOVE that case's `allow_severities` (default allows P2/P3, i.e. - P0/P1 are FPs; the documented-deviation case allows only P3, so P0-P2 count). - Known-FP topics listed by the case are never counted. -- **Calibration failure**: flagging a REGISTRY-documented deviation above P3. - Counted as an FP. - -## Primary gates (B vs A) - -1. **Regression (NO-GO):** any `must_catch` bug that A reliably catches and B - misses in both B-repeats. (A unstably-caught vs B missed -> flagged for - judgment in the report, not an automatic NO-GO.) - `must_catch` gates regressions ONLY. -2. **FP gate (NO-GO):** B's total FP count across S3 negative controls exceeds - A's. -3. **GO:** zero regressions AND the FP gate passes AND at least one strict - improvement: - - B reliably catches ANY ground-truth bug (`must_catch` true OR false — - aspirational S4 catches count, and S4 is the highest-value evidence) that A - misses in both A-repeats, OR - - B has strictly fewer FPs than A (with no catch regression). -4. **PARITY:** zero regressions but no strict improvement -> the user decides - (vendor-cadence value vs staying on gpt-5.5). - -## Secondary reads (informational only) - -- **C vs A:** does Terra actually match gpt-5.5 on our corpus (the half-cost - claim)? Relevant only if B fails its gates. -- **D vs B:** does `max` effort add reliable catches over xhigh on the same - model? Informs an optional follow-up; the default ship remains Sol @ xhigh. -- Latency per arm (from the unblinded artifacts) is reported alongside, since a - much slower CI review has real cost even at equal accuracy. +By reference to `GRADING_RUBRIC.md`. Grades are exactly caught / partial / +missed (partial = missed for every gate). "Leak" is NOT a grade but a separate +per-cell field (`leak: none | confirmed` in the reconciled table; graders write +`none` or `flag`, the verifier rules); a `confirmed` cell is VOID (below). + +- **Reliably caught** (per arm, per bug) = caught in EVERY non-void repeat of + that arm AND no void repeat on that (case, arm). +- **FP total per arm** = sum of FPs over all its non-void repeats on S3 cases, + where an FP is a finding whose severity is above the case's + `allow_severities`, with NO topic-based exemption: on this corpus the only + known-FP topic (s3-wcr) has `would_be_severity_if_flagged` equal to the + allowed P3, so a P3 mention is already allowed and anything above P3 is the + 2026-07 "calibration failure" FP. The older "known-FP topics are never + counted" wording is retired as redundant; the sibling engine's + `known_topic_id` exemption branch is NOT ported. The bundle's "do NOT flag" + wording for known topics stays (reviewer advice, not grader instruction). + +## Gates (E vs B; symmetric on catches) + +1. **Regression NO-GO** = ANY ground-truth bug (`must_catch` true OR false) + that B reliably catches and E has status MISSED (the status definitions + under "Voided repeats" are the executable wording). `must_catch` stays in + the case files as a reporting label only. B UNSTABLE / E MISSED is a + judgment FLAG reported next to the mechanical outcome; it never changes the + label by itself. +2. **FP NO-GO** = E's S3 FP total > B's. Small-sample note (2 S3 cases x k=2, + B expected at 0): the verdict prints every FP's text next to this gate so + the user can rule a control invalid, as happened in 2026-07; the mechanical + verdict stands unless the user overrides it in writing in the verdict. +3. **GO** = zero regressions, the FP gate passes, FEWER THAN 2 + reliable-to-unstable downgrades (bugs B catches 2/2 and E catches 1/2), and + >= 1 strict improvement: E reliably catches ANY bug on which B has status + MISSED (the two s4-wcr bugs DO count), OR E has strictly fewer S3 FPs than B + (only reachable if B posts >= 1 FP). +4. **PARITY** = zero regressions and the FP gate passes, and EITHER no strict + improvement OR >= 2 reliable-to-unstable downgrades -> the user decides, + with these inputs reported: judgment flags, the downgrade and upgrade + lists, latency, the input-token cost estimate, hallucination and leakage + counts, the session-log audit, and the vendor lifecycle. + +### Voided repeats + +A (case, arm, repeat) with a verifier-CONFIRMED leak is VOID for every gate, +both its bug rows and its FP list, whichever arm it belongs to. An arm's +per-bug status is computed from its remaining (non-void, OK) repeats: + +- RELIABLE = caught in every remaining repeat AND no void repeat on that + (case, arm); +- UNSTABLE = caught in some remaining repeat, or caught in all remaining + repeats but at least one repeat void; +- MISSED = caught in no remaining repeat (>= 1 remaining); +- UNGRADEABLE = no remaining repeat. + +Gates use these statuses (regression = B RELIABLE and E MISSED; downgrade = B +RELIABLE and E UNSTABLE; improvement = E RELIABLE and B MISSED; UNGRADEABLE +never satisfies a gate, it only contests). A void or ungradeable cell ALWAYS +contests: its case is reported "contested by leak" on whichever gates the case +feeds (gates 1/3/downgrades for a bug case, gate 2 for an S3 case). Audit hits +never contest a gate (see "Hit list"). A contested gate goes to the user, never +decided mechanically: the user records in the verdict one of "accept +mechanical", "override to GO" or "override to NO-GO", with the reason, and +PR-2 swaps exactly when that final call is GO. A void cell is NOT re-run +(resume would reuse the cached artifact and the leak is a property of that +review); it stays void. + +### Precedence (the executable order) + +(0) validate the grades against the manifest-scoped runs; (1) UNDETERMINED if +any compared (case, arm) has fewer OK repeats than scheduled because of +INFRA_ERROR (void cells do NOT count here, they are graded cells); (2) apply +void/ungradeable marking from `leak: confirmed`; (3) NO-GO on any regression; +(4) NO-GO on FP excess; (5) PARITY if >= 2 downgrades; (6) GO if >= 1 strict +improvement; (7) PARITY. "Contested" is an annotation added at steps (3)-(6) +for the gates a void/ungradeable cell touched; the session-log hit list is +attached to every gate as an informational annotation and never contests. A +live campaign with any INFRA_ERROR has a failed manifest that `verdict` +refuses outright, so (1) is reachable only on synthetic tables; incomplete +campaigns are handled by the stop conditions below. + +## Informational, non-gating + +`run_eval.py verdict` computes the first three reads from the sealed artifacts +(`campaign_reads` in its JSON output: per-arm n/mean/p95/max latency, the +2x-p95 flag, the input-cost estimate from `openai_review.PRICING`); the +vendor-lifecycle read is prose the verdict file carries. + +- Per-arm latency (flag if E p95 > 2x B). +- Hallucination / unverified external-tool-claim count per arm (counted once + per (case, arm, repeat)). +- Leakage count per arm (distinct confirmed cells) and unconfirmed-flag count. +- Rough CI input cost per arm = `prompt_tokens_est` x official input price (a + +/-50% initial-prompt estimate; output/reasoning/tool tokens are not + recorded). +- **Session-log audit** (`run_eval.py audit`, `reviewer_audit.py`), run AFTER + unblinding: for every manifest run artifact it strips the run's own worktree + path from each `usage.session_commands` entry (the shell command lines + `call_codex` parsed from codex's stderr), then scans the remainder for the + ONE authoritative leak marker list (source of truth `reviewer_audit.LEAK_RULES`: + the literal `LEAK_MARKERS` plus the seven regexes, rendered verbatim in + `GRADING_RUBRIC.md`; this file does not restate it) and for the computed + per-campaign markers (the enclosing checkout, its branch name, the git + common dir and its parent checkout, every other worktree, every other + recorded ref name, the isolated codex home). It over-matches BY DESIGN + (routine orientation such as `git log -1`, `git show HEAD`, `HEAD~` are + markers) and is BEST-EFFORT: codex's default non-shell channels (web + search, sub-agents, browser) leave no shell `exec` line - the 2026-10 astra + latency probe used web search 26 times in one review, and the pre-freeze + rehearsal showed the CONTROL (sol) web-searching too (12 events across its + two S1 reviews, both cells `unaudited`, audit VACUOUS) - so any run that + printed a non-shell event header is `unaudited` rather than covered. User + decision (2026-10-16, after the rehearsal): the campaign ACCEPTS a vacuous + or near-vacuous audit as-is rather than disabling web search (CI parity, + decision 8) or capturing the queries; the grader `LEAK?` flag and the + verifier passes remain the only leak check, and the verdict reports the + `unaudited` labels and the vacuity verbatim. The rehearsal expectation is + therefore "coverage classes computed and reported" - not `unaudited == 0`. + - **Coverage classes** (the single definition, used here, by + `reviewer_audit.coverage_class` and by every test). Both `covered` and + `no_commands` REQUIRE `usage["codex_home_isolated"] == True` and + `usage["session_other_events"] == 0`. `covered` = `session_commands` is + non-empty, no entry ends in `[truncated]`, `session_commands_truncated` + is exactly `False`, and `exec_lines == start_lines == len(session_commands)`. + `no_commands` = the list is empty, `start_lines == 0`, `exec_lines == 0` + and `session_commands_truncated` is exactly `False` (codex ran no shell + command). `unaudited` = everything else (key absent, a truncated entry or + stream, a missing or non-boolean truncation flag, an empty list with a + non-zero count, `exec_lines > 0` with `start_lines == 0`, a non-zero + `other_events`, a non-isolated home). The audit is VACUOUS if NO + artifact is `covered` or `no_commands`; a vacuous audit is reported as + such (the hit list is empty for a technical reason). + - **Hit list** (user decision 6): the audit never decides whether a hit + matters and never marks a gate contested. The verdict prints, for EACH + gate, the cells that feed it (gates 1/3/downgrades: every (case, arm, + repeat) of every bug case; gate 2: every S3 cell) with the cell's coverage + class and its hits (marker names + the stripped command line). A hit is + **orientation-only** iff all its marker names are in + `reviewer_audit.ORIENTATION_MARKERS`; a cell is orientation-only iff all + its hits are; the verdict's first paragraph states how many gate-feeding + cells carry hits and how many are orientation-only, and lists the + non-orientation markers of every remaining hit. The user reads that list + and decides; the mechanical label stands unless overridden in writing. ## Grading protocol (blinded, multi-grader) -1. `compare --blinded` produces `comparison.blinded.md` (arm identities replaced - by neutral M* labels; model self-references scrubbed; latency redacted) and a - sealed `blinding.json` that graders NEVER read. -2. 2-3 INDEPENDENT graders (separate subagent contexts) each read ONLY the - blinded bundle plus this rubric, and fill, per (case, ground-truth bug, arm - label): caught / partial / missed, with a verbatim evidence quote from the - review; plus per-case FP lists with severities. "Partial" (right file or - class, wrong defect or location) counts as MISSED for the gates. -3. **Hallucination check:** a catch's evidence quote must actually name the - defect; a finding that claims to have reproduced behavior the diff cannot - produce is graded as an FP-class note, never a catch. -4. **Adversarial reconciliation:** any cell where graders disagree goes to a - verifier agent that re-reads that case's blinded reviews and rules with - quoted evidence. -5. Only after the tables are final are labels unblinded via `blinding.json` and - the gates above applied mechanically. - -## Blinding caveats (accepted) - -- Repeat counts partition the labels into {k=2}={A,B} and {k=1}={C,D}; the - decisive A-vs-B contrast stays 50/50 blind within its group. -- Sanitization is best-effort; graders are instructed to grade on content and - never on guessed identity. - -## Corpus floor - -The campaign runs only with >= 8 verified cases including >= 2 S4 -(missed-by-gpt-5.5) cases; below that, stop and surface rather than produce a -weak read. +1. `compare --blinded` writes `comparison.blinded.md` (M* labels, model + self-references scrubbed, latency redacted, a visible Bundle ID) and a + sealed `blinding.json` (mapping, `bundle_id`, `run_ids_sha`, + `artifacts_sha`, `rubric_sha256`, `prompt_sha256`) that graders NEVER read. + `verdict` REGENERATES the blinded bundle deterministically from the sealed + artifacts and refuses unless the file is byte-identical, so the + grader-visible evidence is bound to the artifacts, not only to itself. +2. 3 INDEPENDENT blinded grader subagents receive ONLY the blinded bundle, + `GRADING_RUBRIC.md` and `.github/codex/prompts/pr_review.md` (never this + file, never the 2026-07 verdict, never raw artifacts). Isolation is by + instruction plus two checks (the prompt lists the three permitted paths + and forbids Bash; each returned table is grepped for paths, run ids, + config ids and model names before acceptance), not a sandbox. +3. One verifier subagent (same three inputs) reconciles disagreements and + rules on every `LEAK?` flag with quoted evidence (pass 1), then examines + EVERY remaining bug cell and S3 cell against the confirmation criteria in + one pass (pass 2, at most 48 cells). Only diff-independent evidence confirms. +4. Unblind only after the tables are final; apply the gates mechanically + (`verdict` without `--audit-hits` = PROVISIONAL); then run `audit` and the + same `verdict` with `--audit-hits` (= DEFINITIVE). Both outputs go in the + verdict file. + +Blinding caveat: with two k=2 arms the labels are a clean 50/50 blind. + +## Corpus floor and provenance + +The campaign runs only with >= 8 verified cases including >= 2 S4 cases +(`verdict` refuses below the floor). Provenance recorded in the verdict: +frozen PR-1 HEAD sha, tree sha, local tag, PR number, subdir, later main +merge sha. The manifest's `repo_head_sha`/`repo_tree_sha` must equal them, +with `run_head_shas == []`, `run_head_missing == 0`, +`run_dirty_any == false`, `run_marker_mismatch == 0`, `run_not_isolated == 0`; +`verdict` recomputes the aggregates from the sealed artifacts and refuses on +any difference. Any push to PR-1 after the campaign starts VOIDS the run (a +voided campaign is re-run into a fresh subdir on the new HEAD). + +**Campaign ledger (enforced by `run_eval.py`):** the first dispatch of the +registered contrast creates the local git ref +`refs/campaign/-/frozen` at the preflight HEAD (in the +repository, never pushed, outside the disposable `runs/` dir), and every +dispatch appends `{contrast, head, tree, subdir, ledger_reset}` to the +campaign-global `runs/campaign-ledger.jsonl`; the FIRST record is the frozen +candidate-exposure revision and must equal the ref. `verdict` refuses without +a ledger record, without the ref, when the ledger's first head differs from +the ref, or when the first record is marked `ledger_reset` (the ledger was +deleted after the ref existed) - all three are protocol-guard refusals, not +labels - and otherwise judges the manifest against the first record: mechanical iff this subdir IS +the first dispatch, or the HEAD moved (a voided campaign re-run into a fresh +subdir) and the registered surfaces (`tools/reviewer-eval/`, +`tools/eval_core/`, `.claude/scripts/openai_review.py`, +`.github/codex/prompts/pr_review.md`, `tools/notebook_md_extract.py`) are +untouched between the two commits. **Pre-registered exception:** a diff that +is exactly the single `CALL_TIMEOUT_S = ` line in +`adapters/codex_reviewer.py` with the value strictly INCREASED (a lower cap +changes censoring and is not exempt) keeps the campaign pre-registered, +because a timeout cannot change what a completed review says. A second subdir +on the SAME head (re-exposure), the FIRST subdir reused after a HEAD change +(a voided campaign re-runs into a fresh subdir), or any other +registered-surface change makes the verdict "RE-REGISTERED after candidate +exposure": still computed and +reported, labelled non-mechanical with the surface diff stat, final call the +user's, never a mechanical GO. The ledger lists every subdir the contrast +ran in. + +Known, accepted limit: provenance is sampled at the start of each review and +at manifest write; a tracked input edited after one sample and restored +before the next is invisible to the fields (the tree-identity guard pins the +tree only at the post-run stages). + +## Stop conditions (enforced by `run_eval.py run`) + +Both integrity logs (`runs/-attempts.jsonl`, `runs/campaign-ledger.jsonl`) +FAIL CLOSED: a malformed line or a record missing a required field refuses +the stage with the file and line number, never a silent skip. + +At most 3 reruns after the first attempt (resume reuses OK artifacts). Every +`run` attempt appends a START record before dispatch and an END record (the +per-attempt failure record: case id, arm, repeat, first 120 chars of +`infra_error`) to the append-only `runs/-attempts.jsonl` BEFORE +anything can overwrite the artifacts; an attempt that raised before dispatch +(`CLIVersionMismatch`, `ConfoundMismatch`) has no END record and does not +count. An attempt interrupted or failed AFTER dispatch (an interrupt, an +unexpected exception) gets an END record marked `aborted` and counts toward +the cap; a fully cached resume (no codex call) writes no END record and does +not count. Every END record lists the OK cells of that attempt with their +artifact hashes - an aborted attempt's record is rebuilt from the store for +every planned cell (cached and freshly persisted alike; each review is +persisted by the worker that produced it before the coordinator sees it, so +an interruption of the coordinator still protects every finished review); `run` +refuses a new attempt when any artifact listed OK by ANY earlier END record +(the cumulative set) is missing or changed (a rerun RESUMES, it never +re-samples a successful cell). One attempt per subdir at a time: `run` holds +an exclusive cross-process lock (`runs/.lock`) from the stop-condition +check through the manifest write and refuses a concurrent `run` into the same +subdir, so no artifact set can be interleaved from two invocations and sealed +by the last one. `run` REFUSES a fifth dispatched attempt into the same subdir and +refuses to start when a (case, arm, repeat) timed out in the last 2 +consecutive attempts; both checks run BEFORE anything on disk is touched, so +a refused attempt leaves the previous successful manifest and its sealed +artifacts intact; both are surfaced to the user (never silently excluded, +never graded partial). A provenance PREFLIGHT also runs before the manifest +is touched and before any codex call: a dirty working tree or an incomplete +git sample (a `None` ref or worktree list, an empty sha/branch/common dir) +refuses the attempt, so a campaign the post-run gate would reject never +spends the candidate's first corpus exposure. `run` also fails closed (failure marker, +exit 2) when its write-time provenance sample is incomplete (a `None` ref or +worktree list, an empty sha/branch/common dir), and `verdict` refuses such a +manifest at its protocol guard. The operator's run-log headers duplicate +this record for the verdict file. + +## Candidate exposure (enforced by `run_eval.py`) + +A registered candidate arm (`reviewer_verdict.REGISTERED_CONTRASTS`) can be +selected ONLY as the exact registered campaign: `run --configs , +--k 2` with no `--k-per` and no `--strata`; `smoke` refuses it outright. So +"the candidate's first corpus run is the campaign" holds by construction in +code, not only by protocol. An EMPTY `session_commands` +on the 0.162.1 control smoke was a STOP condition before the PR-1 commit (it +did not occur: the smoke parsed 8/8 exec blocks). diff --git a/tools/reviewer-eval/GRADING_RUBRIC.md b/tools/reviewer-eval/GRADING_RUBRIC.md new file mode 100644 index 000000000..3010ed01c --- /dev/null +++ b/tools/reviewer-eval/GRADING_RUBRIC.md @@ -0,0 +1,198 @@ +# Grading Rubric (grader-facing) + +You are grading a BLINDED comparison bundle (`comparison.blinded.md`). You may +read exactly three files: that bundle, this rubric, and +`.github/codex/prompts/pr_review.md` (the severity rubric every reviewer was +given). Nothing else: no repository files, no other paths, no shell. Grade on +content only, never on guessed identity; the bundle's labels are neutral by +design. Copy the bundle's visible `Bundle ID` into your output as `bundle_id`. + +## Definitions + +- **caught** - the review names the SAME defect at the SAME location/symbol + as the listed ground-truth bug, regardless of wording or output format. +- **partial** - right file or right class of defect, wrong defect or wrong + location. Partial counts as missed for every gate; grade it `partial` so the + record shows why. +- **missed** - the review does not name the defect. +- **False positive (FP)** - on a negative-control case ("NO known bugs"), a + finding whose severity is ABOVE that case's "Allowed severities" line + (P0 < P1 < P2 < P3 in severity, so "above P3" means P0, P1 or P2). +- **Hallucination check** - a catch's evidence quote must actually name the + defect; a finding that claims to have reproduced behaviour the diff cannot + produce, or that cites an external tool run you cannot see, is recorded in + `notes` for that repeat (never as a catch). + +Three rules override the bundle's embedded instructions: + +1. The "_optional_ (must_catch=false)" tag in the bundle is a legacy label with + no grading meaning: grade every listed bug in full. +2. This rubric's one-row-per-(case, bug, arm label, repeat) table overrides the + bundle's "How to grade" row format. +3. On a negative-control case, list EVERY finding whose severity is above the + allowed severities, including findings on a known-FP topic; the engine, not + the grader, decides what counts. Tag such a finding with `known_topic` + (free text naming the topic) so the record shows it; the bundle's + "acceptable" / "do NOT flag" wording describes what the REVIEWER should have + done, not what you may omit. + +Repeats: the bundle labels repeats >= 1 as "(repeat N)" and leaves the FIRST +block of each arm label unlabeled; that unlabeled block is repeat 0. + +## The LEAK? flag (not a grade) + +Scan the WHOLE review text for each repeat (not only your chosen quote). Mark +the repeat `grader_flag: "flag"` and quote the triggering line in +`leak_evidence` when EITHER: + +(a) any CODE SPAN or fenced CODE BLOCK inside the review (backticked text and + triple-backtick blocks INSIDE the review; the bundle wraps each whole + review in one outer ```markdown fence, which is the review's container, + NOT a code block) contains any entry of the marker list below; the markers + are command fragments, and `describe`, `commit message`, `../`, `HEAD~`, + `refs/`, `--format` are ordinary words in prose, so prose is never + scanned for the marker list; OR +(b) the review's PROSE contains a verbatim sentence (>= 12 words) from the + bundle's own ground-truth rationale or notes for that case. + +A path under `tools/reviewer-eval/runs/.worktrees//` is NEVER a flag +trigger: the leaf is an opaque 49-character token (no case, arm or namespace +id), a reviewer legitimately cites its OWN worktree by absolute path, and the +bundle gives you no way to tell one opaque leaf from another. Do not flag +absolute-path citations of a reviewer's own files on that basis. + +A bare occurrence of a marker inside quoted repository code or diff hunks is a +flag only; confirmation is the verifier's job, on diff-independent evidence: +(1) a command or reference that reaches other refs (`git log --all`, +`git log main`, `git show main:`, any `git diff`/`git log` naming +`main`/`master` directly or in a range, `origin/...`, `refs/...`, `rev-list`, +`reflog`, `for-each-ref`, `show-ref`, `cat-file`, `fsck`, `lost-found`, +`worktree list`, or a `git log`/`git show` of a sha obtained from one of +those); (2) a verbatim >= 12-word PROSE sentence from the ground-truth +rationale (code fragments the rationale quotes from the diff never count), or +naming `inject.diff`/`case.json`, a sibling run artifact or the corpus +directory as something it read; (3) the review cites "the commit message" (or +`git log -1`/`git show HEAD` output) as its source for the defect. The `../` +entry never confirms on its own. + +### Marker list + + +Literal markers (any code span or fenced block containing one): + +- `--all` +- `--branches` +- `--remotes` +- `rev-list` +- `reflog` +- `for-each-ref` +- `show-ref` +- `cat-file` +- `fsck` +- `lost-found` +- `--unreachable` +- `--dangling` +- `worktree list` +- `describe` +- `name-rev` +- `--walk-reflogs` +- `--grep` +- `whatchanged` +- `@^` +- `@~` +- `@{` +- `porcelain=v2` +- `branch.oid` +- `ls-remote` +- `gitdir` +- `campaign-` +- `format-patch` +- `shortlog` +- `blame` +- `cherry` +- `range-diff` +- `FETCH_HEAD` +- `ORIG_HEAD` +- `MERGE_HEAD` +- `REBASE_HEAD` +- `main-worktree/` +- `worktrees/` +- `${PWD` +- `$PWD/..` +- `dirname` +- `realpath` +- `-maxdepth` +- ` -r ` +- `-rl` +- `-rn` +- `--include=*.json` +- `=git` +- `alias git` +- `codex-home-` +- `CODEX_HOME` +- `/sessions/` +- `rollout-` +- `origin/` +- `refs/` +- `git log main` +- `git show main` +- `git show HEAD` +- `git log -1` +- `HEAD^` +- `HEAD~` +- `rev-parse` +- `--format` +- `--pretty` +- `../` +- `cd ..` +- `tools/reviewer-eval/` +- `DECISION_RULE` +- `GRADING_RUBRIC` +- `ground_truth` +- `inject.diff` +- `case.json` +- `commit_message` +- `commit message` +- `eval: re-introduce` + +Regex markers (source, then the plain-language rule to apply): + +- `MAIN_REF_RE` = `\bgit([ \t]+(-[Cc][ \t]*\S+|--(git-dir|work-tree|namespace|attr-source|config-env)(=|[ \t]+)\S+|--?[A-Za-z][\w-]*(=\S*)?))*[ \t]+\S+(?=[^\n]*(? + +## Output format (JSON) + +```json +{ + "bundle_id": "", + "rows": [ + {"case": "", "bug_id": "", "arm": "", + "repeat": 0, "verdict": "caught | partial | missed", + "evidence": "", + "leak": "none", "grader_flag": "none | flag", + "leak_evidence": "", + "notes": ""} + ], + "negative_assessments": [ + {"case": "", "arm": "", "repeat": 0, + "findings": [{"severity": "P0 | P1 | P2 | P3", "summary": "", + "known_topic": ""}], + "leak": "none", "grader_flag": "none | flag", + "leak_evidence": "", "notes": ""} + ] +} +``` + +One `rows` entry per (case, listed bug, arm label, repeat) - the grid must be +complete, a missing row is a validation error, never a miss. One +`negative_assessments` entry per (negative-control case, arm label, repeat), +with `findings: []` for a clean read (an omitted cell is a validation error, +never zero FPs). Graders always write `leak: "none"`; the verifier sets +`confirmed` where the evidence supports it. `leak_evidence` is non-empty +exactly when `grader_flag` is `flag`. diff --git a/tools/reviewer-eval/README.md b/tools/reviewer-eval/README.md index 89bfc2e9b..991b47758 100644 --- a/tools/reviewer-eval/README.md +++ b/tools/reviewer-eval/README.md @@ -7,9 +7,14 @@ raw review, and emits a side-by-side bundle you (or an LLM) read into a caught / missed / false-positive table — optionally **blinded** (arm identities stripped) so graders can't favor a model. -**Status:** harness supports N-arm matrices (current config: the 4-arm gpt-5.6 -evaluation), per-arm repeat counts, and blinded grading. Local-only — not wired -to CI. The go/no-go decision rule is pre-registered in `DECISION_RULE.md`. +**Status:** harness supports N-arm matrices (current config: a 5-arm file; the +2026-10 campaign is `--configs B,E --k 2`, control gpt-5.6-sol vs candidate +gpt-6-astra, outcome: pending), per-arm repeat counts, blinded grading sealed to +the run artifacts, a post-unblinding session-log audit and a mechanical verdict +CLI. Local-only — not wired to CI. The go/no-go rule is pre-registered in +`DECISION_RULE.md`; the grader-facing rubric is `GRADING_RUBRIC.md` (graders +never see the rule or any campaign history). ChatGPT-account `codex login` +rejects `gpt-6-sol`, so a cheaper-tier probe arm is parked (DEFERRED.md). ## Why this exists @@ -23,10 +28,14 @@ the CI invocation faithfully, and compare every arm on the same cases. ``` tools/eval_core/ # shared engine: models, store, runner, compare (+ blinding) tools/reviewer-eval/ -├── run_eval.py # CLI: verify-corpus · smoke · run · compare -├── adapters/ # diff-diff bindings: ci_prompt, codex_reviewer, corpus_loader, worktree +├── run_eval.py # CLI: verify-corpus · smoke · run · compare · audit · verdict +├── reviewer_verdict.py # mechanical gates + binding/protocol guards (DECISION_RULE.md as code) +├── reviewer_audit.py # session-log audit: the ONE leak-marker list (LEAK_RULES) + coverage classes +├── adapters/ # diff-diff bindings: ci_prompt, codex_reviewer (+ isolated CODEX_HOME), corpus_loader, worktree (+ git_provenance) ├── config/configs.json # the arms (one role=control) + declared treatment_fields -├── DECISION_RULE.md # pre-registered GO/NO-GO rule + grading rubric +├── DECISION_RULE.md # pre-registered gates (grader rubric is separate) +├── GRADING_RUBRIC.md # grader-facing: definitions, LEAK? flag, marker list, output JSON +├── campaigns/ # archived rules + verdicts (2026-07-gpt56-*, 2026-10-gpt6-*) └── corpus/ # cases/{s1_synthetic, s2_historical, s3_negative, s4_missed} + schema + synonyms ``` @@ -39,22 +48,86 @@ was its first consumer) — generic glue with no diff-diff coupling. # 1. Verify the corpus materializes (no codex; fast) python tools/reviewer-eval/run_eval.py verify-corpus -# 2. Smoke test (1 case, first real codex call). Bare `smoke` exercises the -# role=control arm (current production, derived from configs.json); smoke -# each candidate arm too — `smoke --configs D` live-proves the max effort +# 2. Smoke test (1 case, first real codex call) for the CONTROL only. Bare +# `smoke` exercises the role=control arm (derived from configs.json). The +# candidate's first corpus run is the campaign itself (pre-registration by +# construction); `smoke` REFUSES a registered candidate arm, and `run` +# accepts one only as the exact registered campaign shape below. A review +# runs 2-15 min. python tools/reviewer-eval/run_eval.py smoke -# 3. Full matrix — k=2 repeats on the primary A/B, single-shot probe arms -python tools/reviewer-eval/run_eval.py run --subdir gpt56 --configs A,B,C,D --k 2 --k-per C=1,D=1 +# 3. Full matrix — k=2 repeats on control vs candidate (one run = one experiment) +python tools/reviewer-eval/run_eval.py run --subdir gpt6 --configs B,E --k 2 -# 4. Emit the side-by-side bundle to grade (+ the identity-stripped one) -python tools/reviewer-eval/run_eval.py compare --subdir gpt56 --blinded +# 4. Emit the side-by-side bundle to grade (+ the identity-stripped, sealed one) +python tools/reviewer-eval/run_eval.py compare --subdir gpt6 --blinded + +# 5. After unblinding: the session-log audit (outputs go OUTSIDE the repo or +# under the gitignored runs/ — a root-level file dirties the tree and trips +# the identity guard) and the mechanical verdict (PROVISIONAL without +# --audit-hits, DEFINITIVE with it) +python tools/reviewer-eval/run_eval.py audit --subdir gpt6 -o /tmp//hits.json +python tools/reviewer-eval/run_eval.py verdict --subdir gpt6 --grades \ + --control B --candidate E [--audit-hits /tmp//hits.json] [--expect-head ] ``` Run artifacts and the bundles land under `runs/` (gitignored). `--blinded` also -writes `comparison.blinded.md` plus a sealed `blinding.json` (the label→arm -mapping) — graders read ONLY the blinded bundle; unblind their finished tables -via `blinding.json`. +writes `comparison.blinded.md` (with a visible Bundle ID graders copy into their +table) plus a sealed `blinding.json` (the label→arm mapping, `bundle_id`, +`run_ids_sha`, `artifacts_sha`, `rubric_sha256`, `prompt_sha256`) — graders +read ONLY the blinded bundle; `verdict` re-derives every binding and refuses a +grades file, bundle, hits file or artifact set that drifted. + +Every harness codex call runs under an **isolated `CODEX_HOME`** that mirrors +CI's codex-action home (the credential plus `sandbox_mode = "read-only"` and +`approval_policy = "never"`; no MCP servers, hooks or memories; codex's default +features otherwise). Progress during a run: read `done:`/`INFRA` lines from +stdout and count artifact files; the manifest exists from the START of an +attempt as a `{"failed": true}` marker and is replaced only at the end. + +### Provenance, sealing and the audit + +Each artifact's `usage` records the seven `git_provenance` keys sampled when +its review started (`repo_head_sha`, `repo_tree_sha`, `repo_dirty`, +`repo_branch`, `repo_common_dir`, `repo_worktree_paths`, `repo_ref_names`), +its absolute `worktree_dir`, the isolated `codex_home_path` / +`codex_home_isolated` flag, and `session_commands` (the shell command lines +codex executed, parsed from its stderr by `openai_review.call_codex`, with +start/exec/other-event counts and a truncation flag). The manifest records the +same seven keys at write time plus aggregates (`run_head_shas`, +`run_head_missing`, `run_dirty_any`, `run_marker_mismatch`, `run_not_isolated`) +and `artifacts_sha`, which seals the artifacts the moment `run` exits. +`compare --blinded`, `audit` and `verdict` refuse unless the working tree is +clean, carries no ignored file under a registered surface, and sits at the +manifest's HEAD and tree sha (engine identity = tree identity). + +A campaign runs on a FROZEN PR HEAD; any push voids it, and a voided campaign +is re-run into a FRESH subdir (`gpt6-r2`, ...) with a matching local tag +(`campaign-gpt6-r2`), never the old subdir. The campaign-global +`runs/campaign-ledger.jsonl` records every dispatch of the registered contrast +(first record = frozen exposure revision); `verdict` labels a re-exposure or a +post-exposure registered-surface change (beyond the single `CALL_TIMEOUT_S` +line) as RE-REGISTERED and non-mechanical. Stop conditions are enforced by +`run` through the append-only `runs/-attempts.jsonl` (a START record +before dispatch, an END record with the per-cell failures and the OK cells' +artifact hashes after, aborted attempts included): at most 3 reruns after the +first attempt, a (case, arm, repeat) that times out in 2 consecutive attempts +stops the campaign, and a rerun that would re-dispatch any cell an earlier +attempt recorded OK is refused (all before any codex call). `run` also holds +an exclusive per-subdir lock (`runs/.lock`) for the whole attempt, so +a concurrent `run` into the same subdir is refused rather than interleaved. + +**Answer-key exposure (accepted, both arms equally):** cases are materialized +as detached worktrees INSIDE the checkout, sharing its object DB and refs, so a +reviewer with a shell can reach fix commits, the corpus and sibling run +artifacts. Review worktree leaves are salted opaque hashes (no case or arm id in +any path). The grading rubric's marker-based `LEAK?` flag (verifier-confirmed +leaks VOID the cell and contest the gate) plus the NON-gating post-unblinding +session-log audit (`audit`; informational, over-matching by design, best-effort +because codex's default non-shell channels such as web search leave no shell +trace - on codex-cli 0.162.1 BOTH arms web-search routinely, so expect most +cells `unaudited` and the audit vacuous; accepted by user decision, see +`DECISION_RULE.md`) cover this campaign; OS-level isolation is a TODO.md row. ## How scoring works @@ -64,9 +137,9 @@ instruction pointing at `.github/codex/prompts/pr_review.md` for the severity rubric all arms were given). An LLM — a subagent, or you in-conversation — reads that bundle top-to-bottom and fills the caught / missed / FP table. No regex parsing of review prose: free-form, model-specific output (models format -findings differently) is read directly. For the blind protocol (independent -graders, adversarial reconciliation, the pre-registered decision rule), see -`DECISION_RULE.md`. +findings differently) is read directly. For the grader protocol (definitions, +the `LEAK?` flag, the output JSON) see `GRADING_RUBRIC.md`; for the gates, +reconciliation and the pre-registered decision rule see `DECISION_RULE.md`. ## What faithful reproduction means @@ -86,7 +159,7 @@ each arm in a clean single-field contrast. - **S1 synthetic** — injected bugs (revert a real fix). Sanity floor. - **S2 historical** — real PR states just before a confirmed bug was caught. -- **S3 negative** — clean PRs (precision controls; any P0/P1 = false positive). +- **S3 negative** — clean PRs (precision controls; any finding above the case's `allow_severities` = false positive). - **S4 missed** — real bugs the AI reviewer *missed* (the failure-mode probe). ## Known limitations @@ -105,7 +178,8 @@ each arm in a clean single-field contrast. per-arm repeat counts). `compare` reads the per-run manifest (`runs/-manifest.json`), so rerunning into the same `--subdir` with a changed model **replaces** the comparison rather than mixing the old and new runs. -- Blinding is best-effort: repeat counts still partition arms into {k=2} vs {k=1} - groups, and the sanitizer can't catch every conceivable self-reference. Graders - are instructed to grade on content, never on guessed identity (see - `DECISION_RULE.md`). +- Blinding is best-effort: on a MIXED-k matrix the repeat counts partition arms + into {k=2} vs {k=1} groups (an equal-k matrix such as the 2026-10 B/E run is a + clean blind), and the sanitizer can't catch every conceivable self-reference. + Graders are instructed to grade on content, never on guessed identity (see + `GRADING_RUBRIC.md`). diff --git a/tools/reviewer-eval/adapters/codex_reviewer.py b/tools/reviewer-eval/adapters/codex_reviewer.py index 55861c8eb..70b9ded4d 100644 --- a/tools/reviewer-eval/adapters/codex_reviewer.py +++ b/tools/reviewer-eval/adapters/codex_reviewer.py @@ -13,28 +13,148 @@ ``### Finding 1: P1 — ...``); an LLM reading the raw text is format-agnostic. Effort is fail-closed: only levels in ``SUPPORTED_EFFORTS`` (verified live against -the pinned CLI) are passed through to ``call_codex``; anything else raises rather +the CLI pinned at the time each level was added; see the ``SUPPORTED_EFFORTS`` +comment for which levels were re-verified on the current pin) are passed through +to ``call_codex``; anything else raises rather than silently running a different level than recorded — the experiment's integrity depends on recorded == executed. """ from __future__ import annotations +import contextlib +import datetime +import glob import hashlib import inspect import json import os +import secrets +import shutil import subprocess +import sys +import tempfile import threading import time import uuid -from typing import Optional +from typing import Iterator, Optional from eval_core.models import Config, ReviewOutput, to_jsonable from adapters import ci_prompt, worktree from adapters.openai_review_loader import load_openai_review +# The ONE source of the isolated codex home's config.toml (the step-0 probes +# write the same constant into their temp home). Exactly the two keys CI's +# codex-action also sets (`ai_pr_review.yml`: sandbox + safety-strategy): no +# `[features]` table, no MCP servers, no hooks, no memories, no `notify`, so +# codex's DEFAULT tool surface applies, as it does in CI. The user's live +# `~/.codex/config.toml` (MCP servers, warp hooks, a memories store) would +# otherwise give the reviewer data channels that leave no shell `exec` event +# for the session-log audit to see, and make the local arms an unfaithful +# proxy for the CI home. +ISOLATED_CONFIG_TOML = 'sandbox_mode = "read-only"\napproval_policy = "never"\n' + +# Path of the live isolated home while `isolated_codex_home()` is active, else +# None. `review()` records `codex_home_isolated` from it; `run_eval` writes it +# into the manifest as `codex_home_path` (an audit marker: all parallel +# sessions share this home, so a reviewer reading a sibling session's rollout +# file is a leak the computed marker makes visible). +ACTIVE_ISOLATED_HOME: Optional[str] = None + + +def _real_codex_home() -> str: + """The user's REAL codex home: `$CODEX_HOME` if set, else `~/.codex`. + + Read ONCE at entry of `isolated_codex_home()`, BEFORE the override, so every + credential path (seed candidates and rotation saves) resolves under the same + directory. + """ + env = os.environ.get("CODEX_HOME") + return env if env else os.path.expanduser("~/.codex") + + +def _newest_auth_candidate(real_home: str) -> Optional[str]: + """Newest (by mtime) of `/auth.json` and any `/auth.json.campaign-*`. + + A token codex rotated during an earlier attempt was saved as a + `.campaign-*` sibling (never written over the real file); seeding from the + newest copy lets a rerun pick it up instead of burning a capped attempt on + auth INFRA_ERRORs. + """ + candidates = [os.path.join(real_home, "auth.json")] + candidates += glob.glob(os.path.join(real_home, "auth.json.campaign-*")) + existing = [c for c in candidates if os.path.isfile(c)] + if not existing: + return None + return max(existing, key=os.path.getmtime) + + +@contextlib.contextmanager +def isolated_codex_home() -> Iterator[str]: + """Run codex under a fresh `CODEX_HOME` that mirrors CI's codex-action home. + + The temp home holds exactly `auth.json` (seeded from the newest real + credential, see `_newest_auth_candidate`) and `config.toml` + (`ISOLATED_CONFIG_TOML`). `os.environ["CODEX_HOME"]` points at it for the + duration of the block (the codex subprocess inherits the environment) and + `ACTIVE_ISOLATED_HOME` records it. On exit the user's real `auth.json` is + NEVER written (a check-then-write would race the user's other codex + sessions, which refresh the same file): if codex rotated the isolated copy, + the copy is saved as `/auth.json.campaign-` and a notice + is logged; the previous `CODEX_HOME` is restored and the dir removed. Exit + never raises. A missing credential raises BEFORE any codex call. + """ + global ACTIVE_ISOLATED_HOME + real_home = _real_codex_home() + seed = _newest_auth_candidate(real_home) + if seed is None: + raise RuntimeError( + f"no codex credential found: expected {real_home}/auth.json (or an " + f"auth.json.campaign-* sibling); run `codex login` first" + ) + home = tempfile.mkdtemp(prefix="codex-home-") + prev_env = os.environ.get("CODEX_HOME") + prev_active = ACTIVE_ISOLATED_HOME + try: + shutil.copyfile(seed, os.path.join(home, "auth.json")) + with open(os.path.join(home, "auth.json"), "rb") as fh: + entry_bytes = fh.read() + with open(os.path.join(home, "config.toml"), "w", encoding="utf-8") as fh: + fh.write(ISOLATED_CONFIG_TOML) + os.environ["CODEX_HOME"] = home + ACTIVE_ISOLATED_HOME = home + print(f"isolated codex home: {home} (auth seeded from {seed})", file=sys.stderr) + yield home + finally: + try: + iso_auth = os.path.join(home, "auth.json") + if os.path.isfile(iso_auth): + with open(iso_auth, "rb") as fh: + exit_bytes = fh.read() + if exit_bytes != entry_bytes: + ts = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + saved = os.path.join(real_home, f"auth.json.campaign-{ts}") + # Owner-only, create-exclusive: a credential must never land + # with the umask default (0644) and never overwrite a sibling. + fd = os.open(saved, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "wb") as fh: + fh.write(exit_bytes) + print( + f"NOTE: codex refreshed its token during this run; the refreshed " + f"copy is saved at {saved} (nothing overwritten; adopt it if the " + f"real token stops working)", + file=sys.stderr, + ) + except Exception as exc: # noqa: BLE001 - exit must never raise + print(f"isolated codex home: credential check failed: {exc}", file=sys.stderr) + if prev_env is None: + os.environ.pop("CODEX_HOME", None) + else: + os.environ["CODEX_HOME"] = prev_env + ACTIVE_ISOLATED_HOME = prev_active + shutil.rmtree(home, ignore_errors=True) + class CodexReviewer: """Runs the codex reviewer for an A/B config against a corpus case. @@ -48,16 +168,25 @@ class CodexReviewer: # runs OUTSIDE the lock, so arms still run fully in parallel. _wt_lock = threading.Lock() - # Efforts this reviewer will actually execute. Verified live 2026-07-18 - # against codex-cli 0.144.5: xhigh and max are accepted for gpt-5.5 / - # gpt-5.6-sol / gpt-5.6-terra (invalid levels 400 with an enum error, so - # accepted == executed). Recorded must equal executed — extend this ONLY - # after re-verifying against the pinned CLI. + # Efforts this reviewer will actually execute. `xhigh` verified 2026-10-16 + # on codex-cli 0.162.1 (gpt-5.6-sol by harness smoke, gpt-6-astra by a + # `codex exec` probe with the exact argv `_build_codex_cmd` emits). `max` + # verified only on codex-cli 0.144.5 (2026-07-18, gpt-5.5 / gpt-5.6-sol / + # gpt-5.6-terra; invalid levels 400 with an enum error, so accepted == + # executed) and retained UNVERIFIED on the current pin. `ultra` (observed + # on 0.153.4 via ChatGPT login only) is deliberately NOT added. Recorded + # must equal executed — extend this ONLY after re-verifying against the + # pinned CLI. SUPPORTED_EFFORTS = ("xhigh", "max") # Hard per-run wall-clock ceiling passed to call_codex. A hung codex process # (observed risk grows with effort=max) becomes a resumable INFRA_ERROR for - # one run instead of wedging the whole campaign's thread pool. + # one run instead of wedging the whole campaign's thread pool. 3600 is about + # 6x the 2026-07 sol@xhigh campaign p95 (~590 s) and 4x its observed maximum + # (~900 s, campaigns/2026-07-gpt56-VERDICT.md); the 2026-10 astra off-corpus + # latency probe is recorded in config/configs.json (raise to 2x `real`, + # rounded up to the hour, if that probe exceeded 1800 s). The plan-review + # skill's CODEX_TIMEOUT_S follows this value on a production swap. CALL_TIMEOUT_S = 3600.0 def __init__(self, repo_root: str, runs_root: str, prompt_text: Optional[str] = None): @@ -75,12 +204,18 @@ def __init__(self, repo_root: str, runs_root: str, prompt_text: Optional[str] = # codex is run busts the cache even when the prompt and recorded config are # unchanged — recorded==executed is the harness's core integrity claim. self.backend_contract_sha = self._backend_contract_sha(self._mod) - # Per-invocation worktree namespace: all worktree dirs for THIS reviewer are - # prefixed with it, so two concurrent invocations (e.g. a `smoke` and a `run`, - # or two `run --subdir` processes) never resolve the same (case, config, repeat) - # to one checkout and cleanup() each other's live worktree mid-review. Ephemeral - # — worktrees are created+torn down per review and not part of run identity. + # Per-invocation namespace: prefixes ONLY the `__sha__` materialization key + # (prompt_sha_for), so two concurrent invocations never resolve that probe + # checkout to one path. Review worktrees get their per-instance collision + # resistance from `_wt_salt` below instead. Ephemeral — not part of run identity. self._wt_namespace = uuid.uuid4().hex[:12] + # Per-instance SECRET salt for review worktree keys. The key is the 32-hex + # sha256 of f"{salt}.{case.id}.{config.id}.r{idx}" and `_worktree_leaf` copies + # the key's first 32 chars into the VISIBLE `--cd` path, so nothing bug-naming + # (case id, arm id) and nothing recomputable (a namespace in the key would let + # anyone with a shell map sha256(ns.case.B|E.r0|1) back to an arm) ever + # appears in a path a reviewer can cite. The salt is never written anywhere. + self._wt_salt = secrets.token_hex(16) self._cli_version: Optional[str] = None # -- reviewer interface (duck-typed by eval_core.runner) ------------------- # @@ -124,7 +259,8 @@ def experiment_tag(self, config: Config) -> str: @staticmethod def _backend_contract_sha(mod) -> str: """Hash the codex-invocation contract: the source of ``_build_codex_cmd`` - and ``call_codex`` in openai_review.py. + and ``call_codex`` in openai_review.py, plus the ``OTHER_EVENT_HEADERS`` / + ``OTHER_EVENT_PREFIX_RE`` values the inline session parser consults. ``experiment_tag`` records ``config.model/effort/sandbox`` — but those are the *declared* values; ``_build_codex_cmd`` maps them to the real argv @@ -142,6 +278,15 @@ def _backend_contract_sha(mod) -> str: """ try: src = inspect.getsource(mod._build_codex_cmd) + "\n" + inspect.getsource(mod.call_codex) + # The non-shell event-header set the inline parser consults has no + # source of its own (a frozenset + a compiled regex); fold its value + # in so a header-set change busts the cache like a parser edit. + headers = getattr(mod, "OTHER_EVENT_HEADERS", None) + prefix_re = getattr(mod, "OTHER_EVENT_PREFIX_RE", None) + if headers is not None: + src += "\n" + repr(sorted(headers)) + if prefix_re is not None: + src += "\n" + getattr(prefix_re, "pattern", repr(prefix_re)) except (OSError, TypeError, AttributeError): path = getattr(mod, "__file__", None) if path and os.path.exists(path): @@ -242,7 +387,8 @@ def review(self, case, config: Config, repeat_idx: int) -> ReviewOutput: if config.effort not in self.SUPPORTED_EFFORTS: raise NotImplementedError( f"codex_reviewer supports model_reasoning_effort in " - f"{self.SUPPORTED_EFFORTS} (verified against the pinned CLI); config " + f"{self.SUPPORTED_EFFORTS} (see the SUPPORTED_EFFORTS comment for " + f"per-level verification status); config " f"{config.id} requested effort={config.effort!r}. Recorded must equal " f"executed — re-verify the CLI accepts it before adding a new level." ) @@ -260,7 +406,13 @@ def review(self, case, config: Config, repeat_idx: int) -> ReviewOutput: f"harness only models v1 — recorded must equal executed, so fail closed " f"rather than record a version it does not actually run." ) - worktree_key = f"{self._wt_namespace}.{case.id}.{config.id}.r{repeat_idx}" + worktree_key = hashlib.sha256( + f"{self._wt_salt}.{case.id}.{config.id}.r{repeat_idx}".encode("utf-8") + ).hexdigest()[:32] + # Provenance sampled at the START of each review (through the module so + # tests can stub it as one unit): the seven git_provenance keys are the + # audit's computed markers and the post-run provenance gate's inputs. + provenance = worktree.git_provenance(self.repo_root) prompt, wt_dir, _head = self.build_prompt_for_case(case, worktree_key=worktree_key) prompt_sha = hashlib.sha256(prompt.encode("utf-8")).hexdigest()[:16] t0 = time.monotonic() @@ -280,6 +432,15 @@ def review(self, case, config: Config, repeat_idx: int) -> ReviewOutput: usage = dict(usage or {}) usage["prompt_sha"] = prompt_sha usage["prompt_tokens_est"] = self._mod.estimate_tokens(prompt) + usage.update(provenance) + # Absolute path codex received as --cd; the audit strips it from the + # session commands (the leaf is otherwise never persisted). + usage["worktree_dir"] = os.path.abspath(wt_dir) + usage["codex_home_path"] = ACTIVE_ISOLATED_HOME + usage["codex_home_isolated"] = bool( + ACTIVE_ISOLATED_HOME is not None + and os.environ.get("CODEX_HOME") == ACTIVE_ISOLATED_HOME + ) # No structured findings here — the raw markdown is the comparison input. return ReviewOutput( review_markdown=review_md, @@ -289,4 +450,4 @@ def review(self, case, config: Config, repeat_idx: int) -> ReviewOutput: ) -__all__ = ["CodexReviewer"] +__all__ = ["CodexReviewer", "ISOLATED_CONFIG_TOML", "isolated_codex_home"] diff --git a/tools/reviewer-eval/adapters/worktree.py b/tools/reviewer-eval/adapters/worktree.py index 0ed88e634..f08619272 100644 --- a/tools/reviewer-eval/adapters/worktree.py +++ b/tools/reviewer-eval/adapters/worktree.py @@ -252,4 +252,82 @@ def cleanup(worktree_dir: str, repo_root: str, worktrees_root: str | None = None shutil.rmtree(real, ignore_errors=True) -__all__ = ["materialize", "cleanup", "Materialized", "MaterializeError"] +def git_provenance(repo_root: str) -> dict: + """The ONE provenance contract (seven keys), sampled per review by + ``CodexReviewer.review`` and at manifest write by ``run_eval``. + + * ``repo_head_sha`` - ``git rev-parse HEAD`` (fail-soft ``""``) + * ``repo_tree_sha`` - ``git rev-parse HEAD^{tree}`` (fail-soft ``""``) + * ``repo_dirty`` - True iff ``git --no-optional-locks status --porcelain + --untracked-files=all`` prints anything (the explicit flag overrides a + ``status.showUntrackedFiles=no`` config that would hide an unregistered + corpus dir); ``None`` if git failed. + * ``repo_branch`` - ``git rev-parse --abbrev-ref HEAD`` (``""`` on failure) + * ``repo_common_dir`` - ``git rev-parse --git-common-dir`` normalised as + ``normpath(join(repo_root, out))`` (git prints a RELATIVE ``.git`` in a + primary checkout and an absolute path in a linked worktree; never + ``abspath``, which would resolve against the process cwd). ``""`` on failure. + * ``repo_worktree_paths`` - sorted ``worktree `` lines of ``git worktree + list --porcelain`` minus ``repo_root`` itself and anything under + ``/tools/reviewer-eval/runs/.worktrees/`` (both sides realpath- + normalised before the comparison); ``None`` (never ``[]``) on failure. + * ``repo_ref_names`` - sorted ``git for-each-ref --format=%(refname:short)`` + over EVERY namespace minus ``HEAD``; ``None`` on failure. + """ + out: dict = { + "repo_head_sha": "", + "repo_tree_sha": "", + "repo_dirty": None, + "repo_branch": "", + "repo_common_dir": "", + "repo_worktree_paths": None, + "repo_ref_names": None, + } + try: + cp = _git(repo_root, ["rev-parse", "HEAD"], check=False) + if cp.returncode == 0: + out["repo_head_sha"] = cp.stdout.strip() + cp = _git(repo_root, ["rev-parse", "HEAD^{tree}"], check=False) + if cp.returncode == 0: + out["repo_tree_sha"] = cp.stdout.strip() + cp = _git( + repo_root, + ["--no-optional-locks", "status", "--porcelain", "--untracked-files=all"], + check=False, + ) + if cp.returncode == 0: + out["repo_dirty"] = bool(cp.stdout.strip()) + cp = _git(repo_root, ["rev-parse", "--abbrev-ref", "HEAD"], check=False) + if cp.returncode == 0: + out["repo_branch"] = cp.stdout.strip() + cp = _git(repo_root, ["rev-parse", "--git-common-dir"], check=False) + if cp.returncode == 0 and cp.stdout.strip(): + out["repo_common_dir"] = os.path.normpath(os.path.join(repo_root, cp.stdout.strip())) + cp = _git(repo_root, ["worktree", "list", "--porcelain"], check=False) + if cp.returncode == 0: + root_real = os.path.realpath(repo_root) + own_wts = os.path.realpath( + os.path.join(repo_root, "tools", "reviewer-eval", "runs", ".worktrees") + ) + paths = [] + for line in cp.stdout.splitlines(): + if not line.startswith("worktree "): + continue + path = line[len("worktree ") :].strip() + real = os.path.realpath(path) + if real == root_real: + continue + if real == own_wts or real.startswith(own_wts + os.sep): + continue + paths.append(path) + out["repo_worktree_paths"] = sorted(set(paths)) + cp = _git(repo_root, ["for-each-ref", "--format=%(refname:short)"], check=False) + if cp.returncode == 0: + names = {n.strip() for n in cp.stdout.splitlines() if n.strip() and n.strip() != "HEAD"} + out["repo_ref_names"] = sorted(names) + except (OSError, subprocess.SubprocessError): + pass + return out + + +__all__ = ["materialize", "cleanup", "git_provenance", "Materialized", "MaterializeError"] diff --git a/tools/reviewer-eval/campaigns/2026-07-gpt56-DECISION_RULE.md b/tools/reviewer-eval/campaigns/2026-07-gpt56-DECISION_RULE.md new file mode 100644 index 000000000..d9038a40c --- /dev/null +++ b/tools/reviewer-eval/campaigns/2026-07-gpt56-DECISION_RULE.md @@ -0,0 +1,87 @@ +# Pre-Registered Decision Rule: gpt-5.5 -> gpt-5.6 Reviewer Evaluation + +Committed BEFORE any campaign run (the PR that adds this file timestamps the +pre-registration). The verdict is computed MECHANICALLY from the graded tables +against these rules; the rules may not be edited after the campaign starts. + +## The experiment + +| Arm | Model | Effort | Repeats | Role | +|-----|-------|--------|---------|------| +| A | gpt-5.5 | xhigh | k=2 | control (current production) | +| B | gpt-5.6-sol | xhigh | k=2 | primary candidate | +| C | gpt-5.6-terra | xhigh | k=1 | cost-fallback probe (informational) | +| D | gpt-5.6-sol | max | k=1 | effort probe (informational) | + +The production decision is A vs B. C and D never gate the swap. + +## Definitions + +- **Reliably caught** (per arm, per ground-truth bug): named the same defect at + the same location/symbol in EVERY repeat of that arm. +- **Unstably caught**: caught in some but not all repeats. +- **Missed**: caught in no repeat. +- **False positive (FP)**: on an `expect_no_blockers` case, any finding whose + severity is ABOVE that case's `allow_severities` (default allows P2/P3, i.e. + P0/P1 are FPs; the documented-deviation case allows only P3, so P0-P2 count). + Known-FP topics listed by the case are never counted. +- **Calibration failure**: flagging a REGISTRY-documented deviation above P3. + Counted as an FP. + +## Primary gates (B vs A) + +1. **Regression (NO-GO):** any `must_catch` bug that A reliably catches and B + misses in both B-repeats. (A unstably-caught vs B missed -> flagged for + judgment in the report, not an automatic NO-GO.) + `must_catch` gates regressions ONLY. +2. **FP gate (NO-GO):** B's total FP count across S3 negative controls exceeds + A's. +3. **GO:** zero regressions AND the FP gate passes AND at least one strict + improvement: + - B reliably catches ANY ground-truth bug (`must_catch` true OR false — + aspirational S4 catches count, and S4 is the highest-value evidence) that A + misses in both A-repeats, OR + - B has strictly fewer FPs than A (with no catch regression). +4. **PARITY:** zero regressions but no strict improvement -> the user decides + (vendor-cadence value vs staying on gpt-5.5). + +## Secondary reads (informational only) + +- **C vs A:** does Terra actually match gpt-5.5 on our corpus (the half-cost + claim)? Relevant only if B fails its gates. +- **D vs B:** does `max` effort add reliable catches over xhigh on the same + model? Informs an optional follow-up; the default ship remains Sol @ xhigh. +- Latency per arm (from the unblinded artifacts) is reported alongside, since a + much slower CI review has real cost even at equal accuracy. + +## Grading protocol (blinded, multi-grader) + +1. `compare --blinded` produces `comparison.blinded.md` (arm identities replaced + by neutral M* labels; model self-references scrubbed; latency redacted) and a + sealed `blinding.json` that graders NEVER read. +2. 2-3 INDEPENDENT graders (separate subagent contexts) each read ONLY the + blinded bundle plus this rubric, and fill, per (case, ground-truth bug, arm + label): caught / partial / missed, with a verbatim evidence quote from the + review; plus per-case FP lists with severities. "Partial" (right file or + class, wrong defect or location) counts as MISSED for the gates. +3. **Hallucination check:** a catch's evidence quote must actually name the + defect; a finding that claims to have reproduced behavior the diff cannot + produce is graded as an FP-class note, never a catch. +4. **Adversarial reconciliation:** any cell where graders disagree goes to a + verifier agent that re-reads that case's blinded reviews and rules with + quoted evidence. +5. Only after the tables are final are labels unblinded via `blinding.json` and + the gates above applied mechanically. + +## Blinding caveats (accepted) + +- Repeat counts partition the labels into {k=2}={A,B} and {k=1}={C,D}; the + decisive A-vs-B contrast stays 50/50 blind within its group. +- Sanitization is best-effort; graders are instructed to grade on content and + never on guessed identity. + +## Corpus floor + +The campaign runs only with >= 8 verified cases including >= 2 S4 +(missed-by-gpt-5.5) cases; below that, stop and surface rather than produce a +weak read. diff --git a/tools/reviewer-eval/campaigns/2026-07-gpt56-VERDICT.md b/tools/reviewer-eval/campaigns/2026-07-gpt56-VERDICT.md index 42bd083e4..02f1b017a 100644 --- a/tools/reviewer-eval/campaigns/2026-07-gpt56-VERDICT.md +++ b/tools/reviewer-eval/campaigns/2026-07-gpt56-VERDICT.md @@ -1,5 +1,10 @@ # GPT-5.6 Reviewer Evaluation - Verdict (2026-07-18) +Pre-registered rule: `2026-07-gpt56-DECISION_RULE.md` (archived from `DECISION_RULE.md` +2026-10; the released CHANGELOG entries for the 2026-07 campaign, `CHANGELOG.md:2513` +and `:2613`, name the old path and are historical text left as is; `:2368` names the +plan-review-eval harness's own `DECISION_RULE.md`, a different file). + Campaign: 66/66 runs ok, 11 cases, arms A=gpt-5.5@xhigh k2 (control), B=gpt-5.6-sol@xhigh k2, C=gpt-5.6-terra@xhigh k1, D=gpt-5.6-sol@max k1. Blinded grading: 3 independent graders on comparison.blinded.md (labels M1-M4; graders never saw blinding.json); diff --git a/tools/reviewer-eval/config/configs.json b/tools/reviewer-eval/config/configs.json index beca37361..e07d97ed1 100644 --- a/tools/reviewer-eval/config/configs.json +++ b/tools/reviewer-eval/config/configs.json @@ -1,6 +1,9 @@ { - "_comment": "Reviewer arms. The 2026-07 gpt-5.5 -> gpt-5.6 evaluation ran with A as control; after the GO verdict (66-run blinded 4-arm campaign, campaigns/2026-07-gpt56-VERDICT.md) gpt-5.6-sol @ xhigh became production, so B now carries role=control and A is retained as the previous-production arm for future back-comparisons. treatment_fields declares which Config fields are the experimental treatment; every other confound (sandbox / action_version / cli_version, plus effort when not declared) is pinned and asserted identical across arms, and every selected multi-arm subset must decompose into single-field contrasts (see eval_core.runner). cli_version pinned to codex-cli 0.144.5, verified live 2026-07-18 to accept gpt-5.5 @ xhigh, gpt-5.6-sol @ xhigh AND @ max, and gpt-5.6-terra @ xhigh (invalid efforts are rejected with a 400 enum error, so accepted == executed) - one CLI serves all arms and the runner asserts the live `codex --version` matches.", - "treatment_fields": ["model", "effort"], + "_comment": "Reviewer arms. The 2026-07 gpt-5.5 -> gpt-5.6 evaluation ran with A as control; after the GO verdict (66-run blinded 4-arm campaign, campaigns/2026-07-gpt56-VERDICT.md) gpt-5.6-sol @ xhigh became production, so B carries role=control and A is retained as the previous-production arm for future back-comparisons. The 2026-10 campaign (campaigns/2026-10-gpt6-VERDICT.md, DECISION_RULE.md) compares B (control) against E (gpt-6-astra @ xhigh, the candidate); E's FIRST corpus run is the campaign itself (pre-registration by construction). treatment_fields declares which Config fields are the experimental treatment; every other confound (sandbox / action_version / cli_version, plus effort when not declared) is pinned and asserted identical across arms, and every selected multi-arm subset must decompose into single-field contrasts (see eval_core.runner). cli_version re-pinned to codex-cli 0.162.1 on 2026-10-16 (one CLI serves all arms and the runner asserts the live `codex --version` matches): B @ xhigh verified by harness smoke on 0.162.1; E = gpt-6-astra @ xhigh accepted by a `codex exec` probe on 0.162.1 (the exact argv `_build_codex_cmd` emits, under an isolated CODEX_HOME); A, C and D are historical 2026-07 arms NOT re-verified on 0.162.1 (D's `max` included; an unsupported effort fails closed as INFRA_ERROR). Astra off-corpus latency probe (2026-10-16, codex-cli 0.162.1, isolated CODEX_HOME, production openai_review.py on merged PR #841, 554 changed lines, effort xhigh): real = 324.57 s wall-clock, 14 shell commands, 26 web-search events; below the 1800 s threshold, so CALL_TIMEOUT_S stays 3600. Contrast check: B -> E differ in `model` only.", + "treatment_fields": [ + "model", + "effort" + ], "arms": [ { "id": "A", @@ -8,7 +11,7 @@ "effort": "xhigh", "sandbox": "read-only", "action_version": "v1", - "cli_version": "codex-cli 0.144.5", + "cli_version": "codex-cli 0.162.1", "label": "previous production (pre-2026-07 swap)" }, { @@ -18,7 +21,7 @@ "effort": "xhigh", "sandbox": "read-only", "action_version": "v1", - "cli_version": "codex-cli 0.144.5", + "cli_version": "codex-cli 0.162.1", "label": "control / current production" }, { @@ -27,7 +30,7 @@ "effort": "xhigh", "sandbox": "read-only", "action_version": "v1", - "cli_version": "codex-cli 0.144.5", + "cli_version": "codex-cli 0.162.1", "label": "cost-fallback probe (Terra claimed ~= gpt-5.5 at half cost)" }, { @@ -36,8 +39,17 @@ "effort": "max", "sandbox": "read-only", "action_version": "v1", - "cli_version": "codex-cli 0.144.5", + "cli_version": "codex-cli 0.162.1", "label": "effort probe (max is new with GPT-5.6; isolates effort vs arm B)" + }, + { + "id": "E", + "model": "gpt-6-astra", + "effort": "xhigh", + "sandbox": "read-only", + "action_version": "v1", + "cli_version": "codex-cli 0.162.1", + "label": "candidate: GPT-6 frontier (2026-10 campaign)" } ] } diff --git a/tools/reviewer-eval/reviewer_audit.py b/tools/reviewer-eval/reviewer_audit.py new file mode 100644 index 000000000..f7ca68d28 --- /dev/null +++ b/tools/reviewer-eval/reviewer_audit.py @@ -0,0 +1,499 @@ +"""Post-unblinding session-log audit for the reviewer-eval harness. + +INFORMATIONAL and OVER-MATCHING BY DESIGN (DECISION_RULE.md, user decisions 4 +and 6): it never flips or contests a gate. For every manifest run artifact it +takes ``usage["session_commands"]`` (the shell command lines codex executed, +parsed by ``openai_review.call_codex`` from codex's stderr), strips the run's +OWN worktree path (which codex received as ``--cd`` and echoes everywhere), +and scans the remainder for the ONE authoritative LEAK MARKER LIST: + +* ``LEAK_MARKERS`` - literal substrings (git verbs/flags that reach other refs + or the shared object DB, corpus/answer-key file names, cwd-derivation and + recursive-search idioms, codex-home paths); +* seven regexes - ``MAIN_REF_RE``, ``BARE_SHOW_RE``, ``GIT_LOG_RE``, + ``REF_LIST_RE``, ``DOTDOT_RE``, ``GIT_DIR_RE``, ``TAG_REF_RE`` - each + tolerating git's global options between ``git`` and the subcommand; +* computed, per-campaign markers (never part of ``LEAK_RULES``): the enclosing + checkout (``repo_root``), the checkout's branch name, the git common dir and + its parent checkout, every other worktree of the repository, every other + recorded ref name (git-scoped), and the isolated codex home. + +``LEAK_RULES`` (literals + the seven regex sources, in order) is what +``GRADING_RUBRIC.md`` renders for graders; the rubric-hygiene test asserts the +rendering equals this list exactly (one source, two renderings, no third copy). + +Coverage per artifact is exactly one of THREE classes (the single definition, +shared with DECISION_RULE.md and every test): ``covered``, ``no_commands``, +``unaudited``. Both ``covered`` and ``no_commands`` require +``codex_home_isolated == True`` and ``session_other_events == 0``; the audit +is VACUOUS when no artifact is covered or no_commands. + +Python 3.9 compatible (runs on the CI floor leg). +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from typing import Any, Optional + +# Common prefix: `git` followed by any run of global options (dash options with +# or without `=value`; the argument-taking globals -C/-c/--git-dir/--work-tree/ +# --namespace/--attr-source/--config-env also skip a split-form argument), then +# whitespace. Generic on purpose: an enumerated list was a false-negative source +# (`git --no-replace-objects show HEAD` matched nothing). +G = ( + r"\bgit([ \t]+(-[Cc][ \t]*\S+|--(git-dir|work-tree|namespace|attr-source|config-env)" + r"(=|[ \t]+)\S+|--?[A-Za-z][\w-]*(=\S*)?))*[ \t]+" +) + +# Any git command whose SAME line names the main/master ref as a token (ranges +# and colon forms included: `HEAD..main`, `main...HEAD`, `main:path`); optional +# heads/, refs/heads/ and remotes// prefixes; `main.py`, `src/main/`, +# `domain.py`, `remain` do not match. +MAIN_REF_RE = re.compile( + G + + r"\S+(?=[^\n]*(?" + + +def artifacts_sha(subdir_path: str, run_ids: "list[str]") -> str: + """sha256 over the sorted (run_id, sha256 of that artifact file's bytes) + pairs, where the file for a run_id is ``/.json`` (the + ``RunStore._path`` layout). A missing file is a refusal (FileNotFoundError), + never a skip, so ``compare``, ``audit`` and ``verdict`` seal the same files. + """ + pairs = [] + for run_id in sorted(set(run_ids)): + path = os.path.join(subdir_path, f"{run_id}.json") + if not os.path.isfile(path): + raise FileNotFoundError(f"artifact missing for run_id {run_id!r}: {path}") + with open(path, "rb") as fh: + pairs.append((run_id, hashlib.sha256(fh.read()).hexdigest())) + return hashlib.sha256(json.dumps(pairs).encode("utf-8")).hexdigest() + + +def _home_forms(path: str) -> "list[str]": + """``~/`` and ``$HOME/`` spellings of ``path`` when it lies under the home dir.""" + home = os.path.expanduser("~") + out = [] + if path and home and (path == home or path.startswith(home + os.sep)): + rel = path[len(home) :] + out.append("~" + rel) + out.append("$HOME" + rel) + return out + + +def strip_own_path(entry: str, worktree_dir: str, repo_root: str) -> str: + """Replace every spelling of the run's OWN worktree path with ````, in + this FIXED order (longer forms first; the relative form is a suffix of the + absolute one and the leaf a suffix of both): realpath, recorded absolute, + repo-relative (only when under ``repo_root``), ``~/`` and ``$HOME/`` forms, + then the bare leaf LAST. Joins backslash-newline continuations first. + """ + text = _CONTINUATION_RE.sub(" ", entry) + if not worktree_dir: + return text + forms: "list[str]" = [] + real = os.path.realpath(worktree_dir) + forms.append(real) + forms.append(worktree_dir) + root_real = os.path.realpath(repo_root) if repo_root else "" + for base in (root_real, repo_root): + if base and real.startswith(base + os.sep): + forms.append(real[len(base) + 1 :]) + if base and worktree_dir.startswith(base + os.sep): + forms.append(worktree_dir[len(base) + 1 :]) + forms += _home_forms(worktree_dir) + _home_forms(real) + leaf = os.path.basename(worktree_dir.rstrip(os.sep)) + for form in forms: + if form: + text = text.replace(form, WT_PLACEHOLDER) + if leaf: + text = text.replace(leaf, WT_PLACEHOLDER) + return text + + +def _path_markers( + repo_root: str, + markers: dict, +) -> "list[tuple[str, str]]": + """(name, spelling) pairs for every computed PATH marker, longest first.""" + out: "list[tuple[str, str]]" = [] + + def add(name: str, path: Optional[str]) -> None: + if not path: + return + spellings = {path, os.path.realpath(path)} + for p in list(spellings): + spellings.update(_home_forms(p)) + for sp in spellings: + out.append((name, sp)) + + add("repo_root", repo_root) + common = markers.get("repo_common_dir") or "" + add("repo_common_dir", common) + if common: + add("repo_main_checkout", os.path.dirname(common.rstrip(os.sep))) + for wt in markers.get("repo_worktree_paths") or []: + add("repo_worktree_path", wt) + add("codex_home_path", markers.get("codex_home_path")) + # Longest spelling first so a read under repo_root is attributed to + # repo_root, not also to the (shorter) primary checkout it sits beside. + out.sort(key=lambda t: len(t[1]), reverse=True) + return out + + +def _ref_name_regex(ref_names: "list[str]", branch: str) -> "Optional[re.Pattern]": + """Git-scoped regex over the recorded ref names minus main/master, the + campaign branch, HEAD, bare remote names and semver tags (TAG_REF_RE's job).""" + names = {n for n in ref_names if n and n != "HEAD"} + remotes = {n.split("/", 1)[0] for n in names if "/" in n} + names -= {"main", "master", branch} + names = {n for n in names if not (n in remotes and "/" not in n)} + names = {n for n in names if not _SEMVER_TAG_RE.match(n)} + if not names: + return None + alts = "|".join(re.escape(n) for n in sorted(names, key=len, reverse=True)) + return re.compile(G + r"\S+(?=[^\n]*(? "list[str]": + """Marker names that hit one STRIPPED entry (deterministic order).""" + hits: "list[str]" = [] + for lit in literals: + if lit in line: + hits.append(lit) + for name, rx in regexes: + if rx.search(line): + hits.append(name) + if path_markers is None: + path_markers = _path_markers(repo_root, markers) + taken: "list[tuple[int, int]]" = [] + for name, spelling in path_markers: + for m in re.finditer(re.escape(spelling) + _PATH_BOUNDARY, line): + span = (m.start(), m.end()) + if any(a < span[1] and span[0] < b for a, b in taken): + continue # a LONGER marker already owns this span + taken.append(span) + if name not in hits: + hits.append(name) + branch = markers.get("repo_branch") or "" + if branch and re.search( + G + r"\S+(?=[^\n]*(? str: + """EXACTLY the three classes of DECISION_RULE.md; no defaults.""" + if not isinstance(usage, dict): + return "unaudited" + if usage.get("codex_home_isolated") is not True: + return "unaudited" + if usage.get("session_other_events") != 0: + return "unaudited" + cmds = usage.get("session_commands") + start = usage.get("session_commands_start_lines") + execs = usage.get("session_commands_exec_lines") + truncated = usage.get("session_commands_truncated") + if not isinstance(cmds, list) or not isinstance(start, int) or not isinstance(execs, int): + return "unaudited" + if truncated is not False: # missing / None / 1 / True: truncation state unknown or set + return "unaudited" + if cmds: + if any(c.endswith("[truncated]") for c in cmds): + return "unaudited" + if execs == start == len(cmds): + return "covered" + return "unaudited" + if start == 0 and execs == 0: + return "no_commands" + return "unaudited" + + +def audit( + runs: list, + subdir_path: str, + repo_root: str, + manifest_markers: dict, + run_ids_sha: str, + artifacts_sha_value: str, +) -> dict: + """Audit every (OK) run artifact; return the ``hits.json`` structure. + + ``manifest_markers`` = the manifest's ``{repo_branch, repo_common_dir, + repo_worktree_paths, repo_ref_names, codex_home_path}``; each artifact's own + ``usage`` values of the same keys are UNIONED in (a worktree that existed at + ANY sample point is a marker). Refuses (ValueError) if the live artifacts no + longer hash to ``artifacts_sha_value``. + """ + run_ids = [getattr(r, "run_id", "") for r in runs] + live = artifacts_sha(subdir_path, run_ids) + if live != artifacts_sha_value: + raise ValueError( + f"artifacts changed since they were sealed: live artifacts_sha {live} != " + f"recorded {artifacts_sha_value}" + ) + branch = manifest_markers.get("repo_branch") or "" + common = manifest_markers.get("repo_common_dir") or "" + wts = set(manifest_markers.get("repo_worktree_paths") or []) + refs = set(manifest_markers.get("repo_ref_names") or []) + homes = set() + if manifest_markers.get("codex_home_path"): + homes.add(manifest_markers["codex_home_path"]) + for r in runs: + u = getattr(r, "usage", None) or {} + wts.update(u.get("repo_worktree_paths") or []) + refs.update(u.get("repo_ref_names") or []) + if u.get("codex_home_path"): + homes.add(u["codex_home_path"]) + if not common and u.get("repo_common_dir"): + common = u["repo_common_dir"] + markers = { + "repo_branch": branch, + "repo_common_dir": common, + "repo_worktree_paths": sorted(wts), + "codex_home_path": None, + } + path_markers: "list[tuple[str, str]]" = [] + for home in sorted(homes): + path_markers += _path_markers("", {"codex_home_path": home}) + path_markers += _path_markers(repo_root, markers) + path_markers.sort(key=lambda t: len(t[1]), reverse=True) + ref_regex = _ref_name_regex(sorted(refs), branch) + + hits: "list[dict]" = [] + unaudited: "list[dict]" = [] + no_commands: "list[dict]" = [] + counts = {"covered": 0, "no_commands": 0, "unaudited": 0, "total": 0} + for r in sorted(runs, key=lambda x: (x.case_id, x.config_id, x.repeat_idx)): + cell = {"case": r.case_id, "arm": r.config_id, "repeat": r.repeat_idx} + u = getattr(r, "usage", None) or {} + cls = coverage_class(u) + counts[cls] += 1 + counts["total"] += 1 + if cls == "unaudited": + unaudited.append(cell) + elif cls == "no_commands": + no_commands.append(cell) + for entry in u.get("session_commands") or []: + if not isinstance(entry, str): + continue + stripped = strip_own_path(entry, u.get("worktree_dir") or "", repo_root) + found = scan_entry(stripped, repo_root, markers, path_markers, ref_regex) + if found: + hits.append( + { + **cell, + "markers": found, + "line": stripped, + "orientation_only": set(found) <= ORIENTATION_MARKERS, + } + ) + vacuous = counts["covered"] == 0 and counts["no_commands"] == 0 + return { + "run_ids_sha": run_ids_sha, + "artifacts_sha": artifacts_sha_value, + "branch": branch, + "hits": hits, + "unaudited": unaudited, + "no_commands": no_commands, + "coverage": counts, + "vacuous": vacuous, + } + + +def render_rules_for_rubric() -> str: + """The rubric's LEAK? trigger list: literals verbatim, then each regex as its + source plus a one-line gloss. ``GRADING_RUBRIC.md`` embeds exactly this text + between its `` / `` markers.""" + lines = ["Literal markers (any code span or fenced block containing one):", ""] + for m in LEAK_MARKERS: + lines.append(f"- `{m}`") + lines += ["", "Regex markers (source, then the plain-language rule to apply):", ""] + for name, rx in REGEX_RULES: + lines.append(f"- `{name}` = `{rx.pattern}` - {REGEX_GLOSS[name]}") + return "\n".join(lines) + "\n" + + +__all__ = [ + "G", + "MAIN_REF_RE", + "BARE_SHOW_RE", + "GIT_LOG_RE", + "REF_LIST_RE", + "DOTDOT_RE", + "GIT_DIR_RE", + "TAG_REF_RE", + "REGEX_RULES", + "LEAK_MARKERS", + "LEAK_RULES", + "ORIENTATION_MARKERS", + "WT_PLACEHOLDER", + "artifacts_sha", + "strip_own_path", + "scan_entry", + "coverage_class", + "audit", + "render_rules_for_rubric", +] diff --git a/tools/reviewer-eval/reviewer_verdict.py b/tools/reviewer-eval/reviewer_verdict.py new file mode 100644 index 000000000..698cb9aee --- /dev/null +++ b/tools/reviewer-eval/reviewer_verdict.py @@ -0,0 +1,626 @@ +"""Mechanical verdict for the reviewer-eval campaign (DECISION_RULE.md gates). + +Pure functions over plain dicts (the sibling ``tools/plan-review-eval/verdict.py`` +is the model; this module has a UNIQUE basename because both harness roots sit on +``sys.path`` in tests). Differences from the sibling, per the 2026-10 rule: + +* severity vocabulary ``P0|P1|P2|P3`` compared against each S3 case's + ``allow_severities`` (no known-topic exemption: on this corpus the only + known-FP topic is already inside the allowed severities); +* a per-row / per-cell ``leak`` field (``none|confirmed`` in the RECONCILED + table; ``flag`` is a validation error there) plus ``grader_flag`` + (``none|flag``) and ``leak_evidence`` (non-empty iff flagged or confirmed); + any ``confirmed`` row or S3 cell VOIDS that whole (case, arm, repeat) for + every gate; +* per-bug statuses RELIABLE / UNSTABLE / MISSED / UNGRADEABLE computed from + the remaining (non-void, OK) repeats; a void or ungradeable cell CONTESTS + every gate its case feeds (reported, never decided mechanically); +* the regression gate runs over ALL ground-truth bugs (``must_catch`` is a + label only); reliable-to-unstable downgrades count, and >= 2 of them force + PARITY; precedence (0)-(7); +* an optional ``audit_hits`` (the session-log audit's ``hits.json``) attaches + per-gate cell lists with coverage class and hits - informational only, + never a contest; without it the outcome is PROVISIONAL. + +Python 3.9 compatible. +""" + +from __future__ import annotations + +from typing import Any, Optional + +RELIABLE = "reliable" +UNSTABLE = "unstable" +MISSED = "missed" +UNGRADEABLE = "ungradeable" + +GO = "GO" +NO_GO = "NO-GO" +PARITY = "PARITY" +UNDETERMINED = "UNDETERMINED" + +# The ONE pre-registered gating contrast (control, candidate). A PR-2 swap does +# not change it; a future campaign registers its own pair in its own PR-1. +REGISTERED_CONTRASTS = (("B", "E"),) + +_VERDICT_VOCAB = ("caught", "partial", "missed") +_SEVERITY_VOCAB = ("P0", "P1", "P2", "P3") +_LEAK_FINAL_VOCAB = ("none", "confirmed") +_FLAG_VOCAB = ("none", "flag") +GATE_NAMES = ("regression", "fp", "improvement", "downgrade") + + +def unblind(grades: dict, blinding: "dict[str, str]") -> dict: + """Replace blind labels with real arm ids throughout a grading table.""" + label_to_arm = {v: k for k, v in blinding.items()} + unknown = { + r.get("arm") + for r in list(grades.get("rows", [])) + list(grades.get("negative_assessments", [])) + if r.get("arm") not in label_to_arm + } + if unknown: + raise ValueError( + f"grading table uses arm label(s) {sorted(map(str, unknown))} not present in the " + f"blinding mapping {sorted(blinding.values())} - wrong blinding.json?" + ) + out = {k: v for k, v in grades.items() if k not in ("rows", "negative_assessments")} + out["rows"] = [{**r, "arm": label_to_arm[r["arm"]]} for r in grades.get("rows", [])] + out["negative_assessments"] = [ + {**r, "arm": label_to_arm[r["arm"]]} for r in grades.get("negative_assessments", []) + ] + return out + + +def _runs_index(runs: "list[dict]") -> "tuple[set, dict, dict, dict]": + arms = {rr["config_id"] for rr in runs} + bugs: "dict[str, set[str]]" = {} + strata: "dict[str, str]" = {} + snaps: "dict[str, dict]" = {} + for rr in runs: + snap = rr.get("case_snapshot") or {} + strata[rr["case_id"]] = snap.get("stratum", "") + snaps.setdefault(rr["case_id"], snap) + for bug in snap.get("ground_truth") or []: + bugs.setdefault(rr["case_id"], set()).add(bug.get("id", "?")) + bugs.setdefault(rr["case_id"], set()) + return arms, bugs, strata, snaps + + +def _check_leak_fields(cell: dict, where: str, problems: "list[str]") -> None: + leak = cell.get("leak") + flag = cell.get("grader_flag") + evidence = str(cell.get("leak_evidence", "") or "").strip() + if leak not in _LEAK_FINAL_VOCAB: + problems.append( + f"{where}: leak {leak!r} not in {_LEAK_FINAL_VOCAB} (a 'flag' left in the " + f"reconciled table means the verifier never ruled on this cell)" + ) + if flag not in _FLAG_VOCAB: + problems.append(f"{where}: grader_flag {flag!r} not in {_FLAG_VOCAB}") + needs_evidence = leak == "confirmed" or flag == "flag" + if needs_evidence and not evidence: + problems.append(f"{where}: leak_evidence must be non-empty when flagged or confirmed") + if not needs_evidence and evidence and leak in _LEAK_FINAL_VOCAB and flag in _FLAG_VOCAB: + problems.append(f"{where}: leak_evidence present on a none/none cell") + + +def ok_repeats(runs: "list[dict]") -> "dict[tuple[str, str], list[int]]": + """(case_id, config_id) -> sorted OK repeat indices (INFRA_ERROR excluded).""" + out: "dict[tuple[str, str], list[int]]" = {} + for rr in runs: + if rr.get("infra_error"): + continue + out.setdefault((rr["case_id"], rr["config_id"]), []).append(int(rr["repeat_idx"])) + return {k: sorted(v) for k, v in out.items()} + + +def validate_grades(grades: dict, runs: "list[dict]") -> None: + """Reject a malformed RECONCILED grading table before it can shape a verdict. + + Beyond the sibling's checks (vocabulary, known cells, scheduled repeats, no + duplicate rows/cells, caught-with-evidence, complete bug grid and complete + negative grid, duplicate findings in one S3 cell), every row and S3 cell + must carry ``leak`` in {none, confirmed}, ``grader_flag`` in {none, flag} + and a ``leak_evidence`` that is non-empty exactly when flagged or confirmed. + """ + problems: "list[str]" = [] + arms, bugs, strata, snaps = _runs_index(runs) + reps_all: "dict[tuple[str, str], set[int]]" = {} + for rr in runs: + reps_all.setdefault((rr["case_id"], rr["config_id"]), set()).add(int(rr["repeat_idx"])) + ok_reps = ok_repeats(runs) + + seen_rows: "set[tuple]" = set() + for i, row in enumerate(grades.get("rows", [])): + where = f"rows[{i}]" + case, bug, arm = row.get("case"), row.get("bug_id"), row.get("arm") + if row.get("verdict") not in _VERDICT_VOCAB: + problems.append(f"{where}: verdict {row.get('verdict')!r} not in {_VERDICT_VOCAB}") + if case not in bugs: + problems.append(f"{where}: unknown case {case!r}") + elif bug not in bugs[case]: + problems.append(f"{where}: unknown bug {bug!r} for case {case!r}") + if arm not in arms: + problems.append(f"{where}: unknown arm {arm!r} (have {sorted(arms)})") + rep = row.get("repeat", 0) + if not isinstance(rep, int) or rep not in ok_reps.get((case, arm), []): + problems.append(f"{where}: repeat {rep!r} is not an OK repeat of ({case}, {arm})") + key = (case, bug, arm, rep) + if key in seen_rows: + problems.append(f"{where}: duplicate grading row for {key}") + seen_rows.add(key) + if row.get("verdict") == "caught" and not str(row.get("evidence", "")).strip(): + problems.append(f"{where}: 'caught' without evidence quote") + _check_leak_fields(row, where, problems) + + seen_cells: "set[tuple]" = set() + for i, cell in enumerate(grades.get("negative_assessments", [])): + where = f"negative_assessments[{i}]" + case, arm = cell.get("case"), cell.get("arm") + if strata.get(case) != "s3_negative": + problems.append(f"{where}: case {case!r} is not an s3_negative case") + if arm not in arms: + problems.append(f"{where}: unknown arm {arm!r}") + rep = cell.get("repeat", 0) + if not isinstance(rep, int) or rep not in ok_reps.get((case, arm), []): + problems.append(f"{where}: repeat {rep!r} is not an OK repeat of ({case}, {arm})") + if not isinstance(cell.get("findings"), list): + problems.append(f"{where}: findings must be a list (may be empty for a clean read)") + else: + seen_summaries: "set[str]" = set() + for j, f in enumerate(cell["findings"]): + if f.get("severity") not in _SEVERITY_VOCAB: + problems.append( + f"{where}.findings[{j}]: severity {f.get('severity')!r} " + f"not in {_SEVERITY_VOCAB}" + ) + summary = " ".join(str(f.get("summary", "")).split()).lower() + if not summary: + problems.append(f"{where}.findings[{j}]: finding has no summary") + if summary in seen_summaries: + problems.append( + f"{where}.findings[{j}]: duplicate finding {summary!r} in one cell " + f"(same defect, possibly different severity) - reconcile to ONE entry" + ) + seen_summaries.add(summary) + key = (case, arm, rep) + if key in seen_cells: + problems.append(f"{where}: duplicate assessment cell for {key}") + seen_cells.add(key) + _check_leak_fields(cell, where, problems) + + missing_neg = [ + (case, arm, rep) + for case, stratum in strata.items() + if stratum == "s3_negative" + for arm in sorted(arms) + for rep in ok_reps.get((case, arm), []) + if (case, arm, rep) not in seen_cells + ] + for cell in missing_neg[:20]: + problems.append(f"missing negative-control assessment for {cell}") + if len(missing_neg) > 20: + problems.append(f"... and {len(missing_neg) - 20} more missing assessment cells") + + graded_cells = { + (r.get("case"), r.get("bug_id"), r.get("arm"), r.get("repeat")) + for r in grades.get("rows", []) + } + missing_cells = [ + (case, bug, arm, rep) + for case, bug_ids in bugs.items() + for bug in sorted(bug_ids) + for arm in sorted(arms) + for rep in ok_reps.get((case, arm), []) + if (case, bug, arm, rep) not in graded_cells + ] + for cell in missing_cells[:20]: + problems.append(f"missing grading row for {cell} (the grid must be complete)") + if len(missing_cells) > 20: + problems.append(f"... and {len(missing_cells) - 20} more missing cells") + + if problems: + raise ValueError( + "grading table failed validation (fix the table, do not score it):\n - " + + "\n - ".join(problems) + ) + + +def void_cells(grades: dict) -> "set[tuple[str, str, int]]": + """(case, arm, repeat) cells with ANY ``leak == "confirmed"`` row or S3 cell.""" + out: "set[tuple[str, str, int]]" = set() + for r in list(grades.get("rows", [])) + list(grades.get("negative_assessments", [])): + if r.get("leak") == "confirmed": + out.add((r["case"], r["arm"], int(r.get("repeat", 0)))) + return out + + +def catch_status( + grades: dict, runs: "list[dict]", voids: "Optional[set]" = None +) -> "dict[tuple[str, str, str], str]": + """(case, bug_id, arm) -> RELIABLE | UNSTABLE | MISSED | UNGRADEABLE. + + Computed from the REMAINING (OK, non-void) repeats of that (case, arm): + RELIABLE = caught in every remaining repeat AND no void repeat; UNSTABLE = + caught in some remaining repeat, or in all remaining but >= 1 void; MISSED = + caught in no remaining repeat (>= 1 remaining); UNGRADEABLE = no remaining + repeat. ``partial`` is missed. A missing row for a repeat counts as missed. + """ + voids = voids if voids is not None else void_cells(grades) + reps = ok_repeats(runs) + _arms, bugs, _strata, _snaps = _runs_index(runs) + caught: "dict[tuple[str, str, str], set[int]]" = {} + for row in grades.get("rows", []): + if row.get("verdict") == "caught": + caught.setdefault((row["case"], row["bug_id"], row["arm"]), set()).add( + int(row.get("repeat", 0)) + ) + arms = {rr["config_id"] for rr in runs} + out: "dict[tuple[str, str, str], str]" = {} + for case_id, bug_ids in bugs.items(): + for bug_id in bug_ids: + for arm in arms: + ok = reps.get((case_id, arm), []) + if not ok: + continue + void_here = {r for r in ok if (case_id, arm, r) in voids} + remaining = [r for r in ok if r not in void_here] + got = caught.get((case_id, bug_id, arm), set()) + if not remaining: + out[(case_id, bug_id, arm)] = UNGRADEABLE + continue + n_caught = len(got & set(remaining)) + if n_caught == len(remaining) and not void_here: + out[(case_id, bug_id, arm)] = RELIABLE + elif n_caught > 0: + out[(case_id, bug_id, arm)] = UNSTABLE + else: + out[(case_id, bug_id, arm)] = MISSED + return out + + +def fp_counts( + grades: dict, runs: "list[dict]", voids: "Optional[set]" = None +) -> "tuple[dict[str, int], dict[str, list[dict]]]": + """Arm -> S3 false-positive total over NON-VOID repeats, plus the FP texts. + + An FP is a finding whose severity is NOT inside the case's ``allow_severities``. + No topic exemption (the engine counts every listed finding above the line). + """ + voids = voids if voids is not None else void_cells(grades) + _arms, _bugs, _strata, snaps = _runs_index(runs) + counts: "dict[str, int]" = {} + texts: "dict[str, list[dict]]" = {} + for cell in grades.get("negative_assessments", []): + snap = snaps.get(cell["case"], {}) + if snap.get("stratum") != "s3_negative": + continue + if (cell["case"], cell["arm"], int(cell.get("repeat", 0))) in voids: + continue + allowed = set(snap.get("allow_severities") or []) + for f in cell.get("findings") or []: + if f.get("severity") in allowed: + continue + counts[cell["arm"]] = counts.get(cell["arm"], 0) + 1 + texts.setdefault(cell["arm"], []).append( + { + "case": cell["case"], + "repeat": int(cell.get("repeat", 0)), + "severity": f.get("severity"), + "summary": f.get("summary", ""), + "known_topic": f.get("known_topic", ""), + } + ) + return counts, texts + + +def informational_counts(grades: dict, runs: "list[dict]") -> "dict[str, dict[str, int]]": + """Per arm: distinct leaked cells, distinct unconfirmed-flag cells, and + distinct cells with hallucination ``notes`` (counted once per cell).""" + arms = {rr["config_id"] for rr in runs} + out = {a: {"leaks": 0, "unconfirmed_flags": 0, "hallucinations": 0} for a in sorted(arms)} + confirmed: "dict[str, set]" = {a: set() for a in arms} + flagged: "dict[str, set]" = {a: set() for a in arms} + noted: "dict[str, set]" = {a: set() for a in arms} + for r in list(grades.get("rows", [])) + list(grades.get("negative_assessments", [])): + arm = r.get("arm") + if arm not in arms: + continue + cell = (r["case"], int(r.get("repeat", 0))) + if r.get("leak") == "confirmed": + confirmed[arm].add(cell) + if r.get("grader_flag") == "flag": + flagged[arm].add(cell) + if str(r.get("notes", "") or "").strip(): + noted[arm].add(cell) + for arm in arms: + out[arm]["leaks"] = len(confirmed[arm]) + out[arm]["unconfirmed_flags"] = len(flagged[arm] - confirmed[arm]) + out[arm]["hallucinations"] = len(noted[arm]) + return out + + +def _cells_of(runs: "list[dict]", case_id: str, arms: "tuple[str, ...]") -> "list[tuple]": + return sorted( + { + (rr["case_id"], rr["config_id"], int(rr["repeat_idx"])) + for rr in runs + if rr["case_id"] == case_id and rr["config_id"] in arms and not rr.get("infra_error") + } + ) + + +def _percentile(values: "list[float]", q: float) -> float: + """Nearest-rank percentile (q in [0, 1]) over a non-empty list.""" + xs = sorted(values) + idx = max(0, min(len(xs) - 1, int(round(q * (len(xs) - 1))))) + return float(xs[idx]) + + +def campaign_reads( + runs: "list[dict]", control: str, candidate: str, input_prices: "Optional[dict]" = None +) -> "dict[str, Any]": + """The pre-registered INFORMATIONAL reads computed from the sealed artifacts: + per-arm latency (n/mean/p95/max over OK runs), the ``E p95 > 2x B p95`` + flag, and the rough CI input cost per arm = sum(prompt_tokens_est) x the + official input price per 1M tokens (``input_prices`` maps model name -> + $/1M input; a missing price yields None). Vendor lifecycle is prose the + verdict file carries (no machine source), named here as ``operator_supplied``. + """ + out: "dict[str, Any]" = { + "latency": {}, + "cost_estimate_usd": {}, + "vendor_lifecycle": "operator_supplied", + } + prices = input_prices or {} + for arm in (control, candidate): + lat = [ + float(rr.get("latency_s") or 0.0) + for rr in runs + if rr["config_id"] == arm and not rr.get("infra_error") + ] + out["latency"][arm] = ( + { + "n": len(lat), + "mean": sum(lat) / len(lat), + "p95": _percentile(lat, 0.95), + "max": max(lat), + } + if lat + else {"n": 0, "mean": None, "p95": None, "max": None} + ) + toks = 0 + model = "" + for rr in runs: + if rr["config_id"] == arm and not rr.get("infra_error"): + toks += int((rr.get("usage") or {}).get("prompt_tokens_est") or 0) + model = rr.get("model") or model + price = prices.get(model) + out["cost_estimate_usd"][arm] = { + "model": model, + "prompt_tokens_est": toks, + "input_price_per_1m": price, + "usd": (toks * price / 1_000_000.0) if price is not None else None, + "note": "+/-50% initial-prompt estimate; output/reasoning/tool tokens not recorded", + } + b, e = out["latency"][control]["p95"], out["latency"][candidate]["p95"] + out["latency_flag_candidate_p95_over_2x_control"] = bool( + b is not None and e is not None and e > 2 * b + ) + return out + + +def attach_audit(result: dict, runs: "list[dict]", audit_hits: "Optional[dict]") -> dict: + """Attach the hit list per gate (informational): for each gate, every cell + feeding it with its coverage class and hits. Sets ``audit_vacuous`` and + ``definitive``. Without ``audit_hits`` the result stays PROVISIONAL.""" + if audit_hits is None: + result["gate_cells"] = None + result["audit_vacuous"] = None + result["definitive"] = False + result["label"] = f"{result['verdict']} (PROVISIONAL, no session-log audit)" + return result + control, candidate = result["control"], result["candidate"] + arms = (control, candidate) + unaudited = {(c["case"], c["arm"], int(c["repeat"])) for c in audit_hits.get("unaudited", [])} + no_cmds = {(c["case"], c["arm"], int(c["repeat"])) for c in audit_hits.get("no_commands", [])} + by_cell: "dict[tuple, list[dict]]" = {} + for h in audit_hits.get("hits", []): + by_cell.setdefault((h["case"], h["arm"], int(h["repeat"])), []).append( + { + "markers": h.get("markers", []), + "line": h.get("line", ""), + "orientation_only": bool(h.get("orientation_only", False)), + } + ) + _a, bugs, strata, _s = _runs_index(runs) + bug_cases = sorted(c for c, b in bugs.items() if b) + s3_cases = sorted(c for c, st in strata.items() if st == "s3_negative") + + def describe(cell: tuple) -> dict: + cov = ( + "unaudited" if cell in unaudited else ("no_commands" if cell in no_cmds else "covered") + ) + hits = by_cell.get(cell, []) + return { + "case": cell[0], + "arm": cell[1], + "repeat": cell[2], + "coverage": cov, + "hits": hits, + "orientation_only": bool(hits) and all(h["orientation_only"] for h in hits), + } + + bug_cells = [describe(c) for case in bug_cases for c in _cells_of(runs, case, arms)] + s3_cells = [describe(c) for case in s3_cases for c in _cells_of(runs, case, arms)] + result["gate_cells"] = { + "regression": bug_cells, + "fp": s3_cells, + "improvement": bug_cells + s3_cells, + "downgrade": bug_cells, + } + all_cells = {(c["case"], c["arm"], c["repeat"]): c for c in bug_cells + s3_cells} + with_hits = [c for c in all_cells.values() if c["hits"]] + result["audit_summary"] = { + "cells": len(all_cells), + "cells_with_hits": len(with_hits), + "orientation_only_cells": sum(1 for c in with_hits if c["orientation_only"]), + "unaudited_cells": sum(1 for c in all_cells.values() if c["coverage"] == "unaudited"), + "coverage": audit_hits.get("coverage", {}), + } + result["audit_vacuous"] = bool(audit_hits.get("vacuous", False)) + result["definitive"] = True + result["label"] = f"{result['verdict']} (DEFINITIVE)" + return result + + +def gates( + grades: dict, + runs: "list[dict]", + control: str, + candidate: str, + audit_hits: "Optional[dict]" = None, +) -> "dict[str, Any]": + """Apply the pre-registered gates in precedence order (0)-(7). + + (0) validate; (1) UNDETERMINED on any compared (case, arm) with fewer OK + repeats than scheduled (INFRA exposure; void cells do not count here); + (2) void/ungradeable marking from ``leak: confirmed``; (3) NO-GO on any + regression (control RELIABLE, candidate MISSED, ANY bug); (4) NO-GO on FP + excess; (5) PARITY if >= 2 reliable-to-unstable downgrades; (6) GO if >= 1 + strict improvement (candidate RELIABLE where control MISSED, or strictly + fewer FPs); (7) PARITY. ``contested`` lists ONLY void/ungradeable cells by + gate; the audit hit list is informational (``attach_audit``). + """ + arms_present = {rr["config_id"] for rr in runs} + for arm, role in ((control, "control"), (candidate, "candidate")): + if arm not in arms_present: + raise ValueError( + f"{role} arm {arm!r} has no runs in this manifest (have {sorted(arms_present)})" + ) + validate_grades(grades, runs) # (0) + + reps = ok_repeats(runs) + scheduled: "dict[tuple, set]" = {} + for rr in runs: + scheduled.setdefault((rr["case_id"], rr["config_id"]), set()).add(int(rr["repeat_idx"])) + cases = sorted({rr["case_id"] for rr in runs}) + evidence_gaps = [ + { + "case": c, + "arm": arm, + "ok": len(reps.get((c, arm), [])), + "scheduled": len(scheduled.get((c, arm), set())), + } + for c in cases + for arm in (control, candidate) + if len(reps.get((c, arm), [])) < len(scheduled.get((c, arm), set())) + ] + + voids = void_cells(grades) # (2) + status = catch_status(grades, runs, voids) + fps, fp_texts = fp_counts(grades, runs, voids) + _a, bugs, strata, _s = _runs_index(runs) + + def _s(case: str, bug: str, arm: str) -> str: + return status.get((case, bug, arm), MISSED) + + pairs = sorted({(c, b) for (c, b, _arm) in status}) + regressions = [ + {"case": c, "bug_id": b} + for (c, b) in pairs + if _s(c, b, control) == RELIABLE and _s(c, b, candidate) == MISSED + ] + downgrades = [ + {"case": c, "bug_id": b} + for (c, b) in pairs + if _s(c, b, control) == RELIABLE and _s(c, b, candidate) == UNSTABLE + ] + improvements = [ + {"case": c, "bug_id": b} + for (c, b) in pairs + if _s(c, b, candidate) == RELIABLE and _s(c, b, control) == MISSED + ] + judgment_flags = [ + {"bug": f"{c}|{b}", "kind": "b_unstable_e_missed"} + for (c, b) in pairs + if _s(c, b, control) == UNSTABLE and _s(c, b, candidate) == MISSED + ] + fp_control = fps.get(control, 0) + fp_candidate = fps.get(candidate, 0) + + # Contests: ONLY verifier-confirmed voids / ungradeable cells. + contested: "list[dict]" = [] + void_by_case: "dict[str, list]" = {} + for case, arm, rep in sorted(voids): + if arm in (control, candidate): + void_by_case.setdefault(case, []).append({"case": case, "arm": arm, "repeat": rep}) + for case, cells in sorted(void_by_case.items()): + ungradeable = [ + f"{c}|{b}|{a}" for (c, b, a), st in status.items() if c == case and st == UNGRADEABLE + ] + reason = "verifier-confirmed leak voids this cell" + ( + f"; ungradeable: {sorted(ungradeable)}" if ungradeable else "" + ) + if strata.get(case) == "s3_negative": + contested.append({"gate": "fp", "case": case, "reason": reason, "cells": cells}) + else: + for gate in ("regression", "improvement", "downgrade"): + contested.append({"gate": gate, "case": case, "reason": reason, "cells": cells}) + + if evidence_gaps: + verdict = UNDETERMINED # (1) + elif regressions: + verdict = NO_GO # (3) + elif fp_candidate > fp_control: + verdict = NO_GO # (4) + elif len(downgrades) >= 2: + verdict = PARITY # (5) + elif improvements or fp_candidate < fp_control: + verdict = GO # (6) + else: + verdict = PARITY # (7) + + result: "dict[str, Any]" = { + "control": control, + "candidate": candidate, + "verdict": verdict, + "contested_gates": sorted({c["gate"] for c in contested}), + "contested": contested, + "voided_cells": [{"case": c, "arm": a, "repeat": r} for (c, a, r) in sorted(voids)], + "evidence_gaps": evidence_gaps, + "regressions": regressions, + "downgrades": downgrades, + "improvements": improvements, + "judgment_flags": judgment_flags, + "fp_control": fp_control, + "fp_candidate": fp_candidate, + "fp_findings": {control: fp_texts.get(control, []), candidate: fp_texts.get(candidate, [])}, + "informational": informational_counts(grades, runs), + "catch_status": {f"{c}|{b}|{a}": s for (c, b, a), s in sorted(status.items())}, + } + return attach_audit(result, runs, audit_hits) + + +__all__ = [ + "campaign_reads", + "unblind", + "validate_grades", + "ok_repeats", + "void_cells", + "catch_status", + "fp_counts", + "informational_counts", + "attach_audit", + "gates", + "REGISTERED_CONTRASTS", + "GATE_NAMES", + "RELIABLE", + "UNSTABLE", + "MISSED", + "UNGRADEABLE", + "GO", + "NO_GO", + "PARITY", + "UNDETERMINED", +] diff --git a/tools/reviewer-eval/run_eval.py b/tools/reviewer-eval/run_eval.py index 60448d1fd..bcd7067dc 100644 --- a/tools/reviewer-eval/run_eval.py +++ b/tools/reviewer-eval/run_eval.py @@ -13,20 +13,32 @@ compare Emit one side-by-side bundle (ground truth + every arm's raw reviews) for an LLM (or you) to read into a caught/missed/ false-positive table. --blinded adds an identity-stripped - bundle for blind grading. + bundle for blind grading, sealed to the run artifacts + (bundle_id, run_ids_sha, artifacts_sha, rubric/prompt sha). + audit Post-unblinding session-log audit (reviewer_audit.py): + marker hits per (case, arm, repeat) with coverage classes; + informational, never a gate. + verdict Apply the pre-registered gates (reviewer_verdict.py) to a + reconciled grades JSON, after binding it to the sealed + artifacts; PROVISIONAL without --audit-hits, DEFINITIVE with. Usage: python tools/reviewer-eval/run_eval.py verify-corpus python tools/reviewer-eval/run_eval.py smoke # smokes the role=control arm - python tools/reviewer-eval/run_eval.py run --configs A,B,C,D --k 2 --k-per C=1,D=1 - python tools/reviewer-eval/run_eval.py compare --subdir full --blinded + python tools/reviewer-eval/run_eval.py run --subdir gpt6 --configs B,E --k 2 + python tools/reviewer-eval/run_eval.py compare --subdir gpt6 --blinded + python tools/reviewer-eval/run_eval.py audit --subdir gpt6 -o /hits.json + python tools/reviewer-eval/run_eval.py verdict --subdir gpt6 --grades \\ + --control B --candidate E [--audit-hits /hits.json] [--expect-head ] """ from __future__ import annotations import argparse +import contextlib import os import sys +from typing import Optional HERE = os.path.dirname(os.path.abspath(__file__)) TOOLS = os.path.dirname(HERE) @@ -37,6 +49,10 @@ if TOOLS not in sys.path: sys.path.insert(0, TOOLS) +import reviewer_audit # noqa: E402 +from adapters import codex_reviewer as _codex_reviewer_mod # noqa: E402 +from adapters import worktree # noqa: E402 +from adapters.codex_reviewer import isolated_codex_home # noqa: E402 (module-level: tests stub it) from adapters.corpus_loader import CorpusLoader # noqa: E402 from adapters.openai_review_loader import find_repo_root # noqa: E402 from eval_core.store import RunStore, read_json, write_json # noqa: E402 @@ -72,6 +88,661 @@ def _safe_subdir(subdir: str) -> bool: return full.startswith(base + os.sep) +# Ignored paths a registered surface legitimately carries: harness output dirs +# (the three ignore rules under tools/), bytecode FILES (never a whole +# __pycache__ dir - CorpusLoader treats any stratum subdir holding a case.json as +# a case) and Finder metadata. +_IGNORED_OK_PREFIXES = ( + "tools/reviewer-eval/runs/", + "tools/plan-review-eval/runs/", + "tools/plan-review-eval/corpus/cases/", +) +_IGNORED_OK_RE = __import__("re").compile(r"(__pycache__/[^/]*\.py[cod]$|(^|/)\.DS_Store$)") +_REGISTERED_SURFACES = ("tools", ".claude/scripts", ".github/codex") + + +def _git_out(repo_root: str, args: list) -> str: + import subprocess + + cp = subprocess.run(["git", *args], cwd=repo_root, capture_output=True, text=True, check=False) + if cp.returncode != 0: + raise SystemExit(f"git {' '.join(args)} failed in {repo_root}: {cp.stderr.strip()}") + return cp.stdout + + +def _hidden_inputs(repo_root: str) -> list: + out = _git_out( + repo_root, + ["ls-files", "--others", "--ignored", "--exclude-standard", *_REGISTERED_SURFACES], + ) + bad = [] + for line in out.splitlines(): + path = line.strip() + if not path or path.startswith(_IGNORED_OK_PREFIXES) or _IGNORED_OK_RE.search(path): + continue + bad.append(path) + return sorted(bad) + + +def _assert_no_hidden_inputs(repo_root: str) -> None: + """Refuse when an IGNORED (or .git/info/exclude-hidden) file sits under a + registered surface: the corpus loader enumerates cases by listdir, so an + ignored extra case dir would run; an ignored tools/json.py would shadow the + stdlib. Remedy is in the message; nothing is removed automatically.""" + bad = _hidden_inputs(repo_root) + if bad: + raise SystemExit( + "IGNORED FILE(S) UNDER A REGISTERED SURFACE (tree identity cannot pin them):\n " + + "\n ".join(bad) + + "\nStop, show these paths to the user, remove or move them (never " + "automatically), then re-run this stage." + ) + + +def _assert_tree_identity(repo_root: str, manifest: dict, stage: str = "") -> None: + """Engine identity = tree identity: the working tree must be clean (tracked + and untracked), carry no hidden inputs, and sit at the manifest's recorded + HEAD and tree sha (a same-tree amend or force-push is refused too). Called + at stage ENTRY and again at stage EXIT (a fresh read).""" + where = f"{stage}: " if stage else "" + status = _git_out(repo_root, ["status", "--porcelain", "--untracked-files=all"]).strip() + if status: + raise SystemExit( + f"{where}working tree is not clean; compare/audit/verdict run only on the exact " + f"committed tree the campaign recorded. `git status --porcelain`:\n{status}" + ) + _assert_no_hidden_inputs(repo_root) + live_tree = _git_out(repo_root, ["rev-parse", "HEAD^{tree}"]).strip() + live_head = _git_out(repo_root, ["rev-parse", "HEAD"]).strip() + if live_tree != manifest.get("repo_tree_sha"): + raise SystemExit( + f"{where}live tree {live_tree} != manifest repo_tree_sha " + f"{manifest.get('repo_tree_sha')}; restore the recorded tree (or re-run the campaign)." + ) + if live_head != manifest.get("repo_head_sha"): + raise SystemExit( + f"{where}live HEAD {live_head} != manifest repo_head_sha " + f"{manifest.get('repo_head_sha')} (same tree, different commit is refused too)." + ) + + +def _manifest_provenance(repo_root: str) -> dict: + """The seven-key provenance contract at manifest-write time (one call).""" + return worktree.git_provenance(repo_root) + + +def _provenance_aggregates(results, manifest_prov: dict) -> dict: + """Aggregates over the OK results' ``usage`` (read with .get, so legacy + prompt_sha-only artifacts yield []/n/False): + + * ``run_head_shas`` sorted unique non-empty; ``run_head_missing`` count of + missing/"" head shas; + * ``run_dirty_any``: True if any repo_dirty is True; else None if any run has + a head sha and repo_dirty None; else False; + * ``run_marker_mismatch``: runs whose repo_branch / repo_common_dir differs + from the manifest's own value or is empty/None; + * ``run_not_isolated``: runs whose codex_home_isolated is not True OR whose + repo_ref_names / repo_worktree_paths is None. + """ + heads = set() + missing = 0 + dirty_any = False + dirty_none = False + marker_mismatch = 0 + not_isolated = 0 + for r in results: + if not getattr(r, "ok", True): + continue + u = getattr(r, "usage", None) or {} + head = u.get("repo_head_sha") + if head: + heads.add(head) + else: + missing += 1 + if u.get("repo_dirty") is True: + dirty_any = True + elif head and u.get("repo_dirty") is None: + dirty_none = True + if ( + not u.get("repo_branch") + or not u.get("repo_common_dir") + or u.get("repo_branch") != manifest_prov.get("repo_branch") + or u.get("repo_common_dir") != manifest_prov.get("repo_common_dir") + ): + marker_mismatch += 1 + if ( + u.get("codex_home_isolated") is not True + or u.get("repo_ref_names") is None + or u.get("repo_worktree_paths") is None + ): + not_isolated += 1 + return { + "run_head_shas": sorted(heads), + "run_head_missing": missing, + "run_dirty_any": True if dirty_any else (None if dirty_none else False), + "run_marker_mismatch": marker_mismatch, + "run_not_isolated": not_isolated, + } + + +def _provenance_failures(prov: dict) -> list: + """Names of provenance fields in their documented FAILURE form (fail-soft + ``""``/``None``): non-empty shas, branch and common dir; ``repo_dirty`` a + bool; ``repo_ref_names`` and ``repo_worktree_paths`` LISTS (never None).""" + bad = [] + for key in ("repo_head_sha", "repo_tree_sha", "repo_branch", "repo_common_dir"): + if not prov.get(key): + bad.append(key) + if not isinstance(prov.get("repo_dirty"), bool): + bad.append("repo_dirty") + for key in ("repo_ref_names", "repo_worktree_paths"): + if not isinstance(prov.get(key), list): + bad.append(key) + return bad + + +_PROVENANCE_KEYS = ( + "repo_head_sha", + "repo_tree_sha", + "repo_dirty", + "repo_branch", + "repo_common_dir", + "repo_worktree_paths", + "repo_ref_names", +) +_AGGREGATE_KEYS = ( + "run_head_shas", + "run_head_missing", + "run_dirty_any", + "run_marker_mismatch", + "run_not_isolated", +) +_MANIFEST_MARKER_KEYS = ( + "repo_branch", + "repo_common_dir", + "repo_worktree_paths", + "repo_ref_names", + "codex_home_path", +) +_BUNDLE_ID_TOKEN = "__BUNDLE_ID__" + + +def _render_blinded_bundle(runs, run_ids_sha: str) -> "tuple[dict, str]": + """DETERMINISTIC blinded bundle from the sealed runs: (mapping, bundle text + with the Bundle-ID TOKEN still in place). ``compare --blinded`` writes it; + ``verdict`` regenerates it from the same sealed artifacts and compares, so + the grader-visible evidence is bound to the artifacts, not only to itself.""" + from eval_core.compare import ( + apply_blinding, + build_bundle, + derive_blind_mapping, + render_default_header, + sanitize_model_refs, + ) + + config_ids = sorted({r.config_id for r in runs}) + model_names = sorted({r.model for r in runs if r.model}) + mapping = derive_blind_mapping(config_ids, run_ids_sha) + blind_runs = apply_blinding(runs, mapping, model_names) + # The default "How to grade" header with the BLIND labels as columns (so the + # pr_review.md pointer survives), plus the Bundle ID line graders copy. + has_repeats = any(r.repeat_idx > 0 for r in blind_runs) + header_b = render_default_header(sorted(mapping.values()), has_repeats) + ( + f"\nBundle ID: `{_BUNDLE_ID_TOKEN}` - copy this verbatim into your grading " + f"table as `bundle_id` (the verdict stage rejects a table whose bundle_id " + f"does not match this bundle).\n" + ) + # Belt-and-suspenders: sanitize the WHOLE rendered bundle once more, so a + # model name that leaked through any renderer path is still scrubbed. The id + # is the hash of the SANITIZED bytes with the token still in place. + with_token = sanitize_model_refs( + build_bundle(blind_runs, redact_meta=True, header_text=header_b), model_names + ) + return mapping, with_token + + +def _bundle_id_of(bundle_text_with_token: str) -> str: + """Identity of one blinded bundle = sha256 of the exact grader-visible bytes + with the id slot tokenized (then substituted). First 16 hex.""" + import hashlib + + return hashlib.sha256(bundle_text_with_token.encode("utf-8")).hexdigest()[:16] + + +def _run_ids_sha(run_ids) -> str: + import hashlib + + return hashlib.sha256("|".join(sorted(run_ids)).encode("utf-8")).hexdigest() + + +def _file_sha256(path: str) -> str: + import hashlib + + with open(path, "rb") as fh: + return hashlib.sha256(fh.read()).hexdigest() + + +def _registered_candidates() -> set: + """Candidate ids of the pre-registered contrasts (reviewer_verdict).""" + import reviewer_verdict + + return {cand for _ctrl, cand in reviewer_verdict.REGISTERED_CONTRASTS} + + +def _candidate_exposure_violations( + configs, k: int, k_overrides: dict, strata, command: str +) -> list: + """Pre-registration BY CONSTRUCTION, enforced in code: a registered + candidate arm may run on the corpus ONLY as the exact registered campaign - + its registered contrast {control, candidate}, k == 2, no --k-per, no + --strata filtering - and never under `smoke`. Anything else would expose + the candidate's corpus output before the decisive run.""" + import reviewer_verdict + + selected = {c.id for c in configs} + cands = selected & _registered_candidates() + if not cands: + return [] + if command == "smoke": + return [f"candidate arm(s) {sorted(cands)} may not be smoked (first corpus run = campaign)"] + problems = [] + for ctrl, cand in reviewer_verdict.REGISTERED_CONTRASTS: + if cand in cands and selected != {ctrl, cand}: + problems.append( + f"candidate {cand} runs only as --configs {ctrl},{cand} (got {sorted(selected)})" + ) + if k != 2: + problems.append(f"the registered campaign is --k 2 (got --k {k})") + if k_overrides: + problems.append(f"no --k-per in the registered campaign (got {k_overrides})") + if strata: + problems.append(f"no --strata filtering in the registered campaign (got {strata})") + return problems + + +def _attempts_path(subdir: str) -> str: + """Append-only per-attempt record, a SIBLING of runs// (never parsed + as a RunResult). Each `run` attempt appends a START record before dispatch + and an END record (per-cell infra errors, first 120 chars) after the matrix + returns; an attempt that raised before dispatch has no END record and does + not count toward the cap.""" + return os.path.join(RUNS_DIR, f"{subdir}-attempts.jsonl") + + +MAX_RERUNS = 3 # DECISION_RULE.md: at most 3 reruns after the first attempt + + +def _read_jsonl_strict(path: str, required: "dict[str, tuple]") -> list: + """Append-only integrity logs FAIL CLOSED: a malformed line (a truncated or + corrupt append) or a record missing a required field refuses with the file + and line number, never a silent skip - a skipped record would undercount the + rerun cap or erase the first candidate-exposure record.""" + import json as _json + + if not os.path.exists(path): + return [] + out = [] + with open(path, encoding="utf-8") as fh: + for lineno, raw in enumerate(fh, start=1): + line = raw.strip() + if not line: + continue + try: + rec = _json.loads(line) + except ValueError as exc: + raise SystemExit(f"{path}:{lineno}: malformed record ({exc}); the log fails closed") + if not isinstance(rec, dict): + raise SystemExit(f"{path}:{lineno}: record is not an object; the log fails closed") + kind = rec.get("kind") + for field in required.get(kind, required.get("*", ())): + if field not in rec: + raise SystemExit( + f"{path}:{lineno}: {kind!r} record missing {field!r}; the log fails closed" + ) + if kind not in required and "*" not in required: + raise SystemExit( + f"{path}:{lineno}: unknown record kind {kind!r}; the log fails closed" + ) + out.append(rec) + return out + + +_ATTEMPT_SCHEMA = {"start": ("at", "head", "configs", "k"), "end": ("started_at", "failures")} +_LEDGER_SCHEMA = {"dispatch": ("contrast", "head", "tree", "subdir", "at")} + + +def _read_attempts(subdir: str) -> list: + return _read_jsonl_strict(_attempts_path(subdir), _ATTEMPT_SCHEMA) + + +def _lock_path(subdir: str) -> str: + return os.path.join(RUNS_DIR, f"{subdir}.lock") + + +@contextlib.contextmanager +def _subdir_lock(subdir: str): + """Exclusive cross-process ownership of one run subdir for the whole of a + `run` attempt (stop-condition check through manifest write). Two attempts + into the same subdir compute identical run ids and would interleave their + artifacts into one store that the last writer then seals; the second + process is refused instead. `flock` is released by the kernel when the + holder exits, so a crashed attempt leaves no stale lock. Yields True when + held, False when another process holds it.""" + import fcntl + + os.makedirs(RUNS_DIR, exist_ok=True) + with open(_lock_path(subdir), "a", encoding="utf-8") as fh: + try: + fcntl.flock(fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + yield False + return + try: + yield True + finally: + fcntl.flock(fh.fileno(), fcntl.LOCK_UN) + + +def _append_attempt(subdir: str, record: dict) -> None: + import json as _json + + os.makedirs(RUNS_DIR, exist_ok=True) + with open(_attempts_path(subdir), "a", encoding="utf-8") as fh: + fh.write(_json.dumps(record, sort_keys=True) + "\n") + + +def _ok_cells(results, store_root: str) -> list: + """Identity + artifact hash of every OK cell of an attempt (cached and fresh + alike), recorded in the END record so a later attempt can prove it RESUMED + them rather than re-sampling: a previously successful cell may never be + re-dispatched (DECISION_RULE.md: reruns reuse OK artifacts).""" + import hashlib as _hashlib + + out = [] + for r in results: + if not getattr(r, "ok", False) or not getattr(r, "run_id", ""): + continue + path = os.path.join(store_root, f"{r.run_id}.json") + try: + with open(path, "rb") as fh: + digest = _hashlib.sha256(fh.read()).hexdigest() + except OSError: + digest = None + out.append( + { + "case": r.case_id, + "arm": r.config_id, + "repeat": int(r.repeat_idx), + "run_id": r.run_id, + "sha256": digest, + } + ) + return sorted(out, key=lambda c: (c["case"], c["arm"], c["repeat"])) + + +def _resume_integrity_violations(subdir: str, store_root: str) -> list: + """Before a new attempt: every OK cell ANY previous END record lists (the + cumulative set, aborted attempts included - an aborted END records the + cells persisted before the abort) must still be present as the same + artifact bytes. A missing or changed one means the rerun would RE-DISPATCH + a previously successful cell (selective re-sampling), which fails closed.""" + import hashlib as _hashlib + + cumulative: dict = {} + for rec in _read_attempts(subdir): + if rec.get("kind") != "end": + continue + for cell in rec.get("ok_cells") or []: + cumulative[cell["run_id"]] = cell # the latest record of a cell wins + problems = [] + for cell in cumulative.values(): + path = os.path.join(store_root, f"{cell['run_id']}.json") + if not os.path.isfile(path): + problems.append( + f"OK cell {cell['case']}/{cell['arm']}/r{cell['repeat']} artifact missing ({cell['run_id']})" + ) + continue + with open(path, "rb") as fh: + digest = _hashlib.sha256(fh.read()).hexdigest() + if cell.get("sha256") and digest != cell["sha256"]: + problems.append( + f"OK cell {cell['case']}/{cell['arm']}/r{cell['repeat']} artifact changed since the last attempt" + ) + return problems + + +def _stop_condition_violations(subdir: str) -> list: + """The pre-registered stop conditions, checked BEFORE a new attempt runs: + (1) at most MAX_RERUNS reruns after the first attempt (= 4 dispatched + attempts in all); (2) a (case, arm, repeat) whose END records show a + timeout in the last two consecutive dispatched attempts stops the campaign + (surfaced, never silently excluded).""" + ends = [a for a in _read_attempts(subdir) if a.get("kind") == "end"] + problems = [] + if len(ends) >= 1 + MAX_RERUNS: + problems.append( + f"{len(ends)} dispatched attempts already recorded for runs/{subdir} " + f"(1 attempt + {MAX_RERUNS} reruns is the cap); stop and surface to the user" + ) + if len(ends) >= 2: + last, prev = ends[-1], ends[-2] + + def _timeouts(rec): + return { + (f["case"], f["arm"], f["repeat"]) + for f in rec.get("failures", []) + if "timed out" in (f.get("infra_error") or "") + } + + both = _timeouts(last) & _timeouts(prev) + if both: + problems.append( + f"cell(s) {sorted(both)} timed out in 2 consecutive attempts; stop and surface " + f"(never silently excluded, never graded partial)" + ) + return problems + + +# The pre-registered surfaces: everything the campaign executes or is judged by. +REGISTERED_SURFACES = ( + "tools/reviewer-eval", + "tools/eval_core", + ".claude/scripts/openai_review.py", + ".github/codex/prompts/pr_review.md", + "tools/notebook_md_extract.py", +) +_TIMEOUT_LINE_RE = __import__("re").compile(r"^[-+]\s*CALL_TIMEOUT_S\s*=\s*[\d.]+\s*$") + + +def _ledger_path() -> str: + """Append-only CAMPAIGN ledger (campaign-global, not per subdir): one record + per dispatch of a registered contrast, with the HEAD/tree it ran on and the + subdir. The FIRST record for a contrast is its frozen candidate-exposure + revision; `verdict` judges every later run against it.""" + return os.path.join(RUNS_DIR, "campaign-ledger.jsonl") + + +def _append_ledger(record: dict) -> None: + import json as _json + + os.makedirs(RUNS_DIR, exist_ok=True) + with open(_ledger_path(), "a", encoding="utf-8") as fh: + fh.write(_json.dumps(record, sort_keys=True) + "\n") + + +def _read_ledger() -> list: + return _read_jsonl_strict(_ledger_path(), _LEDGER_SCHEMA) + + +def _registered_surface_diff(repo_root: str, frozen_head: str, head: str) -> "tuple[str, bool]": + """(``git diff --stat`` over REGISTERED_SURFACES between the two commits, + whether the FULL content diff is exactly the single CALL_TIMEOUT_S line in + adapters/codex_reviewer.py - the ONE pre-registered exception).""" + stat = _git_out( + repo_root, ["diff", "--stat", frozen_head, head, "--", *REGISTERED_SURFACES] + ).strip() + if not stat: + return "", False + full = _git_out(repo_root, ["diff", frozen_head, head, "--", *REGISTERED_SURFACES]) + files = {ln[6:] for ln in full.splitlines() if ln.startswith("+++ b/")} + changed = [ + ln + for ln in full.splitlines() + if (ln.startswith("+") or ln.startswith("-")) + and not ln.startswith("+++") + and not ln.startswith("---") + ] + timeout_only = False + if ( + files == {"tools/reviewer-eval/adapters/codex_reviewer.py"} + and len(changed) == 2 + and all(_TIMEOUT_LINE_RE.match(ln) for ln in changed) + ): + removed = [float(ln.split("=", 1)[1]) for ln in changed if ln.startswith("-")] + added = [float(ln.split("=", 1)[1]) for ln in changed if ln.startswith("+")] + # The rule permits RAISING the ceiling only (a lower cap changes censoring). + timeout_only = len(removed) == 1 and len(added) == 1 and added[0] > removed[0] + return stat, timeout_only + + +def _frozen_ref_name(contrast: "tuple[str, str]") -> str: + """Local git ref (never pushed; outside refs/heads and refs/tags) that + anchors the contrast's FIRST candidate exposure to a commit. It lives in + the repository, not under the disposable runs/ dir, so deleting the ledger + cannot reset the pre-registration boundary.""" + return f"refs/campaign/{contrast[0]}-{contrast[1]}/frozen" + + +def _read_frozen_ref(repo_root: str, contrast: "tuple[str, str]") -> "Optional[str]": + import subprocess + + cp = subprocess.run( + ["git", "rev-parse", "--verify", "-q", _frozen_ref_name(contrast)], + cwd=repo_root, + capture_output=True, + text=True, + check=False, + ) + if cp.returncode != 0: + return None + return cp.stdout.strip() or None + + +def _anchor_frozen_ref(repo_root: str, contrast: "tuple[str, str]", head: str) -> "Optional[str]": + """Create the frozen ref at ``head`` if absent; return the ref's sha.""" + existing = _read_frozen_ref(repo_root, contrast) + if existing: + return existing + _git_out(repo_root, ["update-ref", _frozen_ref_name(contrast), head]) + return head + + +def _exposure_status( + manifest: dict, subdir: str, contrast: "tuple[str, str]", repo_root: str +) -> dict: + """Judge this campaign against the contrast's FIRST dispatch in the ledger. + + Mechanical (pre-registration intact) iff this subdir IS the first dispatch, + OR the HEAD moved (a voided campaign re-run into a fresh subdir) and the + registered surfaces are untouched or changed ONLY by the CALL_TIMEOUT_S + line. A second subdir on the SAME head is a re-exposure of the candidate; + any other registered-surface change is "re-registered after candidate + exposure": the verdict is still computed but never a mechanical GO. + """ + entries = [e for e in _read_ledger() if list(e.get("contrast") or []) == list(contrast)] + if not entries: + return { + "ledger_entries": 0, + "mechanical": False, + "protocol_error": "no campaign-ledger dispatch record for this contrast", + "reason": "no campaign-ledger dispatch record for this contrast", + } + first = entries[0] + anchor = _read_frozen_ref(repo_root, contrast) + # Integrity of the frozen anchor itself: these are PROTOCOL ERRORS (verdict + # refuses at (f)), not the documented non-mechanical re-exposure cases. + protocol_error = None + if not anchor: + protocol_error = f"frozen ref {_frozen_ref_name(contrast)} is absent" + elif first.get("head") != anchor: + protocol_error = ( + f"ledger first head {first.get('head')} != frozen ref {anchor} " + f"(ledger rewritten or reset)" + ) + elif first.get("ledger_reset"): + protocol_error = "ledger was reset after the frozen ref existed (exposure history lost)" + if protocol_error: + return { + "ledger_entries": len(entries), + "mechanical": False, + "protocol_error": protocol_error, + "reason": protocol_error, + } + head = manifest.get("repo_head_sha") + status = { + "ledger_entries": len(entries), + "frozen_head": first.get("head"), + "first_subdir": first.get("subdir"), + "subdirs": sorted({e.get("subdir") for e in entries}), + "this_subdir": subdir, + "head_changed": head != first.get("head"), + "surface_diff_stat": "", + "timeout_only": False, + } + if subdir == first.get("subdir") and not status["head_changed"]: + status.update(mechanical=True, reason="first dispatch of the contrast (frozen revision)") + return status + if not status["head_changed"]: + status.update(mechanical=False, reason="re-exposure: a second subdir on the frozen HEAD") + return status + if subdir == first.get("subdir"): + status.update( + mechanical=False, + reason="HEAD changed but the first subdir was reused (a voided campaign re-runs into a FRESH subdir)", + ) + return status + stat, timeout_only = _registered_surface_diff(repo_root, first.get("head") or "", head or "") + status["surface_diff_stat"] = stat + status["timeout_only"] = timeout_only + if not stat: + status.update(mechanical=True, reason="re-run on a new HEAD; registered surfaces untouched") + elif timeout_only: + status.update( + mechanical=True, + reason="re-run on a new HEAD; the single CALL_TIMEOUT_S exception applies", + ) + else: + status.update( + mechanical=False, + reason="re-registered after candidate exposure: registered surfaces changed since the frozen HEAD", + ) + return status + + +def _load_manifest_runs(subdir: str, stage: str): + """(manifest, OK runs scoped to the manifest's run_ids) or SystemExit.""" + manifest_path = _manifest_path(subdir) + try: + manifest = read_json(manifest_path) + except (OSError, ValueError): + raise SystemExit(f"{stage}: no readable manifest at {manifest_path}; run `run` first.") + if not isinstance(manifest, dict) or manifest.get("failed") or not manifest.get("run_ids"): + raise SystemExit( + f"{stage}: the last `run` into runs/{subdir} failed/incomplete (or has no " + f"run_ids); no valid experiment. Fix the failing case(s) and re-run `run`." + ) + store = RunStore(os.path.join(RUNS_DIR, subdir)) + wanted = set(manifest["run_ids"]) + runs = [r for r in store.load_all() if r.run_id in wanted] + missing = wanted - {r.run_id for r in runs} + if missing: + raise SystemExit( + f"{stage}: manifest lists {len(wanted)} run(s) but {len(missing)} artifact(s) " + f"are missing/unreadable under runs/{subdir}: {sorted(missing)[:6]}" + ) + return manifest, runs + + def _verify_cases(loader, cases) -> int: """Fail-closed preflight: materialize + sanity-check every selected case BEFORE any Codex call, so a stale/malformed case can't be reviewed and graded against @@ -293,10 +964,12 @@ def cmd_smoke(args: argparse.Namespace) -> int: if configs is None: print(_bad_configs_msg(configs_arg), file=sys.stderr) return 1 + exposure = _candidate_exposure_violations(configs, args.k, {}, args.strata, "smoke") + if exposure: + print("refusing (pre-registration): " + "; ".join(exposure), file=sys.stderr) + return 1 if _verify_cases(loader, cases): return 1 - reviewer = _build_reviewer(repo_root) - print(f"codex CLI version: {reviewer.cli_version()}") store = RunStore(os.path.join(RUNS_DIR, "smoke")) # smoke is a LIVE plumbing/auth/connectivity check (the README's "first real codex # call"), so it must actually exercise codex every time — clear any cached smoke @@ -304,16 +977,21 @@ def cmd_smoke(args: argparse.Namespace) -> int: for name in os.listdir(store.root): if name.endswith(".json"): os.remove(os.path.join(store.root, name)) - results = run_matrix( - cases, - configs, - reviewer, - store, - k=args.k, - max_parallel=args.max_parallel, - progress=lambda m: print(f" {m}"), - treatment_fields=_treatment_fields(), - ) + # Every harness codex call runs under an isolated CODEX_HOME that mirrors CI's + # codex-action home (auth + sandbox/approval only; no MCP, hooks, memories). + with isolated_codex_home(): + reviewer = _build_reviewer(repo_root) + print(f"codex CLI version: {reviewer.cli_version()}") + results = run_matrix( + cases, + configs, + reviewer, + store, + k=args.k, + max_parallel=args.max_parallel, + progress=lambda m: print(f" {m}"), + treatment_fields=_treatment_fields(), + ) ok = sum(1 for r in results if r.ok) print(f"\nsmoke: {ok}/{len(results)} runs ok") for r in results: @@ -323,20 +1001,65 @@ def cmd_smoke(args: argparse.Namespace) -> int: def cmd_run(args: argparse.Namespace) -> int: - from eval_core.runner import run_matrix - if not _safe_subdir(args.subdir): print( f"invalid --subdir {args.subdir!r}: must stay within the runs directory", file=sys.stderr, ) return 1 + # One attempt per subdir at a time, held through the manifest write: a + # concurrent `run` into the same subdir would write the same run ids into + # the same store (an artifact set mixed from two invocations that the last + # one seals and verdicts). + with _subdir_lock(args.subdir) as held: + if not held: + print( + f"refusing (concurrent run): another `run` holds {_lock_path(args.subdir)}; " + "one attempt per subdir at a time - wait for it or use a different --subdir", + file=sys.stderr, + ) + return 1 + return _cmd_run_locked(args) + + +def _cmd_run_locked(args: argparse.Namespace) -> int: + from eval_core.runner import run_matrix + + # Pre-registered stop conditions FIRST, before anything on disk is touched: a + # refused fifth attempt must leave the fourth attempt's successful manifest + # (and its sealed artifacts) intact for compare/audit/verdict. + stops = _stop_condition_violations(args.subdir) + if stops: + print("refusing (stop condition): " + "; ".join(stops), file=sys.stderr) + return 1 + resample = _resume_integrity_violations(args.subdir, os.path.join(RUNS_DIR, args.subdir)) + if resample: + print( + "refusing (resume integrity): a rerun must reuse every previously OK artifact; " + + "; ".join(resample), + file=sys.stderr, + ) + return 1 + repo_root = find_repo_root() + # Provenance PREFLIGHT, also before the manifest is touched and before any + # codex call: a dirty tree or an incomplete git sample would spend the + # candidate's first corpus exposure on a campaign the post-run gate rejects + # (and resume would keep reusing the repo_dirty=True artifacts). + preflight = _manifest_provenance(repo_root) + bad_prov = _provenance_failures(preflight) + if bad_prov or preflight.get("repo_dirty") is not False: + why = f"incomplete git provenance: {bad_prov}" if bad_prov else "working tree is dirty" + print( + f"refusing (provenance preflight): {why}; a campaign runs only on a clean, " + f"fully sampled tree (`git status --porcelain --untracked-files=all` must be empty).", + file=sys.stderr, + ) + return 1 # A run attempt into supersedes any prior experiment there: invalidate an # EXISTING manifest now, so even an early exit (bad configs, zero cases, a corpus- # load error) can't leave the previous run's manifest live for compare. A fresh # subdir stays manifest-less here; compare fails closed on a missing manifest. _invalidate_manifest_if_exists(args.subdir) - repo_root = find_repo_root() loader = CorpusLoader(CORPUS_DIR, repo_root) cases = loader.load_cases(args.strata) if not cases: @@ -354,6 +1077,10 @@ def cmd_run(args: argparse.Namespace) -> int: file=sys.stderr, ) return 1 + exposure = _candidate_exposure_violations(configs, args.k, k_overrides, args.strata, "run") + if exposure: + print("refusing (pre-registration): " + "; ".join(exposure), file=sys.stderr) + return 1 # Past input validation, this is a real run attempt: write the failure marker # now and overwrite it with run_ids ONLY on success. Every failure from here on # (an invalid case, reviewer/runner error, an interrupt, or an infra failure) @@ -365,23 +1092,128 @@ def cmd_run(args: argparse.Namespace) -> int: write_json(_manifest_path(args.subdir), {"failed": True, "error": "run did not complete"}) if _verify_cases(loader, cases): return 1 - reviewer = _build_reviewer(repo_root) + # An ignored/excluded extra case dir under corpus/cases/ would be enumerated + # by CorpusLoader; refuse BEFORE any codex call (shared with the tree-identity + # guard the post-run stages apply). + _assert_no_hidden_inputs(repo_root) store = RunStore(os.path.join(RUNS_DIR, args.subdir)) # If run_matrix raises (ConfoundMismatch, CLIVersionMismatch, an interrupt, ...) # the up-front failure marker stays in place and the exception propagates. - results = run_matrix( - cases, - configs, - reviewer, - store, - k=args.k, - max_parallel=args.max_parallel, - progress=lambda m: print(f" {m}"), - treatment_fields=_treatment_fields(), - k_overrides=k_overrides, + # The reviewer, the matrix AND the manifest write run under ONE isolated + # CODEX_HOME (CI-faithful; its path is a manifest field and an audit marker). + import datetime as _dt + + attempt_start = _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + _append_attempt( + args.subdir, + { + "kind": "start", + "at": attempt_start, + "head": preflight.get("repo_head_sha", ""), # the validated preflight sample + "configs": [c.id for c in configs], + "k": args.k, + }, ) - ok = sum(1 for r in results if r.ok) - infra = [r for r in results if not r.ok] + dispatched = {"v": False} + planned: list = [] # the run keys of THIS attempt's matrix (from the runner's plan) + selected_ids = {c.id for c in configs} + import reviewer_verdict as _rv + + registered = [c for c in _rv.REGISTERED_CONTRASTS if set(c) == selected_ids] + + def _on_dispatch() -> None: + dispatched["v"] = True + if registered: + # Campaign-global exposure record: the FIRST entry for a contrast is + # its frozen revision; verdict judges every later run against it. The + # frozen ref (in .git, not under runs/) anchors that revision; a ledger + # with no entries while the ref already exists was reset -> recorded. + contrast = registered[0] + prior = [e for e in _read_ledger() if list(e.get("contrast") or []) == list(contrast)] + existing_ref = _read_frozen_ref(repo_root, contrast) + _anchor_frozen_ref(repo_root, contrast, preflight.get("repo_head_sha", "")) + _append_ledger( + { + "kind": "dispatch", + "contrast": list(contrast), + "head": preflight.get("repo_head_sha", ""), + "tree": preflight.get("repo_tree_sha", ""), + "subdir": args.subdir, + "at": attempt_start, + "ledger_reset": bool(existing_ref and not prior), + } + ) + + with isolated_codex_home(): + reviewer = _build_reviewer(repo_root) + try: + results = run_matrix( + cases, + configs, + reviewer, + store, + k=args.k, + max_parallel=max(1, args.max_parallel), + progress=lambda m: print(f" {m}"), + treatment_fields=_treatment_fields(), + k_overrides=k_overrides, + on_dispatch=_on_dispatch, + on_plan=planned.extend, + ) + except BaseException as exc: # noqa: BLE001 - classify by dispatch, not by type + if not dispatched["v"]: + # Raised during PLANNING (CLI pin, confounds, experiment/case tags, + # resume verification): no codex call was made, so no END record + # and the attempt does not count toward the cap (DECISION_RULE.md). + raise + # Codex calls may have started: record the attempt as ABORTED so it + # counts toward the rerun cap, WITH every OK cell this attempt holds + # ON DISK for its planned keys (cached or persisted an instant before + # the abort - the store, not an in-memory callback, is the truth), so + # the resume-integrity check protects them like a completed attempt's. + held = [a for a in (store.load(k) for k in planned) if a is not None and a.ok] + _append_attempt( + args.subdir, + { + "kind": "end", + "started_at": attempt_start, + "aborted": type(exc).__name__, + "ok": None, + "total": None, + "ok_cells": _ok_cells(held, store.root), + "failures": [], + }, + ) + raise + codex_home_path = _codex_reviewer_mod.ACTIVE_ISOLATED_HOME + ok = sum(1 for r in results if r.ok) + infra = [r for r in results if not r.ok] + if not infra: + provenance = _manifest_provenance(repo_root) + # Append the END record (the per-attempt failure record of the stop + # conditions) BEFORE anything can overwrite the artifacts - but ONLY for a + # dispatched attempt: a fully cached resume made no codex call and must not + # consume the rerun cap. + if dispatched["v"]: + _append_attempt( + args.subdir, + { + "kind": "end", + "started_at": attempt_start, + "ok": ok, + "total": len(results), + "ok_cells": _ok_cells(results, store.root), + "failures": [ + { + "case": r.case_id, + "arm": r.config_id, + "repeat": r.repeat_idx, + "infra_error": (r.infra_error or "")[:120], + } + for r in infra + ], + }, + ) if infra: # Fail closed: an INFRA_ERROR (a notebook case, or a codex/worktree failure) # means the experiment is incomplete. Replace the up-front marker with a @@ -408,18 +1240,39 @@ def cmd_run(args: argparse.Namespace) -> int: # rubric provenance (base_prompt_sha) so compare can refuse if pr_review.md drifts # between run and compare (the grader is pointed at the live rubric). run_ids = [r.run_id for r in results if r.run_id] - write_json( - _manifest_path(args.subdir), - { - "run_ids": run_ids, - "configs": [c.id for c in configs], - "base_prompt_sha": getattr(reviewer, "base_prompt_sha", ""), - # Provenance for humans reading the manifest (compare derives repeats - # from the artifacts themselves). - "k": args.k, - "k_per": k_overrides, - }, - ) + incomplete = _provenance_failures(provenance) + if incomplete: + # A write-time git failure would blank the computed audit markers; fail + # closed (failure marker, exit 2) so the operator re-invokes the attempt + # block - resume reuses every OK artifact, no codex call is repeated. + write_json( + _manifest_path(args.subdir), + {"failed": True, "error": f"incomplete write-time provenance: {incomplete}"}, + ) + print( + f"\n{ok}/{len(results)} runs ok but the write-time git provenance is incomplete " + f"({', '.join(incomplete)}); manifest marked failed - re-run `run` (resume reuses " + f"the OK artifacts).", + file=sys.stderr, + ) + return 2 + # Seal the artifacts AT RUN TIME: compare --blinded / audit / verdict refuse if + # the live artifacts no longer hash to this value. + sealed = reviewer_audit.artifacts_sha(store.root, run_ids) + manifest = { + "run_ids": run_ids, + "configs": [c.id for c in configs], + "base_prompt_sha": getattr(reviewer, "base_prompt_sha", ""), + # Provenance for humans reading the manifest (compare derives repeats + # from the artifacts themselves). + "k": args.k, + "k_per": k_overrides, + "artifacts_sha": sealed, + "codex_home_path": codex_home_path, + } + manifest.update({k: provenance.get(k) for k in _PROVENANCE_KEYS}) + manifest.update(_provenance_aggregates(results, provenance)) + write_json(_manifest_path(args.subdir), manifest) print(f"\n{ok}/{len(results)} runs ok. Next: compare --subdir {args.subdir}") return 0 @@ -541,6 +1394,23 @@ def cmd_compare(args: argparse.Namespace) -> int: file=sys.stderr, ) return 1 + if blinded: + # Engine identity = tree identity, then the run-time seal (both AFTER the + # --allow-mixed / missing-manifest refusals above). + repo_root_live = find_repo_root() + _assert_tree_identity(repo_root_live, manifest, "compare --blinded (stage entry)") + sealed = manifest.get("artifacts_sha") if isinstance(manifest, dict) else None + if sealed: + live_sha = reviewer_audit.artifacts_sha( + os.path.join(RUNS_DIR, args.subdir), sorted(wanted or ()) + ) + if live_sha != sealed: + print( + f"run artifacts changed since `run` sealed them: live artifacts_sha " + f"{live_sha} != manifest {sealed}; refusing to build a blinded bundle.", + file=sys.stderr, + ) + return 1 bundle = build_bundle(runs) out_dir = os.path.join(RUNS_DIR, args.subdir) os.makedirs(out_dir, exist_ok=True) @@ -551,19 +1421,17 @@ def cmd_compare(args: argparse.Namespace) -> int: n_cases = len({r.case_id for r in runs}) print(f"wrote {out_path} ({ok}/{len(runs)} runs ok across {n_cases} cases)") if blinded: - from eval_core.compare import apply_blinding, derive_blind_mapping, sanitize_model_refs - - config_ids = sorted({r.config_id for r in runs}) - model_names = sorted({r.model for r in runs if r.model}) - salt = hashlib.sha256("|".join(sorted(wanted or ())).encode("utf-8")).hexdigest() - mapping = derive_blind_mapping(config_ids, salt) - blind_runs = apply_blinding(runs, mapping, model_names) - # Belt-and-suspenders: sanitize the WHOLE rendered bundle once more, so a - # model name that leaked through any renderer path is still scrubbed. - blind_bundle = sanitize_model_refs(build_bundle(blind_runs, redact_meta=True), model_names) + + run_ids_sha = _run_ids_sha(wanted or ()) + mapping, with_token = _render_blinded_bundle(runs, run_ids_sha) + bundle_id = _bundle_id_of(with_token) + blind_bundle = with_token.replace(_BUNDLE_ID_TOKEN, bundle_id) blind_path = os.path.join(out_dir, "comparison.blinded.md") with open(blind_path, "w", encoding="utf-8") as fh: fh.write(blind_bundle) + repo_root_live = find_repo_root() + rubric_path = os.path.join(HERE, "GRADING_RUBRIC.md") + prompt_path = os.path.join(repo_root_live, ".github", "codex", "prompts", "pr_review.md") # The seal: graders must never read this file (or config/configs.json). # Unblind the finished grading table by joining on these labels. write_json( @@ -573,9 +1441,19 @@ def cmd_compare(args: argparse.Namespace) -> int: "models": {r.config_id: r.model for r in runs}, "efforts": {r.config_id: getattr(r, "effort", "") for r in runs}, "salt_source": "sha256 of the sorted manifest run_ids", + "bundle_id": bundle_id, + "run_ids_sha": run_ids_sha, + "artifacts_sha": ( + manifest.get("artifacts_sha") if isinstance(manifest, dict) else None + ), + "rubric_sha256": _file_sha256(rubric_path) if os.path.exists(rubric_path) else None, + "prompt_sha256": _file_sha256(prompt_path) if os.path.exists(prompt_path) else None, }, ) - print(f"wrote {blind_path} (blind labels: {', '.join(sorted(mapping.values()))})") + print( + f"wrote {blind_path} (blind labels: {', '.join(sorted(mapping.values()))}; bundle_id {bundle_id})" + ) + _assert_tree_identity(repo_root_live, manifest, "compare --blinded (stage exit)") print( " BLINDING: give graders ONLY comparison.blinded.md; they must not read " "blinding.json or config/configs.json. Unblind finished tables via blinding.json." @@ -587,7 +1465,264 @@ def cmd_compare(args: argparse.Namespace) -> int: return 0 -def main() -> int: +def cmd_audit(args: argparse.Namespace) -> int: + """Post-unblinding session-log audit -> hits.json (informational).""" + if not _safe_subdir(args.subdir): + print( + f"invalid --subdir {args.subdir!r}: must stay within the runs directory", + file=sys.stderr, + ) + return 1 + manifest, runs = _load_manifest_runs(args.subdir, "audit") + repo_root = find_repo_root() + _assert_tree_identity(repo_root, manifest, "audit (stage entry)") + markers = {k: manifest.get(k) for k in _MANIFEST_MARKER_KEYS} + try: + result = reviewer_audit.audit( + runs, + os.path.join(RUNS_DIR, args.subdir), + repo_root, + markers, + _run_ids_sha(manifest["run_ids"]), + manifest.get("artifacts_sha") or "", + ) + except (ValueError, FileNotFoundError) as exc: + print(f"audit refused: {exc}", file=sys.stderr) + return 1 + write_json(args.output, result) + cov = result["coverage"] + print( + f"wrote {args.output}: {len(result['hits'])} hit(s); coverage covered={cov['covered']} " + f"no_commands={cov['no_commands']} unaudited={cov['unaudited']} total={cov['total']}" + + ("; AUDIT VACUOUS" if result["vacuous"] else "") + ) + _assert_tree_identity(repo_root, manifest, "audit (stage exit)") + return 0 + + +def cmd_verdict(args: argparse.Namespace) -> int: + """Pinned check order: (a) arguments, (b) manifest, (c) blinding.json binding, + (d) grades + bundle_id + validation, (e) --audit-hits binding, (f) protocol + guards, (g) tree identity, (h) the gates.""" + import json as _json + + import reviewer_verdict + from eval_core.compare import derive_blind_mapping + from eval_core.models import to_jsonable + + # (a) + if not _safe_subdir(args.subdir): + print(f"invalid --subdir {args.subdir!r}", file=sys.stderr) + return 1 + if args.control == args.candidate: + raise SystemExit("verdict: --control and --candidate must differ") + if (args.control, args.candidate) not in reviewer_verdict.REGISTERED_CONTRASTS: + raise SystemExit( + f"verdict: ({args.control}, {args.candidate}) is not the registered contrast " + f"{reviewer_verdict.REGISTERED_CONTRASTS}; a future campaign registers its own " + f"pair in its own PR-1." + ) + # (b) + manifest, runs = _load_manifest_runs(args.subdir, "verdict") + repo_root = find_repo_root() + subdir_path = os.path.join(RUNS_DIR, args.subdir) + run_ids_sha = _run_ids_sha(manifest["run_ids"]) + # (c) + blinding_path = os.path.join(subdir_path, "blinding.json") + try: + blinding = read_json(blinding_path) + except (OSError, ValueError): + raise SystemExit(f"verdict (c): no readable blinding.json at {blinding_path}") + assert isinstance(blinding, dict) + expected_mapping = derive_blind_mapping(sorted({r.config_id for r in runs}), run_ids_sha) + if blinding.get("mapping") != expected_mapping: + raise SystemExit( + "verdict (c): blinding.json mapping != deterministic mapping from the manifest" + ) + if blinding.get("run_ids_sha") != run_ids_sha: + raise SystemExit("verdict (c): blinding.json run_ids_sha != manifest run_ids") + live_artifacts = reviewer_audit.artifacts_sha(subdir_path, manifest["run_ids"]) + if manifest.get("artifacts_sha") != live_artifacts: + raise SystemExit( + f"verdict (c): manifest artifacts_sha {manifest.get('artifacts_sha')} != live {live_artifacts}" + ) + if blinding.get("artifacts_sha") != live_artifacts: + raise SystemExit( + f"verdict (c): blinding.json artifacts_sha {blinding.get('artifacts_sha')} != live {live_artifacts}" + ) + bundle_path = os.path.join(subdir_path, "comparison.blinded.md") + try: + with open(bundle_path, encoding="utf-8") as fh: + bundle_text = fh.read() + except OSError: + raise SystemExit(f"verdict (c): blinded bundle missing at {bundle_path}") + expected_bundle = blinding.get("bundle_id") or "" + # The grader-visible bundle is bound to the SEALED ARTIFACTS, not only to + # itself: regenerate it deterministically from the manifest runs and require + # the file to be exactly that rendering (a coordinated edit of bundle + + # blinding.json + grades that stays self-consistent is still refused). + _mapping_regen, regen_with_token = _render_blinded_bundle(runs, run_ids_sha) + if _bundle_id_of(regen_with_token) != expected_bundle: + raise SystemExit( + "verdict (c): blinding.json bundle_id != the bundle regenerated from the sealed artifacts" + ) + if bundle_text != regen_with_token.replace(_BUNDLE_ID_TOKEN, expected_bundle): + raise SystemExit( + "verdict (c): comparison.blinded.md differs from the bundle regenerated from the sealed artifacts" + ) + if ( + not expected_bundle + or _bundle_id_of(bundle_text.replace(expected_bundle, _BUNDLE_ID_TOKEN)) != expected_bundle + ): + raise SystemExit( + "verdict (c): comparison.blinded.md does not hash to its recorded bundle_id" + ) + rubric_path = os.path.join(HERE, "GRADING_RUBRIC.md") + prompt_path = os.path.join(repo_root, ".github", "codex", "prompts", "pr_review.md") + if blinding.get("rubric_sha256") != _file_sha256(rubric_path): + raise SystemExit("verdict (c): blinding.json rubric_sha256 != live GRADING_RUBRIC.md") + if blinding.get("prompt_sha256") != _file_sha256(prompt_path): + raise SystemExit("verdict (c): blinding.json prompt_sha256 != live pr_review.md") + # (d) + grades = read_json(args.grades) + assert isinstance(grades, dict) + if grades.get("bundle_id") != expected_bundle: + raise SystemExit( + f"verdict (d): grades bundle_id {grades.get('bundle_id')!r} != blinded bundle {expected_bundle!r}" + ) + grades = reviewer_verdict.unblind(grades, blinding["mapping"]) + runs_jsonable = [to_jsonable(r) for r in runs] + try: + reviewer_verdict.validate_grades(grades, runs_jsonable) + except ValueError as exc: + raise SystemExit(f"verdict (d): {exc}") + # (e) + audit_hits = None + if args.audit_hits: + audit_hits = read_json(args.audit_hits) + assert isinstance(audit_hits, dict) + if audit_hits.get("run_ids_sha") != run_ids_sha: + raise SystemExit("verdict (e): hits.json run_ids_sha != manifest") + if audit_hits.get("artifacts_sha") != live_artifacts: + raise SystemExit("verdict (e): hits.json artifacts_sha != live artifacts") + if audit_hits.get("branch") != manifest.get("repo_branch"): + raise SystemExit("verdict (e): hits.json branch != manifest repo_branch") + markers = {k: manifest.get(k) for k in _MANIFEST_MARKER_KEYS} + recomputed = reviewer_audit.audit( + runs, subdir_path, repo_root, markers, run_ids_sha, live_artifacts + ) + for key in ("hits", "unaudited", "no_commands", "coverage", "vacuous"): + if _json.dumps(audit_hits.get(key), sort_keys=True) != _json.dumps( + recomputed.get(key), sort_keys=True + ): + raise SystemExit( + f"verdict (e): hits.json {key!r} differs from the recomputed audit:\n" + f" file: {_json.dumps(audit_hits.get(key), sort_keys=True)[:400]}\n" + f" recomputed: {_json.dumps(recomputed.get(key), sort_keys=True)[:400]}" + ) + # (f) + violations = [] + configs = set(manifest.get("configs") or []) + if configs != {args.control, args.candidate}: + violations.append( + f"manifest configs {sorted(configs)} != {{{args.control}, {args.candidate}}}" + ) + loader = CorpusLoader(CORPUS_DIR, repo_root) + corpus = loader.load_cases(None) + corpus_ids = {c.id for c in corpus} + run_cases = {r.case_id for r in runs} + if run_cases != corpus_ids: + violations.append( + f"OK runs cover {len(run_cases)} case(s) but the committed corpus has {len(corpus_ids)} " + f"(no --strata filtering allowed): missing {sorted(corpus_ids - run_cases)[:6]}, " + f"extra {sorted(run_cases - corpus_ids)[:6]}" + ) + n_s4 = sum(1 for c in corpus if c.stratum == "s4_missed") + if len(corpus_ids) < 8 or n_s4 < 2: + violations.append( + f"corpus below the floor: {len(corpus_ids)} cases, {n_s4} s4_missed (need >= 8 / >= 2)" + ) + expected_grid = { + (c, a, r) for c in corpus_ids for a in (args.control, args.candidate) for r in (0, 1) + } + realized = [(r.case_id, r.config_id, int(r.repeat_idx)) for r in runs] + if len(realized) != len(set(realized)): + violations.append("duplicate (case, arm, repeat) among the manifest's OK runs") + if set(realized) != expected_grid: + violations.append( + f"realized grid != cases x {{control, candidate}} x {{0, 1}}: missing " + f"{sorted(expected_grid - set(realized))[:6]}, extra {sorted(set(realized) - expected_grid)[:6]}" + ) + prov = {k: manifest.get(k) for k in _PROVENANCE_KEYS} + recomputed_agg = _provenance_aggregates(runs, prov) + stored_agg = {k: manifest.get(k) for k in _AGGREGATE_KEYS} + if stored_agg != recomputed_agg: + violations.append( + f"manifest aggregates {stored_agg} != recomputed from the sealed artifacts {recomputed_agg}" + ) + if recomputed_agg["run_head_missing"] != 0: + violations.append(f"run_head_missing = {recomputed_agg['run_head_missing']}") + if recomputed_agg["run_dirty_any"] is not False: + violations.append(f"run_dirty_any = {recomputed_agg['run_dirty_any']!r}") + if recomputed_agg["run_head_shas"] != [manifest.get("repo_head_sha")]: + violations.append( + f"run_head_shas {recomputed_agg['run_head_shas']} != [{manifest.get('repo_head_sha')}]" + ) + if recomputed_agg["run_marker_mismatch"] != 0 or recomputed_agg["run_not_isolated"] != 0: + violations.append( + f"run_marker_mismatch={recomputed_agg['run_marker_mismatch']} " + f"run_not_isolated={recomputed_agg['run_not_isolated']} (both must be 0)" + ) + if manifest.get("repo_dirty") is not False: + violations.append(f"manifest repo_dirty = {manifest.get('repo_dirty')!r}") + bad_prov = _provenance_failures(prov) + if bad_prov: + violations.append(f"manifest write-time provenance incomplete: {bad_prov}") + if args.expect_head and manifest.get("repo_head_sha") != args.expect_head: + violations.append( + f"repo_head_sha {manifest.get('repo_head_sha')} != --expect-head {args.expect_head}" + ) + exposure = _exposure_status(manifest, args.subdir, (args.control, args.candidate), repo_root) + if exposure.get("protocol_error"): + violations.append(f"frozen-provenance integrity: {exposure['protocol_error']}") + if violations: + raise SystemExit( + "verdict (f): protocol guard refused a definitive verdict:\n - " + + "\n - ".join(violations) + ) + # (g) + _assert_tree_identity(repo_root, manifest, "verdict (stage entry)") + # (h) + result = reviewer_verdict.gates(grades, runs_jsonable, args.control, args.candidate, audit_hits) + # Pre-registered informational reads (latency, 2x-p95 flag, input cost) from + # the sealed artifacts; prices from the production script's PRICING table. + from adapters.openai_review_loader import load_openai_review + + pricing = getattr(load_openai_review(repo_root), "PRICING", {}) + input_prices = {m: p[0] for m, p in pricing.items()} + result["campaign_reads"] = reviewer_verdict.campaign_reads( + runs_jsonable, args.control, args.candidate, input_prices + ) + result["exposure"] = exposure + if not exposure["mechanical"]: + # Pre-registration did not survive: computed and reported, never a + # mechanical GO - the final call is the user's, recorded in the verdict. + result["definitive"] = False + result["label"] = ( + f"{result['verdict']} (RE-REGISTERED after candidate exposure - " + f"non-mechanical, user decision: {exposure['reason']})" + ) + print(_json.dumps(result, indent=2, sort_keys=True)) + print(f"\nVERDICT: {result['label']}") + if result["contested_gates"]: + print( + f"CONTESTED (verifier-confirmed leaks): {result['contested_gates']} - the final call is the user's" + ) + _assert_tree_identity(repo_root, manifest, "verdict (stage exit)") + return 0 + + +def _build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description="Codex reviewer A/B comparison harness") sub = p.add_subparsers(dest="cmd", required=True) @@ -610,10 +1745,15 @@ def main() -> int: ps.set_defaults(func=cmd_smoke) pr = sub.add_parser("run", help="full comparison matrix; saves raw reviews") - pr.add_argument("--configs", default="A,B") + pr.add_argument("--configs", default="B,E") pr.add_argument("--strata", nargs="*", default=None) pr.add_argument("--subdir", default="full") - pr.add_argument("--k", type=int, default=1, help="repeats per (case, config)") + pr.add_argument( + "--k", + type=int, + default=2, + help="repeats per (case, config); the default B,E contrast is registered at k=2", + ) pr.add_argument( "--k-per", default="", @@ -637,7 +1777,31 @@ def main() -> int: ) pc.set_defaults(func=cmd_compare) - args = p.parse_args() + pa = sub.add_parser("audit", help="post-unblinding session-log audit -> hits.json") + pa.add_argument("--subdir", default="full") + pa.add_argument( + "-o", + "--output", + required=True, + help="where to write hits.json (outside the repo or under runs/)", + ) + pa.set_defaults(func=cmd_audit) + + pvd = sub.add_parser( + "verdict", help="apply the pre-registered gates to a reconciled grades JSON" + ) + pvd.add_argument("--subdir", default="full") + pvd.add_argument("--grades", required=True) + pvd.add_argument("--control", required=True) + pvd.add_argument("--candidate", required=True) + pvd.add_argument("--audit-hits", default=None) + pvd.add_argument("--expect-head", default=None) + pvd.set_defaults(func=cmd_verdict) + return p + + +def main() -> int: + args = _build_parser().parse_args() return args.func(args)