From 090d87122f85e0ddd9225bd5738eb35e5897ab4f Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:38:13 -0500 Subject: [PATCH 1/3] fix(evals): run invocation probes without subagents and name the run line in run-validity Emitted probe runs now remove Agent beside Read, Glob and Grep: after the skill loaded, the agent handed the task to a subagent that outlived the 300-second timeout, and every timed-out probe run in the #11895 round had dispatched one. run-validity.py gains a probe tools check that warns when a probe run could still call those tools and names the run line. Co-Authored-By: Claude Opus 5.5 --- ...obe-suite-validity-probe-tools-8d4b2f61.md | 7 +++ ...-probe-suite-validity-run-line-3e7a0c95.md | 7 +++ ...-suite-validity-disallow-agent-5c1e9a27.md | 7 +++ plugins/evals/skills/plugin-eval/SKILL.md | 5 +- .../plugin-eval/reference/reading-results.md | 6 +++ .../plugin-eval/scripts/run-validity.py | 49 +++++++++++++++++- .../plugin-eval/scripts/test_run_validity.py | 50 +++++++++++++++++++ .../reference/rewrite-a-plugin.md | 2 +- .../reference/invocation-probes.md | 17 ++++--- .../scripts/measure-invocation.sh | 5 +- .../scripts/measure-invocation.test.sh | 6 +-- 11 files changed, 145 insertions(+), 16 deletions(-) create mode 100644 .changes/evals/fix-probe-suite-validity-probe-tools-8d4b2f61.md create mode 100644 .changes/playbooks/fix-probe-suite-validity-run-line-3e7a0c95.md create mode 100644 .changes/skill-quality/fix-probe-suite-validity-disallow-agent-5c1e9a27.md diff --git a/.changes/evals/fix-probe-suite-validity-probe-tools-8d4b2f61.md b/.changes/evals/fix-probe-suite-validity-probe-tools-8d4b2f61.md new file mode 100644 index 00000000000..4f9ec64860f --- /dev/null +++ b/.changes/evals/fix-probe-suite-validity-probe-tools-8d4b2f61.md @@ -0,0 +1,7 @@ +--- +bump: patch +--- + +### Fixed + +- `run-validity.py` adds a `probe tools` check: it warns when an invocation-probe run could call Read, Glob, Grep or Agent, and names the emitted README's `--disallow-tools Read Glob Grep Agent` run line. A probe run without that line was failing on denied searches and subagent timeouts with no pointer to the cause. `reading-results.md` now states how the gate reads invocation probes. diff --git a/.changes/playbooks/fix-probe-suite-validity-run-line-3e7a0c95.md b/.changes/playbooks/fix-probe-suite-validity-run-line-3e7a0c95.md new file mode 100644 index 00000000000..a19d0998722 --- /dev/null +++ b/.changes/playbooks/fix-probe-suite-validity-run-line-3e7a0c95.md @@ -0,0 +1,7 @@ +--- +bump: patch +--- + +### Fixed + +- `rewrite-a-plugin.md` step 5 runs the emitted trigger probes with the run line from the README beside them, rather than a copy of its flags that went stale. diff --git a/.changes/skill-quality/fix-probe-suite-validity-disallow-agent-5c1e9a27.md b/.changes/skill-quality/fix-probe-suite-validity-disallow-agent-5c1e9a27.md new file mode 100644 index 00000000000..a6115395935 --- /dev/null +++ b/.changes/skill-quality/fix-probe-suite-validity-disallow-agent-5c1e9a27.md @@ -0,0 +1,7 @@ +--- +bump: patch +--- + +### Fixed + +- `measure-invocation.sh emit-plugin-eval` README and `reference/invocation-probes.md` add `Agent` to the probe run line, now `--disallow-tools Read Glob Grep Agent`. After the skill loaded, the agent handed the task to a subagent that outlived the 300-second run timeout, and those timed-out runs made every probe side INVALID. diff --git a/plugins/evals/skills/plugin-eval/SKILL.md b/plugins/evals/skills/plugin-eval/SKILL.md index 3426657428c..ca2e30b85ea 100644 --- a/plugins/evals/skills/plugin-eval/SKILL.md +++ b/plugins/evals/skills/plugin-eval/SKILL.md @@ -297,8 +297,9 @@ Read in this order. Stopping early at any step is the finding. for a plugin file it could not read, so the fact belongs in the hub. The exceptions that turn a denial into a warning (a denial the environment makes in both arms, such as a user-scope read of `~/.claude/standards`; a denial in a case whose denial-free runs scored the same; an invocation - probe's denial), how a Bash denial's paths are read, gating an A/B pass, and the 0-turn - `context: fork` run the gate does not fail are in + probe's denial), how a Bash denial's paths are read, gating an A/B pass, the 0-turn + `context: fork` run the gate does not fail, and how it reads an invocation probe's misses and + tools are in [reading-results.md, Denials](reference/reading-results.md#denials). 2. `partial`. `true` (with `partialReason` of `cost_ceiling`, `interrupted`, or `auth_failed`) means the suite did not finish: report that and keep the document out of any trend. diff --git a/plugins/evals/skills/plugin-eval/reference/reading-results.md b/plugins/evals/skills/plugin-eval/reference/reading-results.md index ff38db98ad9..f90519b7dd2 100644 --- a/plugins/evals/skills/plugin-eval/reference/reading-results.md +++ b/plugins/evals/skills/plugin-eval/reference/reading-results.md @@ -60,6 +60,12 @@ above the plugin's directory, or with no absolute path or no known plugin direct - An invocation probe's denial of a named call other than Skill is a warning when it follows the graded Skill call, or the run held a should-not-fire case whose denial-free runs all held; the gate prints each probe case's fire counts with and without denials. +- An invocation probe is a case whose `prompt.md` tags include `invocation-probe`, as + `/skill-quality:check measure-invocation` emits it. Its should-fire misses are the trigger rate + it measures: `skill fired` reports them as a warning with per-case counts, never a FAIL. Run + errors and denials keep their full force, apart from the two probe exceptions above and a + max-turns error after the graded Skill call. `probe tools` warns when a probe run could call + Read, Glob, Grep or Agent, the tools the emitted README's run line removes. ## Document fields diff --git a/plugins/evals/skills/plugin-eval/scripts/run-validity.py b/plugins/evals/skills/plugin-eval/scripts/run-validity.py index 864b5a4f0ba..29823d4ef81 100755 --- a/plugins/evals/skills/plugin-eval/scripts/run-validity.py +++ b/plugins/evals/skills/plugin-eval/scripts/run-validity.py @@ -55,7 +55,11 @@ skill fired every with-arm run of a should-trigger case fired the skill; in an invocation-probe case a miss is the trigger rate it measures, reported per case and overall as a warning - models the model and judge model, where the result or traces + probe tools no invocation-probe run's session could call Read, Glob, + Grep or Agent, which the emitted README removes with + --disallow-tools; one that could is a warning naming that + run line + models the model and judge model, where the result or traces carry them; runs that disagree are a FAIL judge votes a split llm judge vote is a warning ceiling a case whose without-arm is already 1.00 is a warning and @@ -180,6 +184,10 @@ r"(?:exit \d+: )?Reached maximum number of turns \(\d+\)\.?" ) TRACE_ERRORS = "errors" # on the result line +TRACE_TOOLS = "tools" # on the init line: the tools the session could call +# Tools emit-plugin-eval's README removes from a probe run with --disallow-tools. +# The init line names the Agent tool Task. +PROBE_REMOVED_TOOLS = ("Read", "Glob", "Grep", "Agent", "Task") # case.yaml's context.scaffold_script, in block or flow form, read without a # YAML parser. Any uncommented mention of scaffold_script outside these shapes, @@ -1092,6 +1100,44 @@ def check_fired(cases, runs, root): ) +def check_probe_tools(runs, root): + """An invocation-probe run whose session could call a tool the emitted README + removes ran without that run line: in its empty workspace those calls are + denied, and a subagent it dispatches outlives the run's timeout. Warns, and + names the fix, since the denial and error checks already judge the runs.""" + found = {} # the removable tools a run could call -> the runs that could + for r in runs: + if not is_probe(r.case, root): + continue + for line in r.trace or []: + if (line.get(LINE_TYPE), line.get(LINE_SUBTYPE)) == TRACE_INIT: + tools = line.get(TRACE_TOOLS) + extra = ", ".join( + t for t in (tools if isinstance(tools, list) else []) + if t in PROBE_REMOVED_TOOLS + ) + if extra: + found.setdefault(extra, []).append(r) + break + if found: + return ( + WARN, + "%s of invocation-probe cases could call %s; run the emitted README's" + " line, --disallow-tools Read Glob Grep Agent, so a probe run cannot be" + " denied a search or outlive its timeout in a subagent" + % ( + plural(sum(len(rs) for rs in found.values()), "run"), + "/".join(PROBE_REMOVED_TOOLS), + ), + [ + "%s: %s, first %s" % (tools, plural(len(rs), "run"), rs[0].where) + for tools, rs in found.items() + ], + "invocation probes ran without --disallow-tools Read Glob Grep Agent", + ) + return PASS, "no invocation-probe run could call Read, Glob, Grep or Agent", [], None + + def recorded(result, keys): for scope in (result, as_dict(result.get(SUITE))): for key in keys: @@ -1296,6 +1342,7 @@ def main(argv=None): ("session started", check_started(runs)), ("permission denials", check_denials(runs, plugin_roots, root)), ("skill fired", check_fired(cases, runs, root)), + ("probe tools", check_probe_tools(runs, root)), ("models", check_models(result, runs)), ("judge votes", check_votes(runs)), ("ceiling", check_ceiling(cases, arms)), diff --git a/plugins/evals/skills/plugin-eval/scripts/test_run_validity.py b/plugins/evals/skills/plugin-eval/scripts/test_run_validity.py index 16efef1e3f7..51f79d4ee6e 100755 --- a/plugins/evals/skills/plugin-eval/scripts/test_run_validity.py +++ b/plugins/evals/skills/plugin-eval/scripts/test_run_validity.py @@ -1137,6 +1137,56 @@ def test_an_unfired_case_beside_an_unfired_probe_is_still_invalid(self): ) self.assertNotIn("unfired: case noise-before-gain", proc.stdout) + def probe_with_tools(self, tools, tags="tags: [invocation-probe, train]"): + """Case noise-before-gain, its prompt.md carrying tags, whose with-arm run's + init line lists tools: the session's callable tools as the trace records them.""" + result = self.clean() + case = result["cases"][2] + (self.dir / "suite" / case["dir"] / "prompt.md").write_text( + "---\ndescription: d\n%s\n---\n\nQ\n" % tags, encoding="utf-8" + ) + trace = self.dir / "tools.jsonl" + trace.write_text( + json.dumps({"type": "system", "subtype": "init", + "model": "claude-opus-5-5", "tools": tools}) + + "\n" + + json.dumps({"type": "result", "subtype": "success", "is_error": False}) + + "\n", + encoding="utf-8", + ) + path = self.write(result) + data = load(path) + data["cases"][2]["arms"]["with"][0]["tracePath"] = str(trace) + path.write_text(json.dumps(data), encoding="utf-8") + return self.run_script(path, "--runs", "1") + + def test_a_probe_run_that_could_search_or_dispatch_warns_with_the_run_line(self): + # The tool list a probe run's init line recorded on Claude Code 2.1.316 + # when the run left out the emitted README's --disallow-tools. + proc = self.probe_with_tools( + ["Task", "Glob", "Grep", "Read", "Skill", "TaskStop", "ToolSearch"] + ) + self.assertVerdict(proc, "VALID", 0) + self.assertIn("check probe tools: WARN (1 run of invocation-probe cases", proc.stdout) + self.assertIn("--disallow-tools Read Glob Grep Agent", proc.stdout) + self.assertIn( + " Task, Glob, Grep, Read: 1 run, first case noise-before-gain, with-arm run 1\n", + proc.stdout, + ) + + def test_a_probe_run_with_only_skill_passes_probe_tools(self): + # The init tools a `--disallowed-tools Agent Read Glob Grep` session + # recorded on Claude Code 2.1.316, plus Skill. + proc = self.probe_with_tools(["Bash", "Skill", "TaskStop", "ToolSearch"]) + self.assertVerdict(proc, "VALID", 0) + self.assertIn("check probe tools: PASS", proc.stdout) + + def test_a_case_outside_the_probes_may_search(self): + proc = self.probe_with_tools( + ["Task", "Glob", "Grep", "Read", "Skill"], tags="tags: [knowledge]" + ) + self.assertIn("check probe tools: PASS", proc.stdout) + def capped_probe( self, tool="Skill", diff --git a/plugins/playbooks/skills/skill-authoring/reference/rewrite-a-plugin.md b/plugins/playbooks/skills/skill-authoring/reference/rewrite-a-plugin.md index c1a49daecd8..8f7da2da210 100644 --- a/plugins/playbooks/skills/skill-authoring/reference/rewrite-a-plugin.md +++ b/plugins/playbooks/skills/skill-authoring/reference/rewrite-a-plugin.md @@ -47,7 +47,7 @@ ships; this page says how to run it across a plugin. "Step N" below is the proto pass together: each version on each target model, with and without the plugin, the same `--runs`, judge and threshold, and `--keep-temp`. For triggering, emit cases from the held-out probes with `/skill-quality:check measure-invocation` `emit-plugin-eval` and run them with - `--ablation none --disallow-tools Read Glob Grep`. Write any exclusion rule (a case the harness cannot run, say) before reading a + the run line in the README it writes beside them. Write any exclusion rule (a case the harness cannot run, say) before reading a number, and apply it to both versions. Then, per suite and model: - `/evals:plugin-eval` must judge each result VALID. - Read its two-version noise report, after against before, at the protocol's margin. diff --git a/plugins/skill-quality/reference/invocation-probes.md b/plugins/skill-quality/reference/invocation-probes.md index 97fe79180b9..c338b037cc5 100644 --- a/plugins/skill-quality/reference/invocation-probes.md +++ b/plugins/skill-quality/reference/invocation-probes.md @@ -103,16 +103,19 @@ the span). Should-not-trigger probes are exempt. ## emit-plugin-eval results -Run the emitted cases with `--ablation none --keep-temp --disallow-tools Read Glob Grep`, as the -emitted README shows. The run workspace is empty, so an agent that can search looks outside it, -and `run-validity.py` fails the run on those denied calls. A case's `allowed_tools` only adds -tools, so `--disallow-tools` is what removes them. `run-validity.py` reads the traces, which -resolve only with `--keep-temp`. +Run the emitted cases with `--ablation none --keep-temp --disallow-tools Read Glob Grep Agent`, +as the emitted README shows. The run workspace is empty, so an agent that can search looks +outside it, and `run-validity.py` fails the run on those denied calls. An agent that can dispatch +a subagent, after the skill loads, hands it the task, and the run outlives its timeout. A case's +`allowed_tools` only adds tools, so `--disallow-tools` is what removes them. `run-validity.py` +reads the traces, which resolve only with `--keep-temp`, and its `probe tools` check warns when a +probe run could still call one of these tools. - **Pointer**: `claude plugin eval --help`, the `--disallow-tools` line; the plugin-evals page, , documents grants only. **As of**: - 2026-10-31, Claude Code 2.1.316. **Recheck trigger**: the flag leaves the help, or that page - documents removing a read-only tool. + 2026-11-01, Claude Code 2.1.316. **Recheck trigger**: the flag leaves the help, that page + documents removing a tool per case, or a kept probe trace's init line lists Agent (as `Task`) + with the flag passed. No script ingests `plugin-eval` or `claude -p` results. Turning them into a report JSON that matches `score` output, so `compare` can diff it against the baseline, is a manual step diff --git a/plugins/skill-quality/scripts/measure-invocation.sh b/plugins/skill-quality/scripts/measure-invocation.sh index d5d15630126..8f0d1d389d8 100755 --- a/plugins/skill-quality/scripts/measure-invocation.sh +++ b/plugins/skill-quality/scripts/measure-invocation.sh @@ -597,11 +597,12 @@ Run within one plugin at a time (the CLI loads a single plugin): ``` claude plugin eval plugins/ --eval-dir --trust-plugin \ - --ablation none --keep-temp --disallow-tools Read Glob Grep + --ablation none --keep-temp --disallow-tools Read Glob Grep Agent ``` The run workspace is empty, so without `--disallow-tools` the agent searches -outside it and the denied calls make the run INVALID. +outside it, or hands the task to a subagent that outlives the run's timeout; +the denied calls and timed-out runs make the run INVALID. A with-only `tool_used: Skill` grader is a plugin-fired indicator, not a score. Copy the per-case predicted fire/quiet bits into a report JSON matching diff --git a/plugins/skill-quality/scripts/measure-invocation.test.sh b/plugins/skill-quality/scripts/measure-invocation.test.sh index 74abec36fa8..08bc5b5c8e4 100755 --- a/plugins/skill-quality/scripts/measure-invocation.test.sh +++ b/plugins/skill-quality/scripts/measure-invocation.test.sh @@ -214,10 +214,10 @@ else fail "emit-plugin-eval prompt.md lacks the four-tool allowed_tools line" fi -if grep -qF -- '--disallow-tools Read Glob Grep' "$emit_dir/README.md"; then - pass "emit-plugin-eval README runs the cases without the read-only search tools" +if grep -qF -- '--disallow-tools Read Glob Grep Agent' "$emit_dir/README.md"; then + pass "emit-plugin-eval README runs the cases without the search tools or subagents" else - fail "emit-plugin-eval README run line lacks --disallow-tools Read Glob Grep" + fail "emit-plugin-eval README run line lacks --disallow-tools Read Glob Grep Agent" fi if run emit-plugin-eval "$TMP/probes" "$emit_dir" >/dev/null 2>&1; then From b4ee1c029e1f14a98f89499fe228fb77d2824246 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:39:07 -0500 Subject: [PATCH 2/3] docs(evals): keep the plugin-eval skill body under its line cap Co-Authored-By: Claude Opus 5.5 --- plugins/evals/skills/plugin-eval/SKILL.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/evals/skills/plugin-eval/SKILL.md b/plugins/evals/skills/plugin-eval/SKILL.md index ca2e30b85ea..10699649a99 100644 --- a/plugins/evals/skills/plugin-eval/SKILL.md +++ b/plugins/evals/skills/plugin-eval/SKILL.md @@ -297,10 +297,8 @@ Read in this order. Stopping early at any step is the finding. for a plugin file it could not read, so the fact belongs in the hub. The exceptions that turn a denial into a warning (a denial the environment makes in both arms, such as a user-scope read of `~/.claude/standards`; a denial in a case whose denial-free runs scored the same; an invocation - probe's denial), how a Bash denial's paths are read, gating an A/B pass, the 0-turn - `context: fork` run the gate does not fail, and how it reads an invocation probe's misses and - tools are in - [reading-results.md, Denials](reference/reading-results.md#denials). + probe's denial), Bash denial paths, A/B gating, the 0-turn `context: fork` run the gate passes, + and invocation-probe misses and tools: [reading-results.md, Denials](reference/reading-results.md#denials). 2. `partial`. `true` (with `partialReason` of `cost_ceiling`, `interrupted`, or `auth_failed`) means the suite did not finish: report that and keep the document out of any trend. 3. Per run, `skippedPaidGraders: true` or a non-null `error`. A skipped judge grader is still From b97668626f10db0b124dbc2a943826474cec32ea Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:48:22 -0500 Subject: [PATCH 3/3] fix(evals): leave probe tools unchecked when no probe trace shows its tools check_probe_tools passed when it had read nothing: a probe run with no trace, or an init line with no tools list, added nothing. Such a run is now a warning naming it, and none readable is UNCHECKED, so the run is INVALID. The traces line names probe tools among the checks an unresolved trace leaves unchecked. The clean fixture's init line and the tests' inline init lines now carry a tools list, as real init lines do. Co-Authored-By: Claude Opus 5.5 --- ...obe-suite-validity-probe-tools-8d4b2f61.md | 2 +- .../plugin-eval/reference/reading-results.md | 3 +- .../fixtures/run-validity/traces/clean.jsonl | 2 +- .../plugin-eval/scripts/run-validity.py | 82 +++++++++++++----- .../plugin-eval/scripts/test_run_validity.py | 83 ++++++++++++++++--- 5 files changed, 136 insertions(+), 36 deletions(-) diff --git a/.changes/evals/fix-probe-suite-validity-probe-tools-8d4b2f61.md b/.changes/evals/fix-probe-suite-validity-probe-tools-8d4b2f61.md index 4f9ec64860f..e14acae1745 100644 --- a/.changes/evals/fix-probe-suite-validity-probe-tools-8d4b2f61.md +++ b/.changes/evals/fix-probe-suite-validity-probe-tools-8d4b2f61.md @@ -4,4 +4,4 @@ bump: patch ### Fixed -- `run-validity.py` adds a `probe tools` check: it warns when an invocation-probe run could call Read, Glob, Grep or Agent, and names the emitted README's `--disallow-tools Read Glob Grep Agent` run line. A probe run without that line was failing on denied searches and subagent timeouts with no pointer to the cause. `reading-results.md` now states how the gate reads invocation probes. +- `run-validity.py` adds a `probe tools` check: it warns when an invocation-probe run could call Read, Glob, Grep or Agent, and names the emitted README's `--disallow-tools Read Glob Grep Agent` run line. A probe run whose trace has no init tools list is a warning, and when no probe run's list can be read the check is UNCHECKED, which makes the run INVALID. A probe run without that line was failing on denied searches and subagent timeouts with no pointer to the cause. `reading-results.md` now states how the gate reads invocation probes. diff --git a/plugins/evals/skills/plugin-eval/reference/reading-results.md b/plugins/evals/skills/plugin-eval/reference/reading-results.md index f90519b7dd2..0d97a2a94a1 100644 --- a/plugins/evals/skills/plugin-eval/reference/reading-results.md +++ b/plugins/evals/skills/plugin-eval/reference/reading-results.md @@ -65,7 +65,8 @@ above the plugin's directory, or with no absolute path or no known plugin direct it measures: `skill fired` reports them as a warning with per-case counts, never a FAIL. Run errors and denials keep their full force, apart from the two probe exceptions above and a max-turns error after the graded Skill call. `probe tools` warns when a probe run could call - Read, Glob, Grep or Agent, the tools the emitted README's run line removes. + Read, Glob, Grep or Agent, the tools the emitted README's run line removes, or its trace has no + init tools list; when no probe run's list can be read, the check is UNCHECKED. ## Document fields diff --git a/plugins/evals/skills/plugin-eval/scripts/fixtures/run-validity/traces/clean.jsonl b/plugins/evals/skills/plugin-eval/scripts/fixtures/run-validity/traces/clean.jsonl index b212b89a172..00467e1d334 100644 --- a/plugins/evals/skills/plugin-eval/scripts/fixtures/run-validity/traces/clean.jsonl +++ b/plugins/evals/skills/plugin-eval/scripts/fixtures/run-validity/traces/clean.jsonl @@ -1,2 +1,2 @@ -{"type":"system","subtype":"init","model":"claude-opus-5-5","claude_code_version":"2.1.287","permissionMode":"dontAsk"} +{"type":"system","subtype":"init","model":"claude-opus-5-5","claude_code_version":"2.1.287","permissionMode":"dontAsk","tools":["Skill"]} {"type":"result","subtype":"success","is_error":false,"permission_denials":[]} diff --git a/plugins/evals/skills/plugin-eval/scripts/run-validity.py b/plugins/evals/skills/plugin-eval/scripts/run-validity.py index 29823d4ef81..3114492b240 100755 --- a/plugins/evals/skills/plugin-eval/scripts/run-validity.py +++ b/plugins/evals/skills/plugin-eval/scripts/run-validity.py @@ -58,7 +58,8 @@ probe tools no invocation-probe run's session could call Read, Glob, Grep or Agent, which the emitted README removes with --disallow-tools; one that could is a warning naming that - run line + run line. A probe run whose init tools list cannot be read + is a warning; none readable is UNCHECKED models the model and judge model, where the result or traces carry them; runs that disagree are a FAIL judge votes a split llm judge vote is a warning @@ -1104,38 +1105,79 @@ def check_probe_tools(runs, root): """An invocation-probe run whose session could call a tool the emitted README removes ran without that run line: in its empty workspace those calls are denied, and a subagent it dispatches outlives the run's timeout. Warns, and - names the fix, since the denial and error checks already judge the runs.""" + names the fix, since the denial and error checks already judge the runs. A + probe run with no trace, no init line or no tools list on it is unread: none + read is UNCHECKED, some unread a warning.""" found = {} # the removable tools a run could call -> the runs that could + probes, unread = 0, [] for r in runs: if not is_probe(r.case, root): continue - for line in r.trace or []: - if (line.get(LINE_TYPE), line.get(LINE_SUBTYPE)) == TRACE_INIT: - tools = line.get(TRACE_TOOLS) - extra = ", ".join( - t for t in (tools if isinstance(tools, list) else []) - if t in PROBE_REMOVED_TOOLS - ) - if extra: - found.setdefault(extra, []).append(r) - break - if found: + probes += 1 + tools = next( + ( + line.get(TRACE_TOOLS) + for line in r.trace or [] + if (line.get(LINE_TYPE), line.get(LINE_SUBTYPE)) == TRACE_INIT + ), + None, + ) + if not isinstance(tools, list): + unread.append(r) + continue + extra = ", ".join(t for t in tools if t in PROBE_REMOVED_TOOLS) + if extra: + found.setdefault(extra, []).append(r) + if not probes: + return PASS, "no invocation-probe case", [], None + unread_details = [ + "%s: no init tools list (%s)" % (r.where, r.trace_path or "no trace") + for r in unread + ] + if len(unread) == probes: return ( - WARN, + UNCHECKED, + "no invocation-probe run's init tools list could be read", + unread_details, + "probe tools unchecked in %s with no readable init tools list" + % plural(len(unread), "run"), + ) + summary, reasons = [], [] + if found: + summary.append( "%s of invocation-probe cases could call %s; run the emitted README's" " line, --disallow-tools Read Glob Grep Agent, so a probe run cannot be" " denied a search or outlive its timeout in a subagent" % ( plural(sum(len(rs) for rs in found.values()), "run"), "/".join(PROBE_REMOVED_TOOLS), - ), + ) + ) + reasons.append( + "invocation probes ran without --disallow-tools Read Glob Grep Agent" + ) + if unread: + summary.append( + "%s with no readable init tools list" % plural(len(unread), "probe run") + ) + reasons.append("probe tools unread in %s" % plural(len(unread), "run")) + if summary: + return ( + WARN, + "; ".join(summary), [ "%s: %s, first %s" % (tools, plural(len(rs), "run"), rs[0].where) for tools, rs in found.items() - ], - "invocation probes ran without --disallow-tools Read Glob Grep Agent", + ] + + unread_details, + "; ".join(reasons), ) - return PASS, "no invocation-probe run could call Read, Glob, Grep or Agent", [], None + return ( + PASS, + "no invocation-probe run could call Read, Glob, Grep or Agent", + [], + None, + ) def recorded(result, keys): @@ -1330,8 +1372,8 @@ def main(argv=None): if unresolved: lines.append( "traces: %d of %d tracePaths do not resolve (the run needs --keep-temp, and its" - " directories must still exist); permission denials and the trace model are" - " unchecked for those runs" % (len(unresolved), len(runs)) + " directories must still exist); permission denials, probe tools and the" + " trace model are unchecked for those runs" % (len(unresolved), len(runs)) ) lines += [ " %s: %s" % (r.where, r.trace_path or "no tracePath") for r in unresolved diff --git a/plugins/evals/skills/plugin-eval/scripts/test_run_validity.py b/plugins/evals/skills/plugin-eval/scripts/test_run_validity.py index 51f79d4ee6e..4248d3bed41 100755 --- a/plugins/evals/skills/plugin-eval/scripts/test_run_validity.py +++ b/plugins/evals/skills/plugin-eval/scripts/test_run_validity.py @@ -363,7 +363,7 @@ def test_errored_or_aborted_runs_are_invalid(self): def test_a_trace_that_ends_in_an_error_is_invalid(self): trace = self.dir / "errored.jsonl" trace.write_text( - '{"type":"system","subtype":"init","model":"claude-opus-5-5"}\n' + '{"type":"system","subtype":"init","model":"claude-opus-5-5","tools":["Skill"]}\n' '{"type":"result","subtype":"error_during_execution","is_error":true}\n', encoding="utf-8", ) @@ -402,7 +402,7 @@ def test_a_zero_turn_run_with_an_error_is_reported_once(self): def test_a_zero_turn_run_whose_trace_ends_in_an_error_is_reported_once(self): trace = self.dir / "errored.jsonl" trace.write_text( - '{"type":"system","subtype":"init","model":"claude-opus-5-5"}\n' + '{"type":"system","subtype":"init","model":"claude-opus-5-5","tools":["Skill"]}\n' '{"type":"result","subtype":"error_during_execution","is_error":true}\n', encoding="utf-8", ) @@ -418,7 +418,7 @@ def test_a_zero_turn_run_whose_trace_ends_in_an_error_is_reported_once(self): def test_a_zero_turn_run_whose_trace_holds_a_forked_session_passes(self): trace = self.dir / "forked.jsonl" trace.write_text( - '{"type":"system","subtype":"init","model":"claude-opus-5-5"}\n' + '{"type":"system","subtype":"init","model":"claude-opus-5-5","tools":["Skill"]}\n' '{"type":"assistant","message":{"content":[{"type":"text","text":"done"}]}}\n' '{"type":"result","subtype":"success","is_error":false,"num_turns":0}\n', encoding="utf-8", @@ -444,7 +444,7 @@ def test_a_mention_of_the_sandbox_failure_outside_a_failed_bash_result_passes(se text = "Sandbox is required but failed to initialize" trace = self.dir / "mentions.jsonl" trace.write_text( - '{"type":"system","subtype":"init","model":"claude-opus-5-5"}\n' + '{"type":"system","subtype":"init","model":"claude-opus-5-5","tools":["Skill"]}\n' + json.dumps( {"type": "assistant", "message": {"content": [ {"type": "text", "text": "The gate looks for '%s'." % text}, @@ -479,7 +479,7 @@ def bash_trace(self, name, results): for i, (err, text) in enumerate(results)] trace = self.dir / name trace.write_text( - '{"type":"system","subtype":"init","model":"claude-opus-5-5"}\n' + '{"type":"system","subtype":"init","model":"claude-opus-5-5","tools":["Skill"]}\n' + json.dumps({"type": "assistant", "message": {"content": uses}}) + "\n" + json.dumps({"type": "user", "message": {"content": outs}}) + "\n" + '{"type":"result","subtype":"success","is_error":false,"num_turns":4}\n', @@ -530,7 +530,7 @@ def test_one_failed_bash_call_or_a_mixed_run_passes(self): def test_a_trace_showing_the_sandbox_failing_to_start_is_invalid(self): trace = self.dir / "no-sandbox.jsonl" trace.write_text( - '{"type":"system","subtype":"init","model":"claude-opus-5-5"}\n' + '{"type":"system","subtype":"init","model":"claude-opus-5-5","tools":["Skill"]}\n' '{"type":"assistant","message":{"content":[{"type":"tool_use","id":"t1",' '"name":"Bash","input":{"command":"ls"}}]}}\n' '{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1",' @@ -560,7 +560,7 @@ def test_a_trace_showing_the_sandbox_failing_to_start_is_invalid(self): def test_a_denial_seen_only_as_a_tool_result_is_counted(self): trace = self.dir / "denied.jsonl" trace.write_text( - '{"type":"system","subtype":"init","model":"claude-opus-5-5"}\n' + '{"type":"system","subtype":"init","model":"claude-opus-5-5","tools":["Skill"]}\n' '{"type":"assistant","message":{"content":[{"type":"tool_use","id":"t1","name":"Grep","input":{"pattern":"x"}}]}}\n' '{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1","is_error":true,' '"content":[{"type":"text","text":"Path is denied by your permission settings."}]}]}}\n', @@ -621,7 +621,7 @@ def call(use_id, tool): lines.append({"type": "result", "is_error": False, "permission_denials": entries}) trace = self.dir / "denied-probe.jsonl" trace.write_text( - '{"type":"system","subtype":"init","model":"claude-opus-5-5"}\n' + '{"type":"system","subtype":"init","model":"claude-opus-5-5","tools":["Skill"]}\n' + "".join(json.dumps(line) + "\n" for line in lines), encoding="utf-8", ) @@ -699,7 +699,7 @@ def test_a_non_probe_denied_a_search_inside_the_plugin_is_invalid(self): def test_a_denial_names_the_path_it_was_aimed_at(self): trace = self.dir / "denied-paths.jsonl" trace.write_text( - '{"type":"system","subtype":"init","model":"claude-opus-5-5"}\n' + '{"type":"system","subtype":"init","model":"claude-opus-5-5","tools":["Skill"]}\n' '{"type":"result","is_error":false,"permission_denials":[' '{"tool_name":"Grep","tool_use_id":"g1","tool_input":' '{"pattern":"GRADER_TYPES","path":"/p/validate-cases.py"}},' @@ -749,7 +749,7 @@ def denied_second_run( tool_input["path"] = path trace = self.dir / "denied-root.jsonl" trace.write_text( - '{"type":"system","subtype":"init","model":"claude-opus-5-5"}\n' + '{"type":"system","subtype":"init","model":"claude-opus-5-5","tools":["Skill"]}\n' '{"type":"result","is_error":false,"permission_denials":[' '{"tool_name":"%s","tool_use_id":"g1","tool_input":%s}]}\n' % (tool, json.dumps(tool_input)), @@ -982,7 +982,7 @@ def denied_glob(self, with_input, without_input, tool="Glob"): continue trace = self.dir / ("denied-%s.jsonl" % arm) trace.write_text( - '{"type":"system","subtype":"init","model":"claude-opus-5-5"}\n' + '{"type":"system","subtype":"init","model":"claude-opus-5-5","tools":["Skill"]}\n' '{"type":"result","is_error":false,"permission_denials":[' '{"tool_name":"%s","tool_use_id":"g1","tool_input":%s}]}\n' % (tool, json.dumps(tool_input)), @@ -1042,7 +1042,7 @@ def test_a_skill_denial_in_every_run_of_both_arms_is_a_fail(self): def test_skill_text_quoting_the_denial_is_not_a_denial(self): trace = self.dir / "quoted.jsonl" trace.write_text( - '{"type":"system","subtype":"init","model":"claude-opus-5-5"}\n' + '{"type":"system","subtype":"init","model":"claude-opus-5-5","tools":["Skill"]}\n' '{"type":"user","message":{"content":[{"type":"text","text":"a Read is refused with' ' File is in a directory that is denied by your permission settings"}]}}\n', encoding="utf-8", @@ -1181,6 +1181,63 @@ def test_a_probe_run_with_only_skill_passes_probe_tools(self): self.assertVerdict(proc, "VALID", 0) self.assertIn("check probe tools: PASS", proc.stdout) + def probe_unread(self, arms, missing_trace): + """Case noise-before-gain tagged invocation-probe, whose runs in arms have + either no trace file or an init line with no tools list.""" + result = self.clean() + case = result["cases"][2] + (self.dir / "suite" / case["dir"] / "prompt.md").write_text( + "---\ndescription: d\ntags: [invocation-probe, train]\n---\n\nQ\n", + encoding="utf-8", + ) + trace = self.dir / "no-tools.jsonl" + if not missing_trace: + trace.write_text( + json.dumps({"type": "system", "subtype": "init", + "model": "claude-opus-5-5"}) + + "\n" + + json.dumps({"type": "result", "subtype": "success", "is_error": False}) + + "\n", + encoding="utf-8", + ) + path = self.write(result) + data = load(path) + for arm in arms: + data["cases"][2]["arms"][arm][0]["tracePath"] = str(trace) + path.write_text(json.dumps(data), encoding="utf-8") + return self.run_script(path, "--runs", "1"), trace + + def test_a_probe_run_with_no_trace_warns_and_names_it(self): + proc, trace = self.probe_unread(["with"], missing_trace=True) + self.assertIn( + "check probe tools: WARN (1 probe run with no readable init tools list)", + proc.stdout, + ) + self.assertIn( + " case noise-before-gain, with-arm run 1: no init tools list (%s)" % trace, + proc.stdout, + ) + self.assertIn("permission denials, probe tools and the trace model are", proc.stdout) + + def test_probe_runs_with_no_init_tools_list_are_unchecked_and_invalid(self): + proc, _ = self.probe_unread(["with", "without"], missing_trace=False) + self.assertVerdict(proc, "INVALID", 1) + self.assertIn( + "check probe tools: UNCHECKED (no invocation-probe run's init tools list" + " could be read)", + proc.stdout, + ) + self.assertIn( + "probe tools unchecked in 2 runs with no readable init tools list", + proc.stdout.splitlines()[-1], + ) + + def test_one_probe_run_with_no_init_tools_list_warns(self): + proc, _ = self.probe_unread(["with"], missing_trace=False) + self.assertVerdict(proc, "VALID", 0) + self.assertIn("check probe tools: WARN (1 probe run with no readable", proc.stdout) + self.assertIn("probe tools unread in 1 run", proc.stdout.splitlines()[-1]) + def test_a_case_outside_the_probes_may_search(self): proc = self.probe_with_tools( ["Task", "Glob", "Grep", "Read", "Skill"], tags="tags: [knowledge]" @@ -1206,7 +1263,7 @@ def capped_probe( call = {"skill": "evals:methodology"} if tool == "Skill" else {"file_path": "x"} trace = self.dir / "capped.jsonl" trace.write_text( - '{"type":"system","subtype":"init","model":"claude-opus-5-5"}\n' + '{"type":"system","subtype":"init","model":"claude-opus-5-5","tools":["Skill"]}\n' + json.dumps({"type": "assistant", "message": {"content": [ {"type": "tool_use", "id": "c1", "name": tool, "input": call}]}}) + "\n"