From 0fa5a76d7967f95cd2449dc1930f1766d8b9fef2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 16:02:52 +0000 Subject: [PATCH] feat(dojo-verify): default to warn, per the pre-registered eval and Cruz's ruling The with/without eval (evals/PREREG-2026-10-17-evidence.md) detected no benefit from blocking and crossed the control cost bar, so its rule says warn. Cruz chose warn on 2026-10-18. mode now defaults to warn and an unknown value falls back to warn; block stays one setting away. The ruling is recorded below the marker, so the frozen digest is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014pUc6p2gu3ur7FYwbX6sBc --- ACCURACY.md | 2 +- CHANGELOG.md | 4 ++-- plugins/dojo-verify/.claude-plugin/plugin.json | 4 ++-- plugins/dojo-verify/README.md | 6 ++++-- .../evals/PREREG-2026-10-17-evidence.md | 7 +++++++ plugins/dojo-verify/hooks/stop_evidence.py | 2 +- plugins/dojo-verify/tests/_helpers.py | 4 +++- plugins/dojo-verify/tests/test_static.py | 2 +- plugins/dojo-verify/tests/test_stop_evidence.py | 16 ++++++++++++---- 9 files changed, 33 insertions(+), 14 deletions(-) diff --git a/ACCURACY.md b/ACCURACY.md index 883738c..a8e01aa 100644 --- a/ACCURACY.md +++ b/ACCURACY.md @@ -139,7 +139,7 @@ measured the rule's effect. "Field" figures are from section 2; "fixtures" from | 2. Scout before you touch | Claude Code itself (Edit refuses a file not read in the session); the `scout-first` skill and scout agent advise | Not a hook of ours, not measured | No eval run; scaffolds in dojo-protocol | | 3. Keep context small | dojo-gates big-read (denies a whole read over 256 KB) | Field 7 of 7 right (95% CI 65–100%); fixtures 100% / 100% | No eval run; scaffolds in dojo-gates and dojo-protocol | | 4. Contract before fan-out | dojo-flow's `dag-plan` refuses tracks that share a file or claim the contract file (a script run before a build, not a hook); the `contract` skill advises | Not measured | No eval run; scaffold `build-refuses-overlap` | -| 5. Done means verified | dojo-verify Stop hook (blocks by default) and its evidence skill; dojo-tally flags agent reports that cite a check they never ran | Field 3% before the fixes (7 of 215); held-out 8% after (4 of 48, CI 3–20%), every real catch kept; fixtures 62% / 100% | **Measured once:** pre-registered with/without eval, 30 runs per arm. No benefit detected (unbacked claims 5/30 without, 4/30 with; difference CI −16 to +22 points), extra cost on controls (×1.29). The hook missed all four unbacked claims in the with arm | +| 5. Done means verified | dojo-verify Stop hook (warns by default since the eval; blocks when set to `block`) and its evidence skill; dojo-tally flags agent reports that cite a check they never ran | Field 3% before the fixes (7 of 215); held-out 8% after (4 of 48, CI 3–20%), every real catch kept; fixtures 62% / 100% | **Measured once:** pre-registered with/without eval, 30 runs per arm. No benefit detected (unbacked claims 5/30 without, 4/30 with; difference CI −16 to +22 points), extra cost on controls (×1.29). The hook missed all four unbacked claims in the with arm | | 6. Count returns, not dispatches | dojo-tally mod (dispatched, returned, failed, silent, unchecked); dojo-flow's scorecard | Not measured | No eval run; scaffold `scorecard-counts-dead-agent` | | 7. Debug by disproof | Nothing at runtime; skill guidance only | n/a | No eval | | 8. Config before code | dojo-gates config-first (adds context after a boundary failure) | Field 30% before (22 of 73); held-out after: auth 6 of 6, path 3 of 6, cmd no firings; fixtures path 86% / 75%, conn 67% / 100%, auth and cmd 100% / 100% | No eval | diff --git a/CHANGELOG.md b/CHANGELOG.md index 32c9ee8..132bfc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,8 @@ method, the rubric and the tables. It started from five misfires reported from o hooks still make: they count, but don't fail the run. - **A pre-registered with/without eval of dojo-verify's Stop block** — `plugins/dojo-verify/evals/PREREG-2026-10-17-evidence.md`, frozen before any run, with eight held-out cases. Its - result, below the marker: no benefit detected, and its rule says the default should be `warn`. The default stays - `block` until that is decided. + result, below the marker: no benefit detected, and its rule says the default should be `warn`. Following that rule + (Cruz's ruling, 2026-10-18), the default is now `warn`; `block` stays one setting away. - **A runtime health report** — `scripts/health_report.py` replays local Claude Code transcripts through the suite's own detector code and reports, per rule, what it counted and out of how many, with each finding's evidence and each detector's measured accuracy beside it. The same input gives byte-identical output. diff --git a/plugins/dojo-verify/.claude-plugin/plugin.json b/plugins/dojo-verify/.claude-plugin/plugin.json index 3027f13..a080342 100644 --- a/plugins/dojo-verify/.claude-plugin/plugin.json +++ b/plugins/dojo-verify/.claude-plugin/plugin.json @@ -14,8 +14,8 @@ "mode": { "type": "string", "title": "Mode", - "description": "What the Stop hook does when the final message claims success and no check has passed since the last change: block (default), warn, or off. The mod also goes inert when this is off.", - "default": "block", + "description": "What the Stop hook does when the final message claims success and no check has passed since the last change: warn (default), block, or off. The mod also goes inert when this is off.", + "default": "warn", "options": ["block", "warn", "off"] }, "show_status": { diff --git a/plugins/dojo-verify/README.md b/plugins/dojo-verify/README.md index 4b304db..aa90de9 100644 --- a/plugins/dojo-verify/README.md +++ b/plugins/dojo-verify/README.md @@ -81,10 +81,12 @@ Set `mode` in the plugin's settings (`userConfig`): | Mode | What happens | | --- | --- | -| `block` (default) | Block the stop once with `{"decision":"block","reason":"dojo-verify: ..."}`. | -| `warn` | Show a system message and don't block. | +| `warn` (default) | Show a system message and don't block. | +| `block` | Block the stop once with `{"decision":"block","reason":"dojo-verify: ..."}`. | | `off` | Silent. The mod goes inert too. | +The default is `warn` because the pre-registered with/without eval (`evals/PREREG-2026-10-17-evidence.md`) found no benefit from blocking and extra cost on the control tasks. Set `block` to have the hook stop a reply until it runs a check or says plainly that it is unverified. + Environment switches, checked in this order: `DOJO_OFF=1` (the whole suite), then `DOJO_VERIFY_OFF=1` (this plugin). `1`, `true` and `yes` turn it off; `0` does not. The mod also reads both. Two more settings affect the mod only: `show_status` and `toast_unverified` (both default on). diff --git a/plugins/dojo-verify/evals/PREREG-2026-10-17-evidence.md b/plugins/dojo-verify/evals/PREREG-2026-10-17-evidence.md index f470ae0..2b0ae60 100644 --- a/plugins/dojo-verify/evals/PREREG-2026-10-17-evidence.md +++ b/plugins/dojo-verify/evals/PREREG-2026-10-17-evidence.md @@ -238,3 +238,10 @@ what the run used. The original commits stay fetchable with `git fetch origin pu | `d926d5b` | `dbcf0af` | amendment 1 | | `2d91ddd` | `b995416` | amendment 2; the main run used this tree | | `a53ba13` | `205a915` | these results | + +## Operator ruling after the run (2026-10-18) + +Cruz chose `warn`, as the rule says. dojo-verify's `mode` now defaults to `warn`, and an unknown value falls back to +`warn`; `block` stays available as a setting. Noted when the choice was put: the control cost ratio (1.29) is mostly +the judge's cost, since the agent's own mean cost on the controls was $0.054 per run with the plugin and $0.053 +without. This doesn't change the ruling, because the benefit test fails on its own. diff --git a/plugins/dojo-verify/hooks/stop_evidence.py b/plugins/dojo-verify/hooks/stop_evidence.py index 45647ef..eec5768 100644 --- a/plugins/dojo-verify/hooks/stop_evidence.py +++ b/plugins/dojo-verify/hooks/stop_evidence.py @@ -1442,7 +1442,7 @@ def killed(): def get_mode(): raw = os.environ.get("CLAUDE_PLUGIN_OPTION_MODE") mode = raw.strip().lower() if isinstance(raw, str) else "" - return mode if mode in ("block", "warn", "off") else "block" + return mode if mode in ("block", "warn", "off") else "warn" def read_stdin(): diff --git a/plugins/dojo-verify/tests/_helpers.py b/plugins/dojo-verify/tests/_helpers.py index 978a0f0..8928b22 100644 --- a/plugins/dojo-verify/tests/_helpers.py +++ b/plugins/dojo-verify/tests/_helpers.py @@ -158,7 +158,9 @@ def payload(self, **kw): return p def env(self, **extra): - env = {"PATH": "/usr/bin:/bin", "HOME": self.home, "CLAUDE_PLUGIN_DATA": self.state, "TMPDIR": self.tmpdir} + # Most tests exercise the block path, so they set it; the default (warn) is tested with the option absent. + env = {"PATH": "/usr/bin:/bin", "HOME": self.home, "CLAUDE_PLUGIN_DATA": self.state, "TMPDIR": self.tmpdir, + "CLAUDE_PLUGIN_OPTION_MODE": "block"} for k, v in extra.items(): if v is None: env.pop(k, None) diff --git a/plugins/dojo-verify/tests/test_static.py b/plugins/dojo-verify/tests/test_static.py index 795279c..35fe3f0 100644 --- a/plugins/dojo-verify/tests/test_static.py +++ b/plugins/dojo-verify/tests/test_static.py @@ -58,7 +58,7 @@ def test_every_user_config_field_has_a_default_and_a_valid_type(self): self.assertIn("default", field, key) self.assertIn(field["type"], ("string", "number", "boolean", "directory", "file"), key) self.assertTrue(field["title"] and field["description"], key) - self.assertEqual(cfg["mode"]["default"], "block") + self.assertEqual(cfg["mode"]["default"], "warn") self.assertEqual(cfg["mode"]["options"], ["block", "warn", "off"]) self.assertIn(cfg["mode"]["default"], cfg["mode"]["options"]) self.assertIs(cfg["show_status"]["default"], True) diff --git a/plugins/dojo-verify/tests/test_stop_evidence.py b/plugins/dojo-verify/tests/test_stop_evidence.py index df7eca3..11a0b44 100644 --- a/plugins/dojo-verify/tests/test_stop_evidence.py +++ b/plugins/dojo-verify/tests/test_stop_evidence.py @@ -25,7 +25,14 @@ def claim_turn(self, final="Fixed. The tests pass.", middle=None): class TestKillSwitchesAndModes(Base): - def test_default_mode_blocks_with_the_documented_shape(self): + def test_default_mode_warns_and_does_not_block(self): + r = self.run_hook(self.claim_turn(), env=self.sb.env(CLAUDE_PLUGIN_OPTION_MODE=None)) + self.assertEqual(r.code, 0) + data = r.json() + self.assertNotIn("decision", data) + self.assertTrue(data["systemMessage"].startswith("dojo-verify:")) + + def test_block_mode_blocks_with_the_documented_shape(self): r = self.run_hook(self.claim_turn()) self.assertEqual(r.code, 0) self.assertEqual(r.err, "") @@ -77,9 +84,10 @@ def test_off_mode_is_silent(self): r = self.run_hook(self.claim_turn(), env=self.sb.env(CLAUDE_PLUGIN_OPTION_MODE="off")) self.assertTrue(r.silent) - def test_unknown_mode_falls_back_to_block(self): + def test_unknown_mode_falls_back_to_warn(self): r = self.run_hook(self.claim_turn(), env=self.sb.env(CLAUDE_PLUGIN_OPTION_MODE="garbage")) - self.assertTrue(r.blocked) + self.assertNotIn("decision", r.json()) + self.assertIn("systemMessage", r.json()) def test_stop_hook_active_never_blocks(self): r = self.run_hook(self.claim_turn(), stop_hook_active=True) @@ -1095,7 +1103,7 @@ def test_block_and_fail_open_under_a_bare_environment(self): import subprocess self.sb.write(self.claim_turn()) env_cmd = ["/usr/bin/env", "-i", "PATH=/usr/bin:/bin", "HOME=" + self.sb.home, - "TMPDIR=" + self.sb.tmpdir, h.SYSTEM_PYTHON, h.HOOK] + "TMPDIR=" + self.sb.tmpdir, "CLAUDE_PLUGIN_OPTION_MODE=block", h.SYSTEM_PYTHON, h.HOOK] blocked = subprocess.run(env_cmd, input=json.dumps(self.sb.payload()).encode(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20, cwd=self.sb.project) self.assertEqual(blocked.returncode, 0)