Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions plugins/dojo-verify/hooks/stop_evidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
"""dojo-verify: Stop hook. A reply that claims success needs a passing check behind it.

When the assistant's final message claims success (tests pass, verified, fixed, deployed ...) and no recognized
check ran, and came back clean, after the last change made in this session, this hook blocks the stop once (mode
block), or just says so (mode warn). A session with no change in it is not judged. It reads the newest part of the
transcript, never runs anything, and fails open on every error.
check ran, and came back clean, after the last change made in this session, this hook says so (mode warn, the
default), or blocks the stop once (mode block, opt-in). A session with no change in it is not judged. It reads the
newest part of the transcript, never runs anything, and fails open on every error.

Stdlib only, Python 3.9. The claim and check patterns live in patterns.json (the mod carries a copy in
patterns.ts; a test keeps them equal).
Expand Down
3 changes: 2 additions & 1 deletion plugins/dojo-verify/tests/_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,8 @@ def payload(self, **kw):
return p

def env(self, **extra):
# Most tests exercise the block path, so they set it; the default (warn) is tested with the option absent.
# The detection tests read a firing as a block, so they run in mode block; the default (warn) has its own
# tests, which drop this with CLAUDE_PLUGIN_OPTION_MODE=None.
env = {"PATH": "/usr/bin:/bin", "HOME": self.home, "CLAUDE_PLUGIN_DATA": self.state, "TMPDIR": self.tmpdir,
"CLAUDE_PLUGIN_OPTION_MODE": "block"}
for k, v in extra.items():
Expand Down
21 changes: 15 additions & 6 deletions plugins/dojo-verify/tests/test_stop_evidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,17 @@ def claim_turn(self, final="Fixed. The tests pass.", middle=None):


class TestKillSwitchesAndModes(Base):
def test_default_mode_warns_and_does_not_block(self):
def test_default_mode_warns_and_never_blocks(self):
r = self.run_hook(self.claim_turn(), env=self.sb.env(CLAUDE_PLUGIN_OPTION_MODE=None))
self.assertEqual(r.code, 0)
self.assertEqual(r.err, "")
data = r.json()
self.assertNotIn("decision", data)
self.assertTrue(data["systemMessage"].startswith("dojo-verify:"))
self.assertIn("set mode to block", data["systemMessage"])

def test_block_mode_blocks_with_the_documented_shape(self):
r = self.run_hook(self.claim_turn())
r = self.run_hook(self.claim_turn(), env=self.sb.env(CLAUDE_PLUGIN_OPTION_MODE="block"))
self.assertEqual(r.code, 0)
self.assertEqual(r.err, "")
data = r.json()
Expand Down Expand Up @@ -86,7 +88,7 @@ def test_off_mode_is_silent(self):

def test_unknown_mode_falls_back_to_warn(self):
r = self.run_hook(self.claim_turn(), env=self.sb.env(CLAUDE_PLUGIN_OPTION_MODE="garbage"))
self.assertNotIn("decision", r.json())
self.assertFalse(r.blocked)
self.assertIn("systemMessage", r.json())

def test_stop_hook_active_never_blocks(self):
Expand Down Expand Up @@ -1098,13 +1100,20 @@ def test_an_unwritable_state_dir_still_blocks(self):


class TestPortability(Base):
def test_block_and_fail_open_under_a_bare_environment(self):
def test_warn_block_and_fail_open_under_a_bare_environment(self):
# The same invocation the engine could use on a machine with nothing but the system tools.
import subprocess
self.sb.write(self.claim_turn())
env_cmd = ["/usr/bin/env", "-i", "PATH=/usr/bin:/bin", "HOME=" + self.sb.home,
"TMPDIR=" + self.sb.tmpdir, "CLAUDE_PLUGIN_OPTION_MODE=block", h.SYSTEM_PYTHON, h.HOOK]
blocked = subprocess.run(env_cmd, input=json.dumps(self.sb.payload()).encode(), stdout=subprocess.PIPE,
"TMPDIR=" + self.sb.tmpdir, h.SYSTEM_PYTHON, h.HOOK]
warned = subprocess.run(env_cmd, input=json.dumps(self.sb.payload()).encode(), stdout=subprocess.PIPE,
stderr=subprocess.PIPE, timeout=20, cwd=self.sb.project)
self.assertEqual(warned.returncode, 0)
self.assertNotIn("decision", json.loads(warned.stdout))
self.assertIn("systemMessage", json.loads(warned.stdout))
self.assertEqual(warned.stderr, b"")
block_cmd = env_cmd[:4] + ["CLAUDE_PLUGIN_OPTION_MODE=block"] + env_cmd[4:]
blocked = subprocess.run(block_cmd, input=json.dumps(self.sb.payload()).encode(), stdout=subprocess.PIPE,
stderr=subprocess.PIPE, timeout=20, cwd=self.sb.project)
self.assertEqual(blocked.returncode, 0)
self.assertEqual(json.loads(blocked.stdout)["decision"], "block")
Expand Down
Loading