From 1bfb133e9a0b5bead8e696f309bbbbe668eb9088 Mon Sep 17 00:00:00 2001 From: vk Date: Tue, 1 Sep 2026 19:59:56 +0530 Subject: [PATCH 1/3] Provision python3 in the install-alpine job, the fourth container the census fix missed The v1.61.0 tag was destroyed a fourth time, and this time every part of the machinery was right. install-alpine runs the gate inside an alpine job container whose apk line predates check 63. The check skipped on "python3 not on PATH"; the lane's skip audit refused the undeclared skip, correctly; the job concluded failure for the exact tagged commit; require-checks-green read a live conclusion, correctly; cleanup deleted the tag, correctly. The defect was mine: the D2 fix provisioned python3 in tests/container-matrix.sh's three bootstraps and missed the one alpine container that gates releases. The lane's own header states the rule this follows: install the ONE package a needed behaviour requires and name the behaviour, rather than widening the approved-skip list. The named behaviour is check 63's census self-test, stdlib Python. Not added to the skip allowlist because the skip is not an environment fact of this lane -- the lane exists to prove the gate runs on Alpine, and a gate check that never runs on Alpine is not proven there. Co-Authored-By: Claude Fable 5 --- .github/workflows/verify.yml | 8 +- claude/hooks/inject-session-context.sh | 37 +++-- claude/hooks/skill-mandate.sh | 54 +++++++- tests/mandate-cases.sh | 19 ++- tests/test-breadth-mandate.sh | 47 +++++++ tests/test-session-context-mandate.sh | 185 +++++++++++++++++++++++++ 6 files changed, 332 insertions(+), 18 deletions(-) create mode 100755 tests/test-session-context-mandate.sh diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 191732b..77e6cba 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -784,8 +784,14 @@ jobs: # a digest already falls back to sha256sum, and that fallback is now exercised rather than # assumed. If a future change really does need a GNU tool, install that ONE package and # say which behaviour required it -- do not restore the blanket userland swap. + # python3 is here for one named behaviour: check 63 runs tests/transcript-census.sh + # --self-test, whose engine is stdlib Python. Without it the check skips, and this lane's + # own skip audit -- correctly -- refuses a skip it never declared. That exact sequence + # destroyed the v1.61.0 tag on 2026-09-01 (the fourth destruction): the census fix + # provisioned python3 in tests/container-matrix.sh's three images and missed this fourth + # container, the one wired into release gating. Same defect, fourth copy of the fix. - name: Tools a slim image does not ship (BusyBox kept, deliberately) - run: apk add --no-cache bash git jq curl nodejs npm + run: apk add --no-cache bash git jq curl nodejs npm python3 # check 29 ("shellcheck clean") skips without a validator on PATH, and a check that skips # on a lane specifically kept around to catch BusyBox-vs-GNU differences is a green diff --git a/claude/hooks/inject-session-context.sh b/claude/hooks/inject-session-context.sh index 95ee192..14e9436 100755 --- a/claude/hooks/inject-session-context.sh +++ b/claude/hooks/inject-session-context.sh @@ -130,15 +130,19 @@ if [ "$event" = "UserPromptSubmit" ]; then if [ "$_n" -ge "${VSTACK_GRILL_CHARS:-320}" ] \ || { [ -z "$_seen" ] && [ "$_n" -ge 120 ]; }; then grill=' -GRILL: run the grill-me skill when no skill matches this situation more specifically. A -situation-matched skill outranks it. grill-me is for a request whose shape is still undecided.' +GRILL: run the grill-me skill when no more specific skill matches.' fi fi # Delegation-mandate strike count, re-pinned every prompt instead of stated once at - # SessionStart and then never again. skill-mandate.sh (Stop) is the only writer of these two - # small counter files; this only reads them -- a cat of two tiny files under $TMPDIR, not a - # transcript parse, so it costs nothing like the Stop hook's own evaluation does. It is also - # why this can run every single prompt with no latency argument to make: there is no scan here. + # SessionStart and then never again. skill-mandate.sh (Stop) is the only writer of the + # counter files read here, and it writes ONE FILE PER MANDATE (.unslop/.typescript/ + # .proveitworks for the skill family, .delegate-breadth/-naming/-swarm/-serial for the + # delegation family) -- there is no combined per-family file. This block took the max of two + # filenames the Stop hook had stopped writing when its counters split per-mandate, so both + # reads were always 0 and the MANDATE line below was dead code in every real session; now it + # takes the max across each family's real files. Still just cats of tiny files under + # $TMPDIR, not a transcript parse, so it can run every single prompt with no latency + # argument to make: there is no scan here. # # Two independent counters because skill-mandate.sh's own delegation family (breadth + # agent-naming) no longer shares the skill mandates' (unslop/typescript/prove-it-works) @@ -157,17 +161,26 @@ situation-matched skill outranks it. grill-me is for a request whose shape is st _msid=$(printf '%s' "$in" | "$JQ" -r '.session_id // empty' 2>/dev/null) [ -n "$_msid" ] || _msid="pid$PPID" _mcnt_file="${TMPDIR:-/tmp}/vstack-mandate-$_msid" - _mcnt=$(cat "$_mcnt_file" 2>/dev/null || echo 0) - case "$_mcnt" in ''|*[!0-9]*) _mcnt=0 ;; esac - _mdcnt=$(cat "$_mcnt_file.delegate" 2>/dev/null || echo 0) - case "$_mdcnt" in ''|*[!0-9]*) _mdcnt=0 ;; esac + _mcnt=0 + for _sfx in unslop typescript proveitworks; do + _v=$(cat "$_mcnt_file.$_sfx" 2>/dev/null || echo 0) + case "$_v" in ''|*[!0-9]*) _v=0 ;; esac + [ "$_v" -gt "$_mcnt" ] && _mcnt=$_v + done + _mdcnt=0 + for _sfx in delegate-breadth delegate-naming delegate-swarm delegate-serial; do + _v=$(cat "$_mcnt_file.$_sfx" 2>/dev/null || echo 0) + case "$_v" in ''|*[!0-9]*) _v=0 ;; esac + [ "$_v" -gt "$_mdcnt" ] && _mdcnt=$_v + done if [ "$_mcnt" -ge 1 ] || [ "$_mdcnt" -ge 1 ]; then mandate=" MANDATE skill=$_mcnt/2 delegate=$_mdcnt/2: dispatch + name a call sign now." fi fi - emit "$event" 'TOKENS: grep/ranges, not whole files; batch independent tool calls in ONE message. -DELEGATE: mechanical -> worker/explorer, judgment -> sonnet agents. ACT, do not ask. Skills fire on the situation — call the Skill tool.'"$grill$mandate" + emit "$event" 'TOKENS: grep/ranges, not whole files; summarize, never dump. +DELEGATE: mechanical -> worker/explorer, judgment -> sonnet agents. ACT, do not ask. Skills fire on the situation -- call the Skill tool. +FANOUT: work that splits -> Skill swarm first, ALL Agent calls in ONE message.'"$grill$mandate" exit 0 fi diff --git a/claude/hooks/skill-mandate.sh b/claude/hooks/skill-mandate.sh index eb433ae..1af862e 100755 --- a/claude/hooks/skill-mandate.sh +++ b/claude/hooks/skill-mandate.sh @@ -202,6 +202,7 @@ _read_cnt "$cnt_file.proveitworks"; cnt_proveitworks=$_rc _read_wcnt "$cnt_file.delegate-breadth" "$cnt_file.delegate-breadth-ts" "$now_d" "$DELEGATE_RESET_SECS"; cnt_breadth=$_rc _read_wcnt "$cnt_file.delegate-naming" "$cnt_file.delegate-naming-ts" "$now_d" "$DELEGATE_RESET_SECS"; cnt_naming=$_rc _read_wcnt "$cnt_file.delegate-swarm" "$cnt_file.delegate-swarm-ts" "$now_d" "$DELEGATE_RESET_SECS"; cnt_swarm=$_rc +_read_wcnt "$cnt_file.delegate-serial" "$cnt_file.delegate-serial-ts" "$now_d" "$DELEGATE_RESET_SECS"; cnt_serial=$_rc eval_unslop=1; [ "$cnt_unslop" -ge 2 ] && eval_unslop=0 eval_typescript=1; [ "$cnt_typescript" -ge 2 ] && eval_typescript=0 @@ -209,6 +210,7 @@ eval_proveitworks=1; [ "$cnt_proveitworks" -ge 2 ] && eval_proveitworks=0 eval_breadth=1; [ "$cnt_breadth" -ge 2 ] && eval_breadth=0 eval_naming=1; [ "$cnt_naming" -ge 2 ] && eval_naming=0 eval_swarm=1; [ "$cnt_swarm" -ge 2 ] && eval_swarm=0 +eval_serial=1; [ "$cnt_serial" -ge 2 ] && eval_serial=0 # Second, SHORTER gate: how often to bother re-scanning at all once every skill-family mandate is # already latched, independent of whether delegation has struck. Family-exhausted alone is not a @@ -236,7 +238,7 @@ fi skill_eval=1 [ "$eval_unslop" = 0 ] && [ "$eval_typescript" = 0 ] && [ "$eval_proveitworks" = 0 ] && skill_eval=0 deleg_eval=1 -[ "$eval_breadth" = 0 ] && [ "$eval_naming" = 0 ] && [ "$eval_swarm" = 0 ] && deleg_eval=0 +[ "$eval_breadth" = 0 ] && [ "$eval_naming" = 0 ] && [ "$eval_swarm" = 0 ] && [ "$eval_serial" = 0 ] && deleg_eval=0 # Combined latch: skip the transcript-driven evaluation entirely only when NEITHER family can # still act on it. skill_eval=1 alone is enough to keep paying for the scan every Stop, unchanged @@ -273,6 +275,7 @@ hit_proveitworks=0 hit_breadth=0 hit_naming=0 hit_swarm=0 +hit_serial=0 # We are doing a full scan this Stop for at least one reason (skill_eval=1 or deleg_eval=1 -- # the latch above already exited otherwise). Record it now, unconditionally, so the cooldown @@ -607,6 +610,14 @@ case "$task_count" in ''|*[!0-9]*) task_count=0 ;; esac # there is no identity to merge on, so each such line is its own singleton batch. That is what # makes "two Task calls in two separate lines, neither carrying an id" read as two batches of # one, not one batch of two: the exact serial-loop shape this fix exists to catch. +# One pass, two numbers: " ". serial_tail is the count of +# Task/Agent dispatches AFTER the last 2+-in-one-message batch (all of them when no batch ever +# ran). fanout_batches answers "did a real batch ever happen"; serial_tail answers the question +# it structurally cannot: "what has happened SINCE". A whole-transcript batch count amnesties +# every later serial dispatch -- measured, not hypothesized: session 3ce9f899 batched 3 times +# early and then made 25 one-at-a-time dispatches with the breadth mandate silenced for good, +# and 8959d943 did the same behind 2 batches of 2. The serial-tail mandate below reads only the +# suffix, so history cannot pay for the present. fanout_calc=$( "$JQ" -sr ' ( [ .[] | select(.type=="assistant") | { id: (.message.id // null), @@ -624,10 +635,16 @@ fanout_calc=$( "$JQ" -sr ' ) ) as $folded | ( $folded.runs + [{id: $folded.cid, n: $folded.cn}] ) as $all_runs - | ( $all_runs | map(select(.n >= 2)) | length ) + | ( $all_runs | map(.n) ) as $ns + | ( [ range(0; $ns|length) | select($ns[.] >= 2) ] | last ) as $lastb + | ( [ $ns[] | select(. >= 2) ] | length ) as $batches + | ( (if $lastb == null then $ns else $ns[($lastb+1):] end) | add // 0 ) as $tail + | "\($batches) \($tail)" ' "$tr_" 2>/dev/null | tail -n 1 ) -case "$fanout_calc" in ''|*[!0-9]*) fanout_calc=0 ;; esac -fanout_batches="$fanout_calc" +fanout_batches=${fanout_calc%% *} +serial_tail=${fanout_calc##* } +case "$fanout_batches" in ''|*[!0-9]*) fanout_batches=0 ;; esac +case "$serial_tail" in ''|*[!0-9]*) serial_tail=0 ;; esac # task_fail_count: of those same Task/Agent dispatches, how many resolved with is_error==true on # their tool_result. This is the field the delegation-drift ledger was missing entirely -- it @@ -734,6 +751,26 @@ if [ "$eval_breadth" = 1 ] && [ "$dir_count" -ge 3 ] && [ "$ext_count" -ge 2 ] & hit_breadth=1 fi +# Serial dispatch tail: the shape the breadth gate above structurally cannot see. Its +# fanout_batches==0 condition is evaluated over the whole transcript, so ONE early batch +# satisfies it for every remaining Stop of the session while the model degrades into a serial +# loop -- the amnesty measured in 3ce9f899 and 8959d943 (see the fanout_calc comment). This +# mandate reads $serial_tail instead: dispatches after the last real batch, or all of them +# when none ever ran. Threshold 3, replayed against real sessions before choosing it: tails +# of 12 (e0cd5a40), 25 (3ce9f899) and 9 (8959d943) all fire; 416fb382's post-block tail of 2 +# does not, because two singleton dispatches are the shape two unrelated one-shot asks +# legitimately produce, and a tail of 3 is necessarily 3 separate messages (3 in one message +# would be a batch and would reset the tail). Its own windowed 2-strike latch above bounds +# the cost when this call is wrong. Not gated on dir/ext breadth: the defect it polices is in +# the dispatch pattern itself, not in what files were touched. +if [ "$eval_serial" = 1 ] && [ "$serial_tail" -ge 3 ]; then + serial_n=$serial_tail + [ "$serial_n" -gt 5 ] && serial_n=5 + unmet="$unmet + serial dispatch tail -- $serial_tail Task/Agent dispatch(es) sent one message at a time since the last parallel batch (or session start, if none ever ran). Send the remaining independent work as $serial_n Agent calls in ONE message (call Skill swarm first -- it routes the batch)." + hit_serial=1 +fi + # --- delegation-drift logger (tests/delegation-drift.sh) --------------------------------------- # Every field below -- dir_count, ext_count, task_count, named -- is already sitting in a # variable by this point in the script; the breadth and agent-naming mandates above computed all @@ -962,6 +999,12 @@ if [ "$hit_swarm" = 1 ]; then elif [ "$eval_swarm" = 1 ]; then rm -f "$cnt_file.delegate-swarm" "$cnt_file.delegate-swarm-ts" fi +if [ "$hit_serial" = 1 ]; then + echo $((cnt_serial + 1)) > "$cnt_file.delegate-serial" + date +%s > "$cnt_file.delegate-serial-ts" 2>/dev/null +elif [ "$eval_serial" = 1 ]; then + rm -f "$cnt_file.delegate-serial" "$cnt_file.delegate-serial-ts" +fi [ -n "$unmet" ] || exit 0 @@ -995,6 +1038,9 @@ fi if [ "$hit_swarm" = 1 ]; then reason="$reason$(_strike_line swarm "$((cnt_swarm + 1))" "in this ${DELEGATE_RESET_SECS}s window -- after 2, this mandate alone stops being enforced until the window elapses with no further unmet Stop for it.")" fi +if [ "$hit_serial" = 1 ]; then + reason="$reason$(_strike_line "serial dispatch tail" "$((cnt_serial + 1))" "in this ${DELEGATE_RESET_SECS}s window -- after 2, this mandate alone stops being enforced until the window elapses with no further unmet Stop for it.")" +fi reason="$reason Run each named skill with the Skill tool against the files listed, apply what it says, then finish. For prove-it-works: run the command that proves the change, read its actual output, then restate diff --git a/tests/mandate-cases.sh b/tests/mandate-cases.sh index ba2a7d3..de518ee 100755 --- a/tests/mandate-cases.sh +++ b/tests/mandate-cases.sh @@ -57,7 +57,7 @@ # invoke the hook at all, but this file does not). # shellcheck disable=SC2034 # read by callers that source this file (verify.sh check 27, tests/container-matrix.sh), not used within it -MANDATE_CASE_IDS="a b c d e f g h i j k l m n o p q 9b 10 11 12" +MANDATE_CASE_IDS="a b c d e f g h i j k l m n o p q 9b 9c 9d 9e 10 11 12" # --- fixture records (shared building blocks, same literal shapes check 27 already proved) ------- _MC_W='{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Write","input":{"file_path":"/x/README.md"}}]}}' @@ -82,6 +82,14 @@ _MC_9B_B='{"type":"assistant","message":{"content":[{"type":"tool_use","name":"W _MC_9B_C='{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Write","input":{"file_path":"brs/c/z.yaml","content":""}}]}}' _MC_9B_TASK='{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Task","input":{"tool":"Skill"}}]}}' _MC_9B_TXT='{"type":"assistant","message":{"content":[{"type":"text","text":"RICK: dispatched MEESEEKS to verify, then moved on."}]}}' +# 9c/9d/9e: the serial-tail mandate. Three-plus Task/Agent dispatches with no parallel batch +# after the last one is a serial loop, whatever fanout_batches says about earlier history. +# 9c: three singleton dispatches, swarm called, RICK named, zero writes -- no other mandate +# can fire, tail=3 -> block. 9d: two singletons -> tail=2, below threshold, silent. 9e: a +# real 2-in-one-message batch FIRST, then three singletons -- fanout_batches=1 so the breadth +# rule can never complain again, but the tail since that batch is 3 -> block. 9e is the +# whole-transcript amnesty measured in real sessions (3ce9f899: 3 early batches, then 25 +# serial dispatches, never blocked) that this mandate exists to close. # 10: purely conversational, zero tool_use anywhere in the transcript. _MC_10='{"type":"assistant","message":{"content":[{"type":"text","text":"Explaining how the retry loop computes exponential backoff."}]}}' # 11/12: prove-it-works, unverified vs verified. @@ -110,6 +118,9 @@ mandate_case_lines() { p) printf '%s\n' "$_MC_F2" "$_MC_SW" "$_MC_ON1" "$_MC_ON2" ;; q) printf '%s\n' "$_MC_TB" ;; 9b) printf '%s\n' "$_MC_SW" "$_MC_9B_A" "$_MC_9B_B" "$_MC_9B_C" "$_MC_9B_TASK" "$_MC_9B_TXT" ;; + 9c) printf '%s\n' "$_MC_SW" "$_MC_9B_TASK" "$_MC_9B_TASK" "$_MC_9B_TASK" "$_MC_9B_TXT" ;; + 9d) printf '%s\n' "$_MC_SW" "$_MC_ON1" "$_MC_ON2" ;; + 9e) printf '%s\n' "$_MC_SW" "$_MC_TWO" "$_MC_9B_TASK" "$_MC_9B_TASK" "$_MC_9B_TASK" ;; 10) printf '%s\n' "$_MC_10" ;; 11) printf '%s\n' "$_MC_11_W" "$_MC_DONE" ;; 12) printf '%s\n' "$_MC_12_W" "$_MC_12_BASH" "$_MC_DONE" ;; @@ -137,6 +148,9 @@ mandate_case_expect() { p) printf '%s\n' 'BLOCK:2 subagent call(s), but never 2+ in the same message' ;; q) printf '%s\n' 'BLOCK:the swarm skill' ;; 9b) printf '%s\n' 'BLOCK:1 subagent call(s), but never 2+ in the same message' ;; + 9c) printf '%s\n' 'BLOCK:serial dispatch tail' ;; + 9d) printf '%s\n' 'SILENT' ;; + 9e) printf '%s\n' 'BLOCK:serial dispatch tail' ;; 10) printf '%s\n' 'SILENT' ;; 11) printf '%s\n' 'BLOCK:prove-it-works' ;; 12) printf '%s\n' 'SILENT' ;; @@ -173,6 +187,9 @@ mandate_case_desc() { p) printf '%s\n' 'two dispatches in SEPARATE messages satisfied breadth anyway -- a serial loop cleared the fan-out mandate' ;; q) printf '%s\n' 'dispatched without calling the swarm skill and it did not block' ;; 9b) printf '%s\n' 'breadth-eligible + swarm called + ONE serial dispatch cleared the fan-out mandate anyway' ;; + 9c) printf '%s\n' 'three serial dispatches with no batch did not trip the serial-tail mandate' ;; + 9d) printf '%s\n' 'two serial dispatches (below the tail-3 threshold) tripped the serial-tail mandate' ;; + 9e) printf '%s\n' 'an early parallel batch amnestied three later serial dispatches' ;; 10) printf '%s\n' 'a purely conversational turn (no tool_use) blocked' ;; 11) printf '%s\n' 'edit + completion claim with no verification did not block prove-it-works' ;; 12) printf '%s\n' 'edit + completion claim WITH a Bash call in the turn blocked anyway' ;; diff --git a/tests/test-breadth-mandate.sh b/tests/test-breadth-mandate.sh index 3b9ebd3..43dc705 100755 --- a/tests/test-breadth-mandate.sh +++ b/tests/test-breadth-mandate.sh @@ -8,6 +8,7 @@ # "Task", never the "Agent" name the Claude Agent SDK build actually uses) and the Bash # write-extraction over-match defect ($VAR-containing paths and heredoc-body content read as # real writes) found by dogfooding this hook against a real 15MB transcript. +# PROOFs 22-24 cover the serial-tail mandate; the per-prompt digest's own tests live in tests/test-session-context-mandate.sh. # # The hook's contract (Claude Code Stop-hook protocol, not exit code): a met mandate prints # nothing to stdout and exits 0; an unmet one prints one JSON object on stdout -- @@ -61,6 +62,9 @@ if ! command -v jq >/dev/null 2>&1; then skip "PROOF 19 (setup 1/3): lone prose write trips unslop" "jq not installed" skip "PROOF 19 (setup 2/3): lone TypeScript edit trips typescript-best-practices (unrelated 2nd strike)" "jq not installed" skip "PROOF 19 (the bleed): a THIRD, never-tried mandate still fires after two UNRELATED strikes" "jq not installed" + skip "PROOF 22: three singleton dispatches, no batch -> serial-tail mandate blocks" "jq not installed" + skip "PROOF 23: two singleton dispatches -> serial-tail mandate silent (below threshold)" "jq not installed" + skip "PROOF 24: early 2-in-one-message batch, then three singletons -> serial-tail mandate still blocks (amnesty closed)" "jq not installed" printf 'checks: %d declared, %d ran, %d skipped\n' "$TOTAL" "$RAN" "$SKIPPED" [ "$((RAN + SKIPPED))" -eq "$TOTAL" ] || { printf 'FAIL check accounting\n %d declared check(s) reported nothing\n' "$((TOTAL - RAN - SKIPPED))"; FAIL=1; } [ "$FAIL" -eq 0 ] && echo VERIFIED || echo "VERIFICATION FAILED" @@ -122,6 +126,7 @@ names_piw_(){ printf '%s' "$HOOK_REASON" | grep -qF 'prove-it-works --'; } names_swarm_(){ printf '%s' "$HOOK_REASON" | grep -qF 'swarm --'; } names_unslop_(){ printf '%s' "$HOOK_REASON" | grep -qF 'unslop --'; } names_ts_(){ printf '%s' "$HOOK_REASON" | grep -qF 'typescript-best-practices --'; } +names_serial_(){ printf '%s' "$HOOK_REASON" | grep -qF 'serial dispatch tail --'; } # --- PROOF 1: 5 fixture writes, 1 directory, 1 extension, 0 Task calls ------------------------ # Negative direction. Mechanical repetition (fixtures/case1.json .. case5.json) must not read @@ -480,6 +485,48 @@ else "expected decision=block naming 'prove-it-works --' -- two strikes on unslop+typescript must not silence a mandate neither of them is, got: decision=$HOOK_DECISION reason=[$HOOK_REASON]" fi +# --- PROOF 22: three singleton Task/Agent dispatches, zero batches -> serial-tail blocks -------- +# Positive direction for the serial-tail mandate: three dispatches, each alone in its own +# assistant line (no shared message.id), is the serial loop measured in real transcripts +# (e0cd5a40: 12 singleton dispatches, never a batch). Swarm called and RICK named so no other +# delegation mandate can confound; zero writes so no skill mandate can either. +say_ $'{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"0","name":"Skill","input":{"skill":"swarm"}},{"type":"text","text":"RICK: routing three reviews."}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"1","name":"Task","input":{"prompt":"review a"}}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"2","name":"Task","input":{"prompt":"review b"}}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"3","name":"Agent","input":{"subagent_type":"qa","prompt":"review c","description":"review c"}}]}}' +run_hook_ proof22 +if [ "$HOOK_DECISION" = "block" ] && names_serial_; then + ok "PROOF 22: three singleton dispatches, no batch -> serial-tail mandate blocks" +else + bad "PROOF 22: three singleton dispatches, no batch -> serial-tail mandate blocks" \ + "expected decision=block naming 'serial dispatch tail --', got: decision=$HOOK_DECISION reason=[$HOOK_REASON]" +fi + +# --- PROOF 23: two singleton dispatches -> below the tail-3 threshold, strictly silent ---------- +# Negative direction: two serial dispatches are the shape a session legitimately produces when +# two unrelated asks arrive in two turns (416fb382's post-block tail of 2). Nothing else in the +# fixture can block, so the bar is strict empty stdout, not merely "no serial-tail line". +say_ $'{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"0","name":"Skill","input":{"skill":"swarm"}},{"type":"text","text":"RICK: routing two reviews."}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"1","name":"Task","input":{"prompt":"review a"}}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"2","name":"Task","input":{"prompt":"review b"}}]}}' +run_hook_ proof23 +if [ -z "$HOOK_OUT" ]; then + ok "PROOF 23: two singleton dispatches -> serial-tail mandate silent (below threshold)" +else + bad "PROOF 23: two singleton dispatches -> serial-tail mandate silent (below threshold)" \ + "expected empty stdout, got: decision=$HOOK_DECISION reason=[$HOOK_REASON]" +fi + +# --- PROOF 24: an early real batch, then three singletons -> still blocks (amnesty closed) ------ +# The whole-transcript amnesty itself: fanout_batches=1 from the 2-in-one-message batch, so the +# breadth mandate's own condition (fanout_batches == 0) can never be true again this session -- +# measured in real transcripts (3ce9f899: 3 early batches then 25 serial dispatches unblocked; +# 8959d943: 2 batches of 2 then 9 serial). The serial-tail mandate reads only the dispatches +# AFTER the last batch: three singletons -> block, early batch or not. +say_ $'{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"0","name":"Skill","input":{"skill":"swarm"}},{"type":"text","text":"RICK: batch first."}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"1","name":"Task","input":{"prompt":"review a"}},{"type":"tool_use","id":"2","name":"Agent","input":{"subagent_type":"qa","prompt":"verify","description":"verify"}},{"type":"text","text":"ZEEP and GLOOTIE together."}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"3","name":"Task","input":{"prompt":"review b"}}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"4","name":"Task","input":{"prompt":"review c"}}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"5","name":"Agent","input":{"subagent_type":"qa","prompt":"review d","description":"review d"}}]}}' +run_hook_ proof24 +if [ "$HOOK_DECISION" = "block" ] && names_serial_; then + ok "PROOF 24: early 2-in-one-message batch, then three singletons -> serial-tail mandate still blocks (amnesty closed)" +else + bad "PROOF 24: early 2-in-one-message batch, then three singletons -> serial-tail mandate still blocks (amnesty closed)" \ + "expected decision=block naming 'serial dispatch tail --', got: decision=$HOOK_DECISION reason=[$HOOK_REASON]" +fi + echo printf 'checks: %d declared, %d ran, %d skipped\n' "$TOTAL" "$RAN" "$SKIPPED" if [ "$((RAN + SKIPPED))" -ne "$TOTAL" ]; then diff --git a/tests/test-session-context-mandate.sh b/tests/test-session-context-mandate.sh new file mode 100755 index 0000000..684689b --- /dev/null +++ b/tests/test-session-context-mandate.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# test-session-context-mandate.sh — the hand-runnable reproduction of the MANDATE escalation +# in claude/hooks/inject-session-context.sh's UserPromptSubmit digest. The hook's current code +# (lines ~155-168) reads $TMPDIR/vstack-mandate-$sid and $TMPDIR/vstack-mandate-$sid.delegate +# — files skill-mandate.sh never writes. skill-mandate.sh writes per-mandate suffixes only: +# .unslop .typescript .proveitworks .delegate-breadth .delegate-naming .delegate-swarm +# .delegate-serial. So the escalation line is dead code in production; these tests are written +# RED against that defect and against the digest lacking a FANOUT line, and go green when the +# hook is fixed to (a) read the max of each family's real per-mandate counter files and (b) pin +# the fan-out rule every prompt. +# +# Real-transcript evidence: (1) MANDATE line never rendered in any real session because the +# counters read from non-existent files always stayed 0; (2) serial under-delegation (4.8% +# parallel-batch rate in vstack itself) that the FANOUT re-pin in the digest addresses. +# +# Hook contract: on UserPromptSubmit, emits a JSON object with +# .hookSpecificOutput.additionalContext containing the per-prompt digest (a multi-line string). +# This digest must be kept under 512 bytes total (the grill worst-case in .claude/verify.sh's +# check 18 budget). The tests below parse the additionalContext value via jq and grep it for +# expected lines. +set -uo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +hook="$repo_root/claude/hooks/inject-session-context.sh" +SELF="$repo_root/tests/test-session-context-mandate.sh" + +RAN=0 +SKIPPED=0 +FAIL=0 +TOTAL=$(grep -c '^# --- CASE [0-9]' "$SELF") + +ok(){ printf 'ok %s\n' "$1"; RAN=$((RAN+1)); } +bad(){ printf 'FAIL %s\n %s\n' "$1" "$2"; FAIL=1; RAN=$((RAN+1)); } +skip(){ printf 'skip %s (%s)\n' "$1" "$2"; SKIPPED=$((SKIPPED+1)); } + +if ! command -v jq >/dev/null 2>&1; then + skip "CASE 1: seeded delegate-family counter renders the MANDATE line" "jq not installed" + skip "CASE 2: seeded skill-family counter renders the MANDATE line" "jq not installed" + skip "CASE 3: no counter files -> MANDATE line absent" "jq not installed" + skip "CASE 4: digest pins the fan-out rule every prompt" "jq not installed" + skip "CASE 5: unconditional digest stays under 512-byte budget" "jq not installed" + skip "CASE 6: worst case (grill + both families at cap) stays under budget" "jq not installed" + printf 'checks: %d declared, %d ran, %d skipped\n' "$TOTAL" "$RAN" "$SKIPPED" + [ "$((RAN + SKIPPED))" -eq "$TOTAL" ] || { printf 'FAIL check accounting\n %d declared check(s) reported nothing\n' "$((TOTAL - RAN - SKIPPED))"; FAIL=1; } + [ "$FAIL" -eq 0 ] && echo VERIFIED || echo "VERIFICATION FAILED" + exit "$FAIL" +fi + +if [ ! -x "$hook" ]; then + bad "session-context-mandate proofs" "$hook is missing or not executable" + printf 'checks: %d declared, %d ran, %d skipped\n' "$TOTAL" "$RAN" "$SKIPPED" + echo "VERIFICATION FAILED" + exit 1 +fi + +WORK="$(mktemp -d "${TMPDIR:-/tmp}/vstack-ctx-mandate.XXXXXX")" +trap 'rm -rf "$WORK"' EXIT + +# Invocation helper: runs the hook with a UserPromptSubmit event, reads the digest context. +# The hook reads counter files under ${TMPDIR:-/tmp}, so we set TMPDIR="$WORK" for isolation. +# Takes two invocations in some cases: the hook is stateless apart from the grill first-seen +# marker, which the SECOND call sees as already-marked. Call order therefore matters only for +# grill cases. +run_ctx_(){ # + _payload=$(printf '{"hook_event_name":"UserPromptSubmit","session_id":"%s","prompt":"%s"}' "$1" "$2") + printf '%s' "$_payload" | TMPDIR="$WORK" bash "$hook" 2>/dev/null | jq -r '.hookSpecificOutput.additionalContext // empty' 2>/dev/null +} + +run_ctx_bytes_(){ # — return byte count of raw stdout + _payload=$(printf '{"hook_event_name":"UserPromptSubmit","session_id":"%s","prompt":"%s"}' "$1" "$2") + printf '%s' "$_payload" | TMPDIR="$WORK" bash "$hook" 2>/dev/null | wc -c | tr -d ' ' +} + +# --- CASE 1: seeded delegate-family counter renders the MANDATE line ------------------------- +# Dead-code defect: the hook reads vstack-mandate-scm1.delegate (non-existent) instead of the +# real per-mandate files like vstack-mandate-scm1.delegate-breadth that skill-mandate.sh writes. +# When fixed to read the max of real counter files, a seeded delegate-breadth file must render +# the MANDATE line showing the escalation counters. Direction: positive, guard against silent +# zero-read on wrong filenames. +printf '2\n' > "$WORK/vstack-mandate-scm1.delegate-breadth" +CTX=$(run_ctx_ scm1 "hi") +if printf '%s' "$CTX" | grep -qF 'MANDATE skill=0/2 delegate=2/2'; then + ok "CASE 1: seeded delegate-family counter renders the MANDATE line" +else + bad "CASE 1: seeded delegate-family counter renders the MANDATE line" \ + "expected CTX to contain 'MANDATE skill=0/2 delegate=2/2', got: [$CTX]" +fi + +# --- CASE 2: seeded skill-family counter renders the MANDATE line ---------------------------- +# Same dead-code defect: the hook reads vstack-mandate-scm2 (non-existent generic file) instead +# of real per-mandate skill counters (.unslop, .typescript, .proveitworks). When fixed, a seeded +# skill-family counter must render the MANDATE line showing the escalation counters. Direction: +# positive, guard against silent zero-read on wrong filenames. +printf '1\n' > "$WORK/vstack-mandate-scm2.unslop" +CTX=$(run_ctx_ scm2 "hi") +if printf '%s' "$CTX" | grep -qF 'MANDATE skill=1/2 delegate=0/2'; then + ok "CASE 2: seeded skill-family counter renders the MANDATE line" +else + bad "CASE 2: seeded skill-family counter renders the MANDATE line" \ + "expected CTX to contain 'MANDATE skill=1/2 delegate=0/2', got: [$CTX]" +fi + +# --- CASE 3: no counter files -> MANDATE line absent ---------------------------------------- +# Negative direction: the common case in most sessions. No counter files seeded, so both +# families read as 0/0. The digest must stay silent (MANDATE line absent) — most prompts in most +# sessions never tripped either mandate, and the byte budget (check 18) is a hard cap on every +# prompt, so a line that always rendered would blow the budget for the common case. +CTX=$(run_ctx_ scm3 "hi") +if ! printf '%s' "$CTX" | grep -qF MANDATE; then + ok "CASE 3: no counter files -> MANDATE line absent" +else + bad "CASE 3: no counter files -> MANDATE line absent" \ + "expected CTX to NOT contain MANDATE, got: [$CTX]" +fi + +# --- CASE 4: the digest pins the fan-out rule every prompt ----------------------------------- +# Defect: the digest today lacks a FANOUT line re-pinning the fan-out rule on every prompt. +# Parallel-batch rate in real sessions is 4.8%, a signal that the rule is drifting. When fixed, +# the digest must emit a FANOUT line describing the fan-out contract, a DELEGATE line naming the +# mechanical/judgment split, AND a line about batching in ONE message. Direction: positive, +# guard the digest's completeness and the rule's re-pinning every turn. +CTX=$(run_ctx_ scm4 "hi") +has_fanout=$(printf '%s' "$CTX" | grep -qF 'FANOUT:' && echo 1 || echo 0) +has_delegate=$(printf '%s' "$CTX" | grep -qF 'DELEGATE: mechanical' && echo 1 || echo 0) +has_batch=$(printf '%s' "$CTX" | grep -qF 'ALL Agent calls in ONE message' && echo 1 || echo 0) +if [ "$has_fanout" = 1 ] && [ "$has_delegate" = 1 ] && [ "$has_batch" = 1 ]; then + ok "CASE 4: the digest pins the fan-out rule every prompt" +else + bad "CASE 4: the digest pins the fan-out rule every prompt" \ + "expected FANOUT (got $has_fanout), DELEGATE (got $has_delegate), batch rule (got $has_batch) in digest: [$CTX]" +fi + +# --- CASE 5: unconditional digest stays inside the verify.sh check-18 budget ------------------ +# Boundary test: the digest (grill + MANDATE + FANOUT) is pinned every prompt on every machine, +# so it must fit within the byte cap check 18 measures (512 bytes worst-case). The hook can emit +# empty additionalContext (0 bytes) when grill and MANDATE are both silent, or up to ~512 when +# both are armed. This case has no counters seeded and grill disabled by short prompt, so it +# measures the unconditional part (TOKENS + DELEGATE + FANOUT + batch rule). Direction: +# negative, guard against budget overflow from new digest lines. +RAW_BYTES=$(run_ctx_bytes_ scm5 "hi") +if [ "$RAW_BYTES" -ge 128 ] && [ "$RAW_BYTES" -le 512 ]; then + ok "CASE 5: unconditional digest stays inside the verify.sh check-18 budget" +else + bad "CASE 5: unconditional digest stays inside the verify.sh check-18 budget" \ + "expected RAW_BYTES between 128 and 512, got: $RAW_BYTES" +fi + +# --- CASE 6: worst case (grill + both MANDATE families at cap) stays under 512 ---------------- +# Boundary test, the worst case: grill fires + both mandate families at 2/2 (the cap). The hook +# must stay under 512 bytes even with all three pieces armed. Seed both families at max +# (2 counters each), build a 400-char prompt to arm the grill (>= 320 chars), measure the FIRST +# invocation's raw byte count (to catch the grill armed, not cached). Also assert the output +# contains 'GRILL: run the grill-me skill' to prove grill actually fired (worst case is real). +# Direction: positive boundary, guard against budget overflow under worst-case load. +# NOTE: measure bytes on FIRST invocation when grill fires; grill marker is cached so second +# call would see it as already-marked. Fresh session id, one invocation only. +printf '2\n' > "$WORK/vstack-mandate-scm6.delegate-breadth" +printf '2\n' > "$WORK/vstack-mandate-scm6.unslop" +_p="" +_i=0 +while [ $_i -lt 400 ]; do + _p="${_p}x" + _i=$((_i+1)) +done +_payload_6=$(printf '{"hook_event_name":"UserPromptSubmit","session_id":"scm6","prompt":"%s"}' "$_p") +_out_6=$(printf '%s' "$_payload_6" | TMPDIR="$WORK" bash "$hook" 2>/dev/null) +_bytes_out=$(printf '%s' "$_out_6" | wc -c | tr -d ' ') +_ctx=$(printf '%s' "$_out_6" | jq -r '.hookSpecificOutput.additionalContext // empty' 2>/dev/null) +has_grill=$(printf '%s' "$_ctx" | grep -qF 'GRILL: run the grill-me skill' && echo 1 || echo 0) +if [ "$_bytes_out" -le 512 ] && [ "$has_grill" = 1 ]; then + ok "CASE 6: worst case (grill + both MANDATE families at cap) stays under budget" +else + bad "CASE 6: worst case (grill + both MANDATE families at cap) stays under budget" \ + "expected bytes <= 512 (got $_bytes_out) and GRILL present (got $has_grill)" +fi + +echo +printf 'checks: %d declared, %d ran, %d skipped\n' "$TOTAL" "$RAN" "$SKIPPED" +if [ "$((RAN + SKIPPED))" -ne "$TOTAL" ]; then + printf 'FAIL check accounting\n %d declared check(s) reported nothing\n' "$((TOTAL - RAN - SKIPPED))" + FAIL=1 +fi +[ "$FAIL" -eq 0 ] && [ "$RAN" -gt 0 ] && echo VERIFIED || echo "VERIFICATION FAILED" +[ "$FAIL" -eq 0 ] && [ "$RAN" -gt 0 ] +exit $? From b8216bf9135678d239098de2adc62edd0bd201c1 Mon Sep 17 00:00:00 2001 From: vk Date: Tue, 1 Sep 2026 20:08:22 +0530 Subject: [PATCH 2/3] Back out auto-enforcement work swept into 1bfb133 by a concurrent commit 1bfb133's commit -a in this shared checkout picked up another session's staged hook/test changes (skill-mandate.sh serial-tail mandate, inject-session-context.sh digest changes, three test files) alongside the alpine python3 fix it describes. That payload drift is what turned 'declared version matches what installs' and the inventory digest red against the in-flight v1.61.0. This restores those five paths to their 59c99f4 state; the mandate work continues on the auto-enforcement branch. Co-Authored-By: Claude Fable 5 --- claude/hooks/inject-session-context.sh | 37 ++--- claude/hooks/skill-mandate.sh | 54 +------- tests/mandate-cases.sh | 19 +-- tests/test-breadth-mandate.sh | 47 ------- tests/test-session-context-mandate.sh | 185 ------------------------- 5 files changed, 17 insertions(+), 325 deletions(-) delete mode 100755 tests/test-session-context-mandate.sh diff --git a/claude/hooks/inject-session-context.sh b/claude/hooks/inject-session-context.sh index 14e9436..95ee192 100755 --- a/claude/hooks/inject-session-context.sh +++ b/claude/hooks/inject-session-context.sh @@ -130,19 +130,15 @@ if [ "$event" = "UserPromptSubmit" ]; then if [ "$_n" -ge "${VSTACK_GRILL_CHARS:-320}" ] \ || { [ -z "$_seen" ] && [ "$_n" -ge 120 ]; }; then grill=' -GRILL: run the grill-me skill when no more specific skill matches.' +GRILL: run the grill-me skill when no skill matches this situation more specifically. A +situation-matched skill outranks it. grill-me is for a request whose shape is still undecided.' fi fi # Delegation-mandate strike count, re-pinned every prompt instead of stated once at - # SessionStart and then never again. skill-mandate.sh (Stop) is the only writer of the - # counter files read here, and it writes ONE FILE PER MANDATE (.unslop/.typescript/ - # .proveitworks for the skill family, .delegate-breadth/-naming/-swarm/-serial for the - # delegation family) -- there is no combined per-family file. This block took the max of two - # filenames the Stop hook had stopped writing when its counters split per-mandate, so both - # reads were always 0 and the MANDATE line below was dead code in every real session; now it - # takes the max across each family's real files. Still just cats of tiny files under - # $TMPDIR, not a transcript parse, so it can run every single prompt with no latency - # argument to make: there is no scan here. + # SessionStart and then never again. skill-mandate.sh (Stop) is the only writer of these two + # small counter files; this only reads them -- a cat of two tiny files under $TMPDIR, not a + # transcript parse, so it costs nothing like the Stop hook's own evaluation does. It is also + # why this can run every single prompt with no latency argument to make: there is no scan here. # # Two independent counters because skill-mandate.sh's own delegation family (breadth + # agent-naming) no longer shares the skill mandates' (unslop/typescript/prove-it-works) @@ -161,26 +157,17 @@ GRILL: run the grill-me skill when no more specific skill matches.' _msid=$(printf '%s' "$in" | "$JQ" -r '.session_id // empty' 2>/dev/null) [ -n "$_msid" ] || _msid="pid$PPID" _mcnt_file="${TMPDIR:-/tmp}/vstack-mandate-$_msid" - _mcnt=0 - for _sfx in unslop typescript proveitworks; do - _v=$(cat "$_mcnt_file.$_sfx" 2>/dev/null || echo 0) - case "$_v" in ''|*[!0-9]*) _v=0 ;; esac - [ "$_v" -gt "$_mcnt" ] && _mcnt=$_v - done - _mdcnt=0 - for _sfx in delegate-breadth delegate-naming delegate-swarm delegate-serial; do - _v=$(cat "$_mcnt_file.$_sfx" 2>/dev/null || echo 0) - case "$_v" in ''|*[!0-9]*) _v=0 ;; esac - [ "$_v" -gt "$_mdcnt" ] && _mdcnt=$_v - done + _mcnt=$(cat "$_mcnt_file" 2>/dev/null || echo 0) + case "$_mcnt" in ''|*[!0-9]*) _mcnt=0 ;; esac + _mdcnt=$(cat "$_mcnt_file.delegate" 2>/dev/null || echo 0) + case "$_mdcnt" in ''|*[!0-9]*) _mdcnt=0 ;; esac if [ "$_mcnt" -ge 1 ] || [ "$_mdcnt" -ge 1 ]; then mandate=" MANDATE skill=$_mcnt/2 delegate=$_mdcnt/2: dispatch + name a call sign now." fi fi - emit "$event" 'TOKENS: grep/ranges, not whole files; summarize, never dump. -DELEGATE: mechanical -> worker/explorer, judgment -> sonnet agents. ACT, do not ask. Skills fire on the situation -- call the Skill tool. -FANOUT: work that splits -> Skill swarm first, ALL Agent calls in ONE message.'"$grill$mandate" + emit "$event" 'TOKENS: grep/ranges, not whole files; batch independent tool calls in ONE message. +DELEGATE: mechanical -> worker/explorer, judgment -> sonnet agents. ACT, do not ask. Skills fire on the situation — call the Skill tool.'"$grill$mandate" exit 0 fi diff --git a/claude/hooks/skill-mandate.sh b/claude/hooks/skill-mandate.sh index 1af862e..eb433ae 100755 --- a/claude/hooks/skill-mandate.sh +++ b/claude/hooks/skill-mandate.sh @@ -202,7 +202,6 @@ _read_cnt "$cnt_file.proveitworks"; cnt_proveitworks=$_rc _read_wcnt "$cnt_file.delegate-breadth" "$cnt_file.delegate-breadth-ts" "$now_d" "$DELEGATE_RESET_SECS"; cnt_breadth=$_rc _read_wcnt "$cnt_file.delegate-naming" "$cnt_file.delegate-naming-ts" "$now_d" "$DELEGATE_RESET_SECS"; cnt_naming=$_rc _read_wcnt "$cnt_file.delegate-swarm" "$cnt_file.delegate-swarm-ts" "$now_d" "$DELEGATE_RESET_SECS"; cnt_swarm=$_rc -_read_wcnt "$cnt_file.delegate-serial" "$cnt_file.delegate-serial-ts" "$now_d" "$DELEGATE_RESET_SECS"; cnt_serial=$_rc eval_unslop=1; [ "$cnt_unslop" -ge 2 ] && eval_unslop=0 eval_typescript=1; [ "$cnt_typescript" -ge 2 ] && eval_typescript=0 @@ -210,7 +209,6 @@ eval_proveitworks=1; [ "$cnt_proveitworks" -ge 2 ] && eval_proveitworks=0 eval_breadth=1; [ "$cnt_breadth" -ge 2 ] && eval_breadth=0 eval_naming=1; [ "$cnt_naming" -ge 2 ] && eval_naming=0 eval_swarm=1; [ "$cnt_swarm" -ge 2 ] && eval_swarm=0 -eval_serial=1; [ "$cnt_serial" -ge 2 ] && eval_serial=0 # Second, SHORTER gate: how often to bother re-scanning at all once every skill-family mandate is # already latched, independent of whether delegation has struck. Family-exhausted alone is not a @@ -238,7 +236,7 @@ fi skill_eval=1 [ "$eval_unslop" = 0 ] && [ "$eval_typescript" = 0 ] && [ "$eval_proveitworks" = 0 ] && skill_eval=0 deleg_eval=1 -[ "$eval_breadth" = 0 ] && [ "$eval_naming" = 0 ] && [ "$eval_swarm" = 0 ] && [ "$eval_serial" = 0 ] && deleg_eval=0 +[ "$eval_breadth" = 0 ] && [ "$eval_naming" = 0 ] && [ "$eval_swarm" = 0 ] && deleg_eval=0 # Combined latch: skip the transcript-driven evaluation entirely only when NEITHER family can # still act on it. skill_eval=1 alone is enough to keep paying for the scan every Stop, unchanged @@ -275,7 +273,6 @@ hit_proveitworks=0 hit_breadth=0 hit_naming=0 hit_swarm=0 -hit_serial=0 # We are doing a full scan this Stop for at least one reason (skill_eval=1 or deleg_eval=1 -- # the latch above already exited otherwise). Record it now, unconditionally, so the cooldown @@ -610,14 +607,6 @@ case "$task_count" in ''|*[!0-9]*) task_count=0 ;; esac # there is no identity to merge on, so each such line is its own singleton batch. That is what # makes "two Task calls in two separate lines, neither carrying an id" read as two batches of # one, not one batch of two: the exact serial-loop shape this fix exists to catch. -# One pass, two numbers: " ". serial_tail is the count of -# Task/Agent dispatches AFTER the last 2+-in-one-message batch (all of them when no batch ever -# ran). fanout_batches answers "did a real batch ever happen"; serial_tail answers the question -# it structurally cannot: "what has happened SINCE". A whole-transcript batch count amnesties -# every later serial dispatch -- measured, not hypothesized: session 3ce9f899 batched 3 times -# early and then made 25 one-at-a-time dispatches with the breadth mandate silenced for good, -# and 8959d943 did the same behind 2 batches of 2. The serial-tail mandate below reads only the -# suffix, so history cannot pay for the present. fanout_calc=$( "$JQ" -sr ' ( [ .[] | select(.type=="assistant") | { id: (.message.id // null), @@ -635,16 +624,10 @@ fanout_calc=$( "$JQ" -sr ' ) ) as $folded | ( $folded.runs + [{id: $folded.cid, n: $folded.cn}] ) as $all_runs - | ( $all_runs | map(.n) ) as $ns - | ( [ range(0; $ns|length) | select($ns[.] >= 2) ] | last ) as $lastb - | ( [ $ns[] | select(. >= 2) ] | length ) as $batches - | ( (if $lastb == null then $ns else $ns[($lastb+1):] end) | add // 0 ) as $tail - | "\($batches) \($tail)" + | ( $all_runs | map(select(.n >= 2)) | length ) ' "$tr_" 2>/dev/null | tail -n 1 ) -fanout_batches=${fanout_calc%% *} -serial_tail=${fanout_calc##* } -case "$fanout_batches" in ''|*[!0-9]*) fanout_batches=0 ;; esac -case "$serial_tail" in ''|*[!0-9]*) serial_tail=0 ;; esac +case "$fanout_calc" in ''|*[!0-9]*) fanout_calc=0 ;; esac +fanout_batches="$fanout_calc" # task_fail_count: of those same Task/Agent dispatches, how many resolved with is_error==true on # their tool_result. This is the field the delegation-drift ledger was missing entirely -- it @@ -751,26 +734,6 @@ if [ "$eval_breadth" = 1 ] && [ "$dir_count" -ge 3 ] && [ "$ext_count" -ge 2 ] & hit_breadth=1 fi -# Serial dispatch tail: the shape the breadth gate above structurally cannot see. Its -# fanout_batches==0 condition is evaluated over the whole transcript, so ONE early batch -# satisfies it for every remaining Stop of the session while the model degrades into a serial -# loop -- the amnesty measured in 3ce9f899 and 8959d943 (see the fanout_calc comment). This -# mandate reads $serial_tail instead: dispatches after the last real batch, or all of them -# when none ever ran. Threshold 3, replayed against real sessions before choosing it: tails -# of 12 (e0cd5a40), 25 (3ce9f899) and 9 (8959d943) all fire; 416fb382's post-block tail of 2 -# does not, because two singleton dispatches are the shape two unrelated one-shot asks -# legitimately produce, and a tail of 3 is necessarily 3 separate messages (3 in one message -# would be a batch and would reset the tail). Its own windowed 2-strike latch above bounds -# the cost when this call is wrong. Not gated on dir/ext breadth: the defect it polices is in -# the dispatch pattern itself, not in what files were touched. -if [ "$eval_serial" = 1 ] && [ "$serial_tail" -ge 3 ]; then - serial_n=$serial_tail - [ "$serial_n" -gt 5 ] && serial_n=5 - unmet="$unmet - serial dispatch tail -- $serial_tail Task/Agent dispatch(es) sent one message at a time since the last parallel batch (or session start, if none ever ran). Send the remaining independent work as $serial_n Agent calls in ONE message (call Skill swarm first -- it routes the batch)." - hit_serial=1 -fi - # --- delegation-drift logger (tests/delegation-drift.sh) --------------------------------------- # Every field below -- dir_count, ext_count, task_count, named -- is already sitting in a # variable by this point in the script; the breadth and agent-naming mandates above computed all @@ -999,12 +962,6 @@ if [ "$hit_swarm" = 1 ]; then elif [ "$eval_swarm" = 1 ]; then rm -f "$cnt_file.delegate-swarm" "$cnt_file.delegate-swarm-ts" fi -if [ "$hit_serial" = 1 ]; then - echo $((cnt_serial + 1)) > "$cnt_file.delegate-serial" - date +%s > "$cnt_file.delegate-serial-ts" 2>/dev/null -elif [ "$eval_serial" = 1 ]; then - rm -f "$cnt_file.delegate-serial" "$cnt_file.delegate-serial-ts" -fi [ -n "$unmet" ] || exit 0 @@ -1038,9 +995,6 @@ fi if [ "$hit_swarm" = 1 ]; then reason="$reason$(_strike_line swarm "$((cnt_swarm + 1))" "in this ${DELEGATE_RESET_SECS}s window -- after 2, this mandate alone stops being enforced until the window elapses with no further unmet Stop for it.")" fi -if [ "$hit_serial" = 1 ]; then - reason="$reason$(_strike_line "serial dispatch tail" "$((cnt_serial + 1))" "in this ${DELEGATE_RESET_SECS}s window -- after 2, this mandate alone stops being enforced until the window elapses with no further unmet Stop for it.")" -fi reason="$reason Run each named skill with the Skill tool against the files listed, apply what it says, then finish. For prove-it-works: run the command that proves the change, read its actual output, then restate diff --git a/tests/mandate-cases.sh b/tests/mandate-cases.sh index de518ee..ba2a7d3 100755 --- a/tests/mandate-cases.sh +++ b/tests/mandate-cases.sh @@ -57,7 +57,7 @@ # invoke the hook at all, but this file does not). # shellcheck disable=SC2034 # read by callers that source this file (verify.sh check 27, tests/container-matrix.sh), not used within it -MANDATE_CASE_IDS="a b c d e f g h i j k l m n o p q 9b 9c 9d 9e 10 11 12" +MANDATE_CASE_IDS="a b c d e f g h i j k l m n o p q 9b 10 11 12" # --- fixture records (shared building blocks, same literal shapes check 27 already proved) ------- _MC_W='{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Write","input":{"file_path":"/x/README.md"}}]}}' @@ -82,14 +82,6 @@ _MC_9B_B='{"type":"assistant","message":{"content":[{"type":"tool_use","name":"W _MC_9B_C='{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Write","input":{"file_path":"brs/c/z.yaml","content":""}}]}}' _MC_9B_TASK='{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Task","input":{"tool":"Skill"}}]}}' _MC_9B_TXT='{"type":"assistant","message":{"content":[{"type":"text","text":"RICK: dispatched MEESEEKS to verify, then moved on."}]}}' -# 9c/9d/9e: the serial-tail mandate. Three-plus Task/Agent dispatches with no parallel batch -# after the last one is a serial loop, whatever fanout_batches says about earlier history. -# 9c: three singleton dispatches, swarm called, RICK named, zero writes -- no other mandate -# can fire, tail=3 -> block. 9d: two singletons -> tail=2, below threshold, silent. 9e: a -# real 2-in-one-message batch FIRST, then three singletons -- fanout_batches=1 so the breadth -# rule can never complain again, but the tail since that batch is 3 -> block. 9e is the -# whole-transcript amnesty measured in real sessions (3ce9f899: 3 early batches, then 25 -# serial dispatches, never blocked) that this mandate exists to close. # 10: purely conversational, zero tool_use anywhere in the transcript. _MC_10='{"type":"assistant","message":{"content":[{"type":"text","text":"Explaining how the retry loop computes exponential backoff."}]}}' # 11/12: prove-it-works, unverified vs verified. @@ -118,9 +110,6 @@ mandate_case_lines() { p) printf '%s\n' "$_MC_F2" "$_MC_SW" "$_MC_ON1" "$_MC_ON2" ;; q) printf '%s\n' "$_MC_TB" ;; 9b) printf '%s\n' "$_MC_SW" "$_MC_9B_A" "$_MC_9B_B" "$_MC_9B_C" "$_MC_9B_TASK" "$_MC_9B_TXT" ;; - 9c) printf '%s\n' "$_MC_SW" "$_MC_9B_TASK" "$_MC_9B_TASK" "$_MC_9B_TASK" "$_MC_9B_TXT" ;; - 9d) printf '%s\n' "$_MC_SW" "$_MC_ON1" "$_MC_ON2" ;; - 9e) printf '%s\n' "$_MC_SW" "$_MC_TWO" "$_MC_9B_TASK" "$_MC_9B_TASK" "$_MC_9B_TASK" ;; 10) printf '%s\n' "$_MC_10" ;; 11) printf '%s\n' "$_MC_11_W" "$_MC_DONE" ;; 12) printf '%s\n' "$_MC_12_W" "$_MC_12_BASH" "$_MC_DONE" ;; @@ -148,9 +137,6 @@ mandate_case_expect() { p) printf '%s\n' 'BLOCK:2 subagent call(s), but never 2+ in the same message' ;; q) printf '%s\n' 'BLOCK:the swarm skill' ;; 9b) printf '%s\n' 'BLOCK:1 subagent call(s), but never 2+ in the same message' ;; - 9c) printf '%s\n' 'BLOCK:serial dispatch tail' ;; - 9d) printf '%s\n' 'SILENT' ;; - 9e) printf '%s\n' 'BLOCK:serial dispatch tail' ;; 10) printf '%s\n' 'SILENT' ;; 11) printf '%s\n' 'BLOCK:prove-it-works' ;; 12) printf '%s\n' 'SILENT' ;; @@ -187,9 +173,6 @@ mandate_case_desc() { p) printf '%s\n' 'two dispatches in SEPARATE messages satisfied breadth anyway -- a serial loop cleared the fan-out mandate' ;; q) printf '%s\n' 'dispatched without calling the swarm skill and it did not block' ;; 9b) printf '%s\n' 'breadth-eligible + swarm called + ONE serial dispatch cleared the fan-out mandate anyway' ;; - 9c) printf '%s\n' 'three serial dispatches with no batch did not trip the serial-tail mandate' ;; - 9d) printf '%s\n' 'two serial dispatches (below the tail-3 threshold) tripped the serial-tail mandate' ;; - 9e) printf '%s\n' 'an early parallel batch amnestied three later serial dispatches' ;; 10) printf '%s\n' 'a purely conversational turn (no tool_use) blocked' ;; 11) printf '%s\n' 'edit + completion claim with no verification did not block prove-it-works' ;; 12) printf '%s\n' 'edit + completion claim WITH a Bash call in the turn blocked anyway' ;; diff --git a/tests/test-breadth-mandate.sh b/tests/test-breadth-mandate.sh index 43dc705..3b9ebd3 100755 --- a/tests/test-breadth-mandate.sh +++ b/tests/test-breadth-mandate.sh @@ -8,7 +8,6 @@ # "Task", never the "Agent" name the Claude Agent SDK build actually uses) and the Bash # write-extraction over-match defect ($VAR-containing paths and heredoc-body content read as # real writes) found by dogfooding this hook against a real 15MB transcript. -# PROOFs 22-24 cover the serial-tail mandate; the per-prompt digest's own tests live in tests/test-session-context-mandate.sh. # # The hook's contract (Claude Code Stop-hook protocol, not exit code): a met mandate prints # nothing to stdout and exits 0; an unmet one prints one JSON object on stdout -- @@ -62,9 +61,6 @@ if ! command -v jq >/dev/null 2>&1; then skip "PROOF 19 (setup 1/3): lone prose write trips unslop" "jq not installed" skip "PROOF 19 (setup 2/3): lone TypeScript edit trips typescript-best-practices (unrelated 2nd strike)" "jq not installed" skip "PROOF 19 (the bleed): a THIRD, never-tried mandate still fires after two UNRELATED strikes" "jq not installed" - skip "PROOF 22: three singleton dispatches, no batch -> serial-tail mandate blocks" "jq not installed" - skip "PROOF 23: two singleton dispatches -> serial-tail mandate silent (below threshold)" "jq not installed" - skip "PROOF 24: early 2-in-one-message batch, then three singletons -> serial-tail mandate still blocks (amnesty closed)" "jq not installed" printf 'checks: %d declared, %d ran, %d skipped\n' "$TOTAL" "$RAN" "$SKIPPED" [ "$((RAN + SKIPPED))" -eq "$TOTAL" ] || { printf 'FAIL check accounting\n %d declared check(s) reported nothing\n' "$((TOTAL - RAN - SKIPPED))"; FAIL=1; } [ "$FAIL" -eq 0 ] && echo VERIFIED || echo "VERIFICATION FAILED" @@ -126,7 +122,6 @@ names_piw_(){ printf '%s' "$HOOK_REASON" | grep -qF 'prove-it-works --'; } names_swarm_(){ printf '%s' "$HOOK_REASON" | grep -qF 'swarm --'; } names_unslop_(){ printf '%s' "$HOOK_REASON" | grep -qF 'unslop --'; } names_ts_(){ printf '%s' "$HOOK_REASON" | grep -qF 'typescript-best-practices --'; } -names_serial_(){ printf '%s' "$HOOK_REASON" | grep -qF 'serial dispatch tail --'; } # --- PROOF 1: 5 fixture writes, 1 directory, 1 extension, 0 Task calls ------------------------ # Negative direction. Mechanical repetition (fixtures/case1.json .. case5.json) must not read @@ -485,48 +480,6 @@ else "expected decision=block naming 'prove-it-works --' -- two strikes on unslop+typescript must not silence a mandate neither of them is, got: decision=$HOOK_DECISION reason=[$HOOK_REASON]" fi -# --- PROOF 22: three singleton Task/Agent dispatches, zero batches -> serial-tail blocks -------- -# Positive direction for the serial-tail mandate: three dispatches, each alone in its own -# assistant line (no shared message.id), is the serial loop measured in real transcripts -# (e0cd5a40: 12 singleton dispatches, never a batch). Swarm called and RICK named so no other -# delegation mandate can confound; zero writes so no skill mandate can either. -say_ $'{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"0","name":"Skill","input":{"skill":"swarm"}},{"type":"text","text":"RICK: routing three reviews."}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"1","name":"Task","input":{"prompt":"review a"}}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"2","name":"Task","input":{"prompt":"review b"}}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"3","name":"Agent","input":{"subagent_type":"qa","prompt":"review c","description":"review c"}}]}}' -run_hook_ proof22 -if [ "$HOOK_DECISION" = "block" ] && names_serial_; then - ok "PROOF 22: three singleton dispatches, no batch -> serial-tail mandate blocks" -else - bad "PROOF 22: three singleton dispatches, no batch -> serial-tail mandate blocks" \ - "expected decision=block naming 'serial dispatch tail --', got: decision=$HOOK_DECISION reason=[$HOOK_REASON]" -fi - -# --- PROOF 23: two singleton dispatches -> below the tail-3 threshold, strictly silent ---------- -# Negative direction: two serial dispatches are the shape a session legitimately produces when -# two unrelated asks arrive in two turns (416fb382's post-block tail of 2). Nothing else in the -# fixture can block, so the bar is strict empty stdout, not merely "no serial-tail line". -say_ $'{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"0","name":"Skill","input":{"skill":"swarm"}},{"type":"text","text":"RICK: routing two reviews."}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"1","name":"Task","input":{"prompt":"review a"}}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"2","name":"Task","input":{"prompt":"review b"}}]}}' -run_hook_ proof23 -if [ -z "$HOOK_OUT" ]; then - ok "PROOF 23: two singleton dispatches -> serial-tail mandate silent (below threshold)" -else - bad "PROOF 23: two singleton dispatches -> serial-tail mandate silent (below threshold)" \ - "expected empty stdout, got: decision=$HOOK_DECISION reason=[$HOOK_REASON]" -fi - -# --- PROOF 24: an early real batch, then three singletons -> still blocks (amnesty closed) ------ -# The whole-transcript amnesty itself: fanout_batches=1 from the 2-in-one-message batch, so the -# breadth mandate's own condition (fanout_batches == 0) can never be true again this session -- -# measured in real transcripts (3ce9f899: 3 early batches then 25 serial dispatches unblocked; -# 8959d943: 2 batches of 2 then 9 serial). The serial-tail mandate reads only the dispatches -# AFTER the last batch: three singletons -> block, early batch or not. -say_ $'{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"0","name":"Skill","input":{"skill":"swarm"}},{"type":"text","text":"RICK: batch first."}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"1","name":"Task","input":{"prompt":"review a"}},{"type":"tool_use","id":"2","name":"Agent","input":{"subagent_type":"qa","prompt":"verify","description":"verify"}},{"type":"text","text":"ZEEP and GLOOTIE together."}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"3","name":"Task","input":{"prompt":"review b"}}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"4","name":"Task","input":{"prompt":"review c"}}]}}\n{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"5","name":"Agent","input":{"subagent_type":"qa","prompt":"review d","description":"review d"}}]}}' -run_hook_ proof24 -if [ "$HOOK_DECISION" = "block" ] && names_serial_; then - ok "PROOF 24: early 2-in-one-message batch, then three singletons -> serial-tail mandate still blocks (amnesty closed)" -else - bad "PROOF 24: early 2-in-one-message batch, then three singletons -> serial-tail mandate still blocks (amnesty closed)" \ - "expected decision=block naming 'serial dispatch tail --', got: decision=$HOOK_DECISION reason=[$HOOK_REASON]" -fi - echo printf 'checks: %d declared, %d ran, %d skipped\n' "$TOTAL" "$RAN" "$SKIPPED" if [ "$((RAN + SKIPPED))" -ne "$TOTAL" ]; then diff --git a/tests/test-session-context-mandate.sh b/tests/test-session-context-mandate.sh deleted file mode 100755 index 684689b..0000000 --- a/tests/test-session-context-mandate.sh +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/env bash -# test-session-context-mandate.sh — the hand-runnable reproduction of the MANDATE escalation -# in claude/hooks/inject-session-context.sh's UserPromptSubmit digest. The hook's current code -# (lines ~155-168) reads $TMPDIR/vstack-mandate-$sid and $TMPDIR/vstack-mandate-$sid.delegate -# — files skill-mandate.sh never writes. skill-mandate.sh writes per-mandate suffixes only: -# .unslop .typescript .proveitworks .delegate-breadth .delegate-naming .delegate-swarm -# .delegate-serial. So the escalation line is dead code in production; these tests are written -# RED against that defect and against the digest lacking a FANOUT line, and go green when the -# hook is fixed to (a) read the max of each family's real per-mandate counter files and (b) pin -# the fan-out rule every prompt. -# -# Real-transcript evidence: (1) MANDATE line never rendered in any real session because the -# counters read from non-existent files always stayed 0; (2) serial under-delegation (4.8% -# parallel-batch rate in vstack itself) that the FANOUT re-pin in the digest addresses. -# -# Hook contract: on UserPromptSubmit, emits a JSON object with -# .hookSpecificOutput.additionalContext containing the per-prompt digest (a multi-line string). -# This digest must be kept under 512 bytes total (the grill worst-case in .claude/verify.sh's -# check 18 budget). The tests below parse the additionalContext value via jq and grep it for -# expected lines. -set -uo pipefail - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -hook="$repo_root/claude/hooks/inject-session-context.sh" -SELF="$repo_root/tests/test-session-context-mandate.sh" - -RAN=0 -SKIPPED=0 -FAIL=0 -TOTAL=$(grep -c '^# --- CASE [0-9]' "$SELF") - -ok(){ printf 'ok %s\n' "$1"; RAN=$((RAN+1)); } -bad(){ printf 'FAIL %s\n %s\n' "$1" "$2"; FAIL=1; RAN=$((RAN+1)); } -skip(){ printf 'skip %s (%s)\n' "$1" "$2"; SKIPPED=$((SKIPPED+1)); } - -if ! command -v jq >/dev/null 2>&1; then - skip "CASE 1: seeded delegate-family counter renders the MANDATE line" "jq not installed" - skip "CASE 2: seeded skill-family counter renders the MANDATE line" "jq not installed" - skip "CASE 3: no counter files -> MANDATE line absent" "jq not installed" - skip "CASE 4: digest pins the fan-out rule every prompt" "jq not installed" - skip "CASE 5: unconditional digest stays under 512-byte budget" "jq not installed" - skip "CASE 6: worst case (grill + both families at cap) stays under budget" "jq not installed" - printf 'checks: %d declared, %d ran, %d skipped\n' "$TOTAL" "$RAN" "$SKIPPED" - [ "$((RAN + SKIPPED))" -eq "$TOTAL" ] || { printf 'FAIL check accounting\n %d declared check(s) reported nothing\n' "$((TOTAL - RAN - SKIPPED))"; FAIL=1; } - [ "$FAIL" -eq 0 ] && echo VERIFIED || echo "VERIFICATION FAILED" - exit "$FAIL" -fi - -if [ ! -x "$hook" ]; then - bad "session-context-mandate proofs" "$hook is missing or not executable" - printf 'checks: %d declared, %d ran, %d skipped\n' "$TOTAL" "$RAN" "$SKIPPED" - echo "VERIFICATION FAILED" - exit 1 -fi - -WORK="$(mktemp -d "${TMPDIR:-/tmp}/vstack-ctx-mandate.XXXXXX")" -trap 'rm -rf "$WORK"' EXIT - -# Invocation helper: runs the hook with a UserPromptSubmit event, reads the digest context. -# The hook reads counter files under ${TMPDIR:-/tmp}, so we set TMPDIR="$WORK" for isolation. -# Takes two invocations in some cases: the hook is stateless apart from the grill first-seen -# marker, which the SECOND call sees as already-marked. Call order therefore matters only for -# grill cases. -run_ctx_(){ # - _payload=$(printf '{"hook_event_name":"UserPromptSubmit","session_id":"%s","prompt":"%s"}' "$1" "$2") - printf '%s' "$_payload" | TMPDIR="$WORK" bash "$hook" 2>/dev/null | jq -r '.hookSpecificOutput.additionalContext // empty' 2>/dev/null -} - -run_ctx_bytes_(){ # — return byte count of raw stdout - _payload=$(printf '{"hook_event_name":"UserPromptSubmit","session_id":"%s","prompt":"%s"}' "$1" "$2") - printf '%s' "$_payload" | TMPDIR="$WORK" bash "$hook" 2>/dev/null | wc -c | tr -d ' ' -} - -# --- CASE 1: seeded delegate-family counter renders the MANDATE line ------------------------- -# Dead-code defect: the hook reads vstack-mandate-scm1.delegate (non-existent) instead of the -# real per-mandate files like vstack-mandate-scm1.delegate-breadth that skill-mandate.sh writes. -# When fixed to read the max of real counter files, a seeded delegate-breadth file must render -# the MANDATE line showing the escalation counters. Direction: positive, guard against silent -# zero-read on wrong filenames. -printf '2\n' > "$WORK/vstack-mandate-scm1.delegate-breadth" -CTX=$(run_ctx_ scm1 "hi") -if printf '%s' "$CTX" | grep -qF 'MANDATE skill=0/2 delegate=2/2'; then - ok "CASE 1: seeded delegate-family counter renders the MANDATE line" -else - bad "CASE 1: seeded delegate-family counter renders the MANDATE line" \ - "expected CTX to contain 'MANDATE skill=0/2 delegate=2/2', got: [$CTX]" -fi - -# --- CASE 2: seeded skill-family counter renders the MANDATE line ---------------------------- -# Same dead-code defect: the hook reads vstack-mandate-scm2 (non-existent generic file) instead -# of real per-mandate skill counters (.unslop, .typescript, .proveitworks). When fixed, a seeded -# skill-family counter must render the MANDATE line showing the escalation counters. Direction: -# positive, guard against silent zero-read on wrong filenames. -printf '1\n' > "$WORK/vstack-mandate-scm2.unslop" -CTX=$(run_ctx_ scm2 "hi") -if printf '%s' "$CTX" | grep -qF 'MANDATE skill=1/2 delegate=0/2'; then - ok "CASE 2: seeded skill-family counter renders the MANDATE line" -else - bad "CASE 2: seeded skill-family counter renders the MANDATE line" \ - "expected CTX to contain 'MANDATE skill=1/2 delegate=0/2', got: [$CTX]" -fi - -# --- CASE 3: no counter files -> MANDATE line absent ---------------------------------------- -# Negative direction: the common case in most sessions. No counter files seeded, so both -# families read as 0/0. The digest must stay silent (MANDATE line absent) — most prompts in most -# sessions never tripped either mandate, and the byte budget (check 18) is a hard cap on every -# prompt, so a line that always rendered would blow the budget for the common case. -CTX=$(run_ctx_ scm3 "hi") -if ! printf '%s' "$CTX" | grep -qF MANDATE; then - ok "CASE 3: no counter files -> MANDATE line absent" -else - bad "CASE 3: no counter files -> MANDATE line absent" \ - "expected CTX to NOT contain MANDATE, got: [$CTX]" -fi - -# --- CASE 4: the digest pins the fan-out rule every prompt ----------------------------------- -# Defect: the digest today lacks a FANOUT line re-pinning the fan-out rule on every prompt. -# Parallel-batch rate in real sessions is 4.8%, a signal that the rule is drifting. When fixed, -# the digest must emit a FANOUT line describing the fan-out contract, a DELEGATE line naming the -# mechanical/judgment split, AND a line about batching in ONE message. Direction: positive, -# guard the digest's completeness and the rule's re-pinning every turn. -CTX=$(run_ctx_ scm4 "hi") -has_fanout=$(printf '%s' "$CTX" | grep -qF 'FANOUT:' && echo 1 || echo 0) -has_delegate=$(printf '%s' "$CTX" | grep -qF 'DELEGATE: mechanical' && echo 1 || echo 0) -has_batch=$(printf '%s' "$CTX" | grep -qF 'ALL Agent calls in ONE message' && echo 1 || echo 0) -if [ "$has_fanout" = 1 ] && [ "$has_delegate" = 1 ] && [ "$has_batch" = 1 ]; then - ok "CASE 4: the digest pins the fan-out rule every prompt" -else - bad "CASE 4: the digest pins the fan-out rule every prompt" \ - "expected FANOUT (got $has_fanout), DELEGATE (got $has_delegate), batch rule (got $has_batch) in digest: [$CTX]" -fi - -# --- CASE 5: unconditional digest stays inside the verify.sh check-18 budget ------------------ -# Boundary test: the digest (grill + MANDATE + FANOUT) is pinned every prompt on every machine, -# so it must fit within the byte cap check 18 measures (512 bytes worst-case). The hook can emit -# empty additionalContext (0 bytes) when grill and MANDATE are both silent, or up to ~512 when -# both are armed. This case has no counters seeded and grill disabled by short prompt, so it -# measures the unconditional part (TOKENS + DELEGATE + FANOUT + batch rule). Direction: -# negative, guard against budget overflow from new digest lines. -RAW_BYTES=$(run_ctx_bytes_ scm5 "hi") -if [ "$RAW_BYTES" -ge 128 ] && [ "$RAW_BYTES" -le 512 ]; then - ok "CASE 5: unconditional digest stays inside the verify.sh check-18 budget" -else - bad "CASE 5: unconditional digest stays inside the verify.sh check-18 budget" \ - "expected RAW_BYTES between 128 and 512, got: $RAW_BYTES" -fi - -# --- CASE 6: worst case (grill + both MANDATE families at cap) stays under 512 ---------------- -# Boundary test, the worst case: grill fires + both mandate families at 2/2 (the cap). The hook -# must stay under 512 bytes even with all three pieces armed. Seed both families at max -# (2 counters each), build a 400-char prompt to arm the grill (>= 320 chars), measure the FIRST -# invocation's raw byte count (to catch the grill armed, not cached). Also assert the output -# contains 'GRILL: run the grill-me skill' to prove grill actually fired (worst case is real). -# Direction: positive boundary, guard against budget overflow under worst-case load. -# NOTE: measure bytes on FIRST invocation when grill fires; grill marker is cached so second -# call would see it as already-marked. Fresh session id, one invocation only. -printf '2\n' > "$WORK/vstack-mandate-scm6.delegate-breadth" -printf '2\n' > "$WORK/vstack-mandate-scm6.unslop" -_p="" -_i=0 -while [ $_i -lt 400 ]; do - _p="${_p}x" - _i=$((_i+1)) -done -_payload_6=$(printf '{"hook_event_name":"UserPromptSubmit","session_id":"scm6","prompt":"%s"}' "$_p") -_out_6=$(printf '%s' "$_payload_6" | TMPDIR="$WORK" bash "$hook" 2>/dev/null) -_bytes_out=$(printf '%s' "$_out_6" | wc -c | tr -d ' ') -_ctx=$(printf '%s' "$_out_6" | jq -r '.hookSpecificOutput.additionalContext // empty' 2>/dev/null) -has_grill=$(printf '%s' "$_ctx" | grep -qF 'GRILL: run the grill-me skill' && echo 1 || echo 0) -if [ "$_bytes_out" -le 512 ] && [ "$has_grill" = 1 ]; then - ok "CASE 6: worst case (grill + both MANDATE families at cap) stays under budget" -else - bad "CASE 6: worst case (grill + both MANDATE families at cap) stays under budget" \ - "expected bytes <= 512 (got $_bytes_out) and GRILL present (got $has_grill)" -fi - -echo -printf 'checks: %d declared, %d ran, %d skipped\n' "$TOTAL" "$RAN" "$SKIPPED" -if [ "$((RAN + SKIPPED))" -ne "$TOTAL" ]; then - printf 'FAIL check accounting\n %d declared check(s) reported nothing\n' "$((TOTAL - RAN - SKIPPED))" - FAIL=1 -fi -[ "$FAIL" -eq 0 ] && [ "$RAN" -gt 0 ] && echo VERIFIED || echo "VERIFICATION FAILED" -[ "$FAIL" -eq 0 ] && [ "$RAN" -gt 0 ] -exit $? From f8d784c80080aeaafca8e5cc9e5a8b5feeea36aa Mon Sep 17 00:00:00 2001 From: vk Date: Wed, 2 Sep 2026 03:01:40 +0530 Subject: [PATCH 3/3] fix(format.sh): stop matching devDependencies.prettier as a config find_prettier_cfg() gated package.json on `grep -q '"prettier"[[:space:]]*:'`, which matches `"prettier": "^3.4.2"` inside devDependencies/dependencies just as happily as a real top-level "prettier" key. package.json is tried before .prettierrc in the same directory (matching prettier's own cosmiconfig search order), so a repo with prettier merely listed as a devDependency had its package.json win that slot, get handed to `prettier --config package.json`, find no real "prettier" field, and silently fall back to built-in defaults -- the sibling .prettierrc, with the project's real settings, was never reached. Replace the grep with pkg_has_top_level_prettier(), which parses the JSON via node (found through find_bin, falling back to `command -v node`) and checks Object.prototype.hasOwnProperty on the top-level key. No node, or a package.json that fails to parse, falls through via `continue` so the walk keeps going toward .prettierrc -- degrading to the dedicated config file is the safe direction, silently formatting with wrong defaults is not. Config filename order and the --config argument are unchanged; the cfg_has_plugins/is_project_trusted gate downstream still sees exactly which file will be loaded, and tests/repro/formatter-config.sh still passes. Adds tests/repro/formatter-config-devdep.sh: builds a repo with a devDependency-only "prettier" mention plus a sibling .prettierrc, proves the fixed hook picks .prettierrc, and re-runs the same case against the newest committed format.sh predating the pkg_has_top_level_prettier guard (derived from git history, not a pinned SHA) to prove the case is not vacuous. --- claude/hooks/format.sh | 28 +++++- tests/repro/formatter-config-devdep.sh | 133 +++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 1 deletion(-) create mode 100755 tests/repro/formatter-config-devdep.sh diff --git a/claude/hooks/format.sh b/claude/hooks/format.sh index 812301b..22707b3 100755 --- a/claude/hooks/format.sh +++ b/claude/hooks/format.sh @@ -104,6 +104,28 @@ is_project_trusted() { else h=$(sha256sum "$v" 2>/dev/null | cut -d' ' -f1); fi [ -n "$h" ] && grep -qxF "$h $v" "$ts" 2>/dev/null } +# package.json only *configures* prettier via a top-level "prettier" key. A grep for the +# substring `"prettier":` cannot tell that key apart from the identically-shaped entry +# `"prettier": "^3.4.2"` sitting in dependencies/devDependencies -- a version pin, not a +# config -- so it matched there too, won the package.json slot ahead of a sibling .prettierrc +# in the same directory (package.json is first in the name list, matching prettier's own +# cosmiconfig search order), and got handed to `prettier --config package.json`. Cosmiconfig +# then finds no real "prettier" field in it and silently falls back to prettier's built-in +# defaults instead of erroring -- so the file gets reformatted to the wrong style, and the +# .prettierrc this loop would otherwise have reached next is never tried. Parse the JSON and +# check for a genuine top-level key instead of pattern-matching the source text. +pkg_has_top_level_prettier() { # $1: package.json path + nb=$(find_bin node) || nb=$(command -v node) || return 1 + "$nb" -e ' + const fs = require("fs"); + try { + const pkg = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); + process.exit(Object.prototype.hasOwnProperty.call(pkg, "prettier") ? 0 : 1); + } catch (e) { + process.exit(1); + } + ' "$1" 2>/dev/null +} find_prettier_cfg() { d="$dir" while [ "$d" != "/" ] && [ -n "$d" ]; do @@ -114,7 +136,11 @@ find_prettier_cfg() { p="$d/$name" [ -e "$p" ] || continue if [ "$name" = package.json ]; then - grep -q '"prettier"[[:space:]]*:' "$p" 2>/dev/null || continue + # No node, or a package.json that fails to parse: fall through via `continue` so the + # loop keeps walking toward .prettierrc in this same directory. Degrading to the + # dedicated config file is the safe direction; silently formatting with built-in + # defaults because we couldn't check is not. + pkg_has_top_level_prettier "$p" || continue fi printf '%s\n' "$p" return 0 diff --git a/tests/repro/formatter-config-devdep.sh b/tests/repro/formatter-config-devdep.sh new file mode 100755 index 0000000..dbb5795 --- /dev/null +++ b/tests/repro/formatter-config-devdep.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# tests/repro/formatter-config-devdep.sh +# +# format.sh's find_prettier_cfg() walks up from the edited file's directory and, for each +# directory, tries config filenames in order with package.json FIRST -- matching prettier's own +# cosmiconfig search order. To decide whether a given package.json actually configures prettier, +# it used to grep it for `"prettier"[[:space:]]*:`. That regex matches the source text of +# `"prettier": "^3.4.2"` sitting inside "devDependencies" just as happily as a real top-level +# "prettier" key -- a version pin is not a config, but the grep cannot tell them apart. So a +# perfectly ordinary package.json with prettier merely listed as a devDependency won the +# package.json slot ahead of a sibling .prettierrc in the very same directory, and the hook ran +# `prettier --config package.json --write`. Prettier finds no real "prettier" field in that +# file, silently falls back to its own built-in defaults (this is not an error prettier raises; +# cosmiconfig just treats "found the file, no matching field" as "nothing configured"), and +# reformats the file to those defaults -- the repo's real .prettierrc is never reached, because +# find_prettier_cfg already returned on the first name it tried. +# +# This builds one throwaway repo with a package.json whose only "prettier" mention is a +# devDependency version range, and a sibling .prettierrc with {"singleQuote": true}. It writes a +# double-quoted .ts file and fires format.sh at it exactly the way Claude Code's PostToolUse +# hook would. Then, to prove this is not a no-op, it re-runs the identical case against the +# newest committed blob of format.sh that predates the parse-instead-of-grep fix -- derived by +# walking this file's history newest-first, the same technique tests/repro/formatter-config.sh +# uses for its own no-op proof, so the baseline cannot rot once the fix is committed, rebased, +# or reverted. +# +# Exit 0 -- fixed: .prettierrc's singleQuote wins, and the pre-fix blob demonstrably picked +# package.json instead (proving the test measures something real). +# Exit 1 -- the devDependency version range still wins, or the harness itself is broken. +set -uo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "$SCRIPT_DIR/../.." && pwd) +HOOK="$REPO_ROOT/claude/hooks/format.sh" + +FAIL=0 +note(){ printf '%s\n' "$1"; } +ok(){ printf 'ok %s\n' "$1"; } +bad(){ printf 'FAIL %s\n' "$1"; FAIL=1; } + +[ -x "$HOOK" ] || { echo "FAIL $HOOK missing or not executable"; exit 1; } +command -v npm >/dev/null 2>&1 || { echo "skip npm not installed on this host; cannot install a real prettier to exercise config resolution"; exit 0; } +command -v git >/dev/null 2>&1 || { echo "skip git not installed; cannot fetch the pre-fix blob for the no-op proof"; exit 0; } + +# Own sandbox variable, never $HOME itself -- guard-destructive.sh blocks rm -rf "$HOME"-shaped +# commands even with HOME reassigned, and everything below cleans up its own tree, not HOME. +FMT_SANDBOX=$(mktemp -d "${TMPDIR:-/tmp}/vstack-formatter-devdep.XXXXXX") +FMT_SANDBOX=$(cd "$FMT_SANDBOX" && pwd) # normalize away a doubled slash from a trailing-slash TMPDIR +cleanup(){ rm -rf "$FMT_SANDBOX"; } +trap cleanup EXIT + +VENDOR="$FMT_SANDBOX/vendor" +mkdir -p "$VENDOR" + +note "-- installing prettier@3.9.6 once, shared across repo copies --" +if ! npm install prettier@3.9.6 --no-save --prefix "$VENDOR" >"$FMT_SANDBOX/npm-install.log" 2>&1; then + echo "skip npm install prettier failed (offline?); see $FMT_SANDBOX/npm-install.log before cleanup races it" + cat "$FMT_SANDBOX/npm-install.log" + exit 0 +fi +[ -x "$VENDOR/node_modules/.bin/prettier" ] || { echo "skip prettier install did not produce an executable; cannot exercise config resolution"; exit 0; } + +make_repo() { # $1: repo dir + d="$1" + mkdir -p "$d" + ln -s "$VENDOR/node_modules" "$d/node_modules" + # The only "prettier" mention in this package.json is a devDependency version range -- never + # a config. .prettierrc is the real config, and it disagrees with prettier's built-in default + # (singleQuote:false) so the two outcomes are distinguishable by inspecting quote style alone. + cat > "$d/package.json" <<'EOF' +{ + "name": "devdep-repro", + "private": true, + "devDependencies": { + "prettier": "^3.4.2" + } +} +EOF + cat > "$d/.prettierrc" <<'EOF' +{"singleQuote": true, "printWidth": 90} +EOF + printf 'const x = "hello";\n' > "$d/target.ts" +} + +run_hook() { # $1: hook script, $2: repo dir + f="$2/target.ts" + echo '{"tool_input":{"file_path":"'"$f"'"}}' | env HOME="$FMT_SANDBOX" CLAUDE_PROJECT_DIR="$2" "$1" >/dev/null 2>&1 +} + +# --- the claimed defect: fixed hook must prefer .prettierrc over the devDependency line ------ +REPO_FIXED="$FMT_SANDBOX/fixed-repo" +make_repo "$REPO_FIXED" +run_hook "$HOOK" "$REPO_FIXED" +if grep -q "'hello'" "$REPO_FIXED/target.ts"; then + ok ".prettierrc won: devDependency version range no longer mistaken for a prettier config" +elif grep -q '"hello"' "$REPO_FIXED/target.ts"; then + bad "HOLE OPEN: package.json's devDependency line still won the package.json slot -- file was formatted (or left unformatted) with built-in defaults, not .prettierrc" +else + bad "target.ts contents unrecognized after running the fixed hook: $(cat "$REPO_FIXED/target.ts" 2>/dev/null)" +fi + +# --- no-op proof: same case against the newest committed blob predating the parse fix -------- +PREFIX_HOOK="$FMT_SANDBOX/format-prefix.sh" +# The guard function's own name is the discriminator, anchored on the function rather than a +# pinned SHA -- a SHA is a second place to remember, and the one that goes stale first. +GUARD_MARK='pkg_has_top_level_prettier' +PREFIX_SHA='' +for _c in $(git -C "$REPO_ROOT" log --format=%H -- claude/hooks/format.sh); do + if ! git -C "$REPO_ROOT" show "$_c:claude/hooks/format.sh" 2>/dev/null | grep -q "$GUARD_MARK"; then + PREFIX_SHA="$_c" + break + fi +done +if [ -z "$PREFIX_SHA" ]; then + bad "no commit of claude/hooks/format.sh predates the $GUARD_MARK guard -- the no-op proof has no baseline to revert to, so this test cannot show it measures anything" + exit 1 +fi +if ! git -C "$REPO_ROOT" show "$PREFIX_SHA:claude/hooks/format.sh" > "$PREFIX_HOOK" 2>/dev/null; then + bad "could not read $PREFIX_SHA:claude/hooks/format.sh to build the no-op proof" + exit 1 +fi +chmod +x "$PREFIX_HOOK" +REPO_REVERT="$FMT_SANDBOX/revert-repo" +make_repo "$REPO_REVERT" +run_hook "$PREFIX_HOOK" "$REPO_REVERT" +if grep -q '"hello"' "$REPO_REVERT/target.ts" && ! grep -q "'hello'" "$REPO_REVERT/target.ts"; then + ok "no-op proof: format.sh at $PREFIX_SHA does mistake the devDependency line for a config -- this test is not vacuous" +else + bad "no-op proof failed: reverting to $PREFIX_SHA (the newest format.sh without $GUARD_MARK) did NOT reproduce the false match -- this test cannot detect the bug it claims to" +fi + +[ "$FAIL" -eq 0 ] && echo "FORMATTER PACKAGE.JSON FALSE MATCH: CLOSED" || echo "FORMATTER PACKAGE.JSON FALSE MATCH: OPEN" +exit "$FAIL"