From 106d1e5efa92cffd58501b9e5d82e4fb6e87b9a5 Mon Sep 17 00:00:00 2001 From: Younggi Choi <74581798+choiyounggi@users.noreply.github.com> Date: Fri, 21 Aug 2026 23:41:35 +0900 Subject: [PATCH 1/3] knowledge: session context/token budget (infrastructure/agent-orchestration) Add a sourced wiki page on token-efficient context management for long-lived coordinator/worker agent sessions: cache pricing mechanics, /compact vs /clear at phase boundaries, delegating visual verification to subagents (with the current image patch-token formula), bounding tool output, and sizing orchestrated runs against measured Anthropic multipliers. Routes into wiki/infrastructure/index.md and INDEX.md. t1 of 3 (t1-wiki-token-efficiency): t2 will cite this page's slug from the orchestrate skill docs. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WjskPUNcUjsE9qzcxhuLGv --- INDEX.md | 2 +- .../session-context-token-budget.md | 107 ++++++++++++++++++ wiki/infrastructure/index.md | 1 + 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 wiki/infrastructure/agent-orchestration/session-context-token-budget.md diff --git a/INDEX.md b/INDEX.md index 8f59d1a..dd39256 100644 --- a/INDEX.md +++ b/INDEX.md @@ -14,7 +14,7 @@ follow the cross-pointers in their index or take the next matching seeded domain | [databases](wiki/databases/index.md) | **seeded** | Designing schemas/tables/keys, choosing or evaluating indexes, writing or optimizing queries, choosing transaction/isolation behavior, surveying live data to derive a rule, verifying additive migrations | | [backend](wiki/backend/index.md) | **seeded** | Server-side application code — language-agnostic (`common/`: API contracts, call-site enumeration before a contract change, idempotency, JWT, timeouts/retries, caching, jobs, transactions in app code, shared state/pools, errors, consuming LLM APIs (completion validation, context budgeting), MAPE-aligned point-prediction calibration, consuming external-API responses, externally-owned defaults, object-storage references, sync-vs-async integration choice, WebSocket/SSE connection lifecycle) plus stack subtrees: `java/` (JPA, Spring proxies, JVM threads/memory), `node/` (event loop, promises, runtime validation, shutdown), `python/` (GIL/asyncio, pydantic, WSGI/ASGI workers, language traps, packaging data files with `importlib.resources`) | | [frontend](wiki/frontend/index.md) | **seeded** | Web UI code: state placement, rendering performance, in-UI data fetching (races, infinite scroll), auth token handling, forms, XSS-safe output, accessibility, agent-facing tool surfaces (WebMCP) | -| [infrastructure](wiki/infrastructure/index.md) | **seeded** | CI/CD pipelines, secrets in build/deploy, container image builds, rollout/rollback strategy, observability (logs/metrics/alerting), per-environment/path-valued config, multi-agent orchestration (worker liveness signals, shared run state, tmux pane delivery, completion gates, worktree-isolated workers) | +| [infrastructure](wiki/infrastructure/index.md) | **seeded** | CI/CD pipelines, secrets in build/deploy, container image builds, rollout/rollback strategy, observability (logs/metrics/alerting), per-environment/path-valued config, multi-agent orchestration (worker liveness signals, shared run state, tmux pane delivery, completion gates, worktree-isolated workers, session context/token budgeting) | | [testing](wiki/testing/index.md) | **seeded** | Writing or structuring automated tests: level choice, cases/assertions, test data, mock decisions, flaky tests (release-process quality → qa) | | [qa](wiki/qa/index.md) | **seeded** | Release-quality process: release gates, regression scoping, bug reports, severity/priority triage, exploratory testing (guarded-path coverage, override matrices), scope-purity gates, sourcing deliverable documents from generated artifacts, verifying the quantitative claims in a document before publishing it, automated verification of document deliverables (spec/RFC gates) (writing automated test code → testing) | | [debugging](wiki/debugging/index.md) | **seeded** | Diagnosing a failure — finding what is wrong and why: reproducing, bisection, hypothesis testing, traces/logs, intermittent failures (fixing the diagnosed fault → its owning domain) | diff --git a/wiki/infrastructure/agent-orchestration/session-context-token-budget.md b/wiki/infrastructure/agent-orchestration/session-context-token-budget.md new file mode 100644 index 0000000..c23e719 --- /dev/null +++ b/wiki/infrastructure/agent-orchestration/session-context-token-budget.md @@ -0,0 +1,107 @@ +--- +id: infrastructure-agent-orchestration-session-context-token-budget +domain: infrastructure +category: agent-orchestration +applies_to: [general] +confidence: verified +sources: + - https://platform.claude.com/docs/en/build-with-claude/prompt-caching + - https://code.claude.com/docs/en/costs + - https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents + - https://code.claude.com/docs/en/best-practices + - https://code.claude.com/docs/en/sub-agents + - https://platform.claude.com/docs/en/build-with-claude/vision + - https://www.anthropic.com/engineering/multi-agent-research-system +last_verified: 2026-08-21 +related: [infrastructure-agent-orchestration-worktree-isolated-workers, infrastructure-agent-orchestration-usage-limit-paused-workers, backend-common-llm-context-window-budget] +--- + +# Token Budget for Long-Lived Coordinator and Worker Agent Sessions + +## When this applies + +Planning or running a long-lived coordinator or worker agent session and deciding +when to compact or clear context; a run's cost is dominated by cache-read tokens +rather than any single call; a screenshot or a large file read is about to enter a +long-lived session's history; choosing slot counts or per-phase token budgets for +an orchestrated run. + +## Do this + +1. **Track accumulated context as the cost driver, not turn count.** Cache reads + cost ~0.1x the base input token price; 5-minute cache writes cost ~1.25x, 1-hour + writes cost 2x. Every call resends the full conversation history (billed at the + cached rate for the cached portion), so cost per turn grows with how much + context has accumulated — a session's total cost grows faster than linearly with + turn count because each of its turns re-bills a history that keeps growing. + +2. **Reset context at phase boundaries, not only when the window is full.** Run + `/compact` between milestones inside one task (optionally with instructions on + what to preserve — architectural decisions, unresolved bugs, implementation + details survive; redundant tool output does not) and `/clear` when switching to + an unrelated task. Waiting until the window is nearly full to compact means the + session already paid a quality cost first: accuracy at recalling earlier context + degrades as the window fills, an effect distinct from the token-cost mechanism + in directive 1. + +3. **Delegate visual verification to a throwaway subagent.** A subagent runs in its + own isolated context window and returns only its summary to the caller, so + verbose search results, logs, or file contents it produced never enter the + long-lived session. This matters most for images: once a screenshot enters a + session's history, its bytes are resent in full on every subsequent turn, so + check it with a subagent that returns a text verdict instead of Reading it + directly into a coordinator or worker's own context. When an image does need to + stay in context across many turns, upload it through the Files API and + reference it by `file_id` — the payload then stays flat regardless of how many + images accumulate. Size the check itself against the current image-token + formula, `⌈width / 28⌉ × ⌈height / 28⌉` visual tokens (28x28-pixel patches) — an + older `(width × height) / 750` figure circulates in outdated summaries and no + longer matches the documented pricing. + +4. **Bound tool output before it reaches context.** Prefer a search mode that + returns matching filenames over one that returns full matched content; read a + large file by range (offset/limit) instead of in full when only part of it is + needed; pipe verbose command output through a filter (e.g. `grep` + `head`) so a + 10,000-line log becomes the handful of matching lines instead of the whole file. + +5. **Size an orchestrated run's slot count against its architecture, not habit.** + Anthropic's own multi-agent research system measured multi-agent work (an + orchestrator plus parallel subagents, each with an independent context window) + at roughly 15x the tokens of a single chat interaction, with each individual + agent in that fleet running roughly 4x a single chat interaction on its own; + Claude Code's parallel-instance "agent teams" feature separately measured + roughly 7x when teammates run in plan mode, since token usage there scales with + team size and how long each teammate stays active. Both sources tie this + multiplier to justification, not prohibition: reserve the parallel/orchestrated + shape for tasks whose value justifies it or whose scope exceeds one context + window, keep spawn prompts focused since everything in one adds to that + teammate's context from the start, and end a worker's session once its phase + completes rather than leaving it idle and still billing. + +## Edge cases + +| Case | Then | +|------|------| +| A subagent's result must reach the main session | Have it return only a text summary — the verbose intermediate output stays in the subagent's own context and is never itself pasted into the caller | +| Several images must stay referenceable across a long session | Upload each through the Files API and pass its `file_id` instead of inlining base64 bytes on every turn | +| The session is a single chat interaction rather than an orchestrated run | The 4x/15x/7x multipliers above do not apply; the accumulated-context mechanism in directive 1 still does | + +## Instead of + +| If you are about to | Do this instead | Why | +|---------------------|-----------------|-----| +| Let a long session run until the context window is nearly full before compacting | Compact at each phase or milestone boundary | Recall accuracy degrades as the window fills, so waiting until it is full means the session already paid that cost before the reset | +| Read a large log or test-output file directly into the main session to find one error | Filter it first (grep/head) or delegate the search to a subagent | The main session then holds the few matching lines instead of the whole file | +| Treat an image already in conversation history as a one-time cost | Treat it as a recurring per-turn cost for the rest of that session | Base64 image bytes are part of the full conversation payload resent on every subsequent turn | +| Pick a fleet size for an orchestrated run by habit or convenience | Size it against the measured 4x (single agent) / 15x (multi-agent) / 7x (agent teams in plan mode) multipliers and the task's value | An orchestrated run's cost is structurally higher than a single session's, independent of how efficiently any one agent in it runs | + +## Sources + +- https://platform.claude.com/docs/en/build-with-claude/prompt-caching — cache read (~0.1x) and cache write (~1.25x for 5-minute, 2x for 1-hour) token price multipliers +- https://code.claude.com/docs/en/costs — full conversation history resent (at the cached rate) on every request; `/clear` between unrelated tasks and `/compact ` mid-task; delegating verbose operations (tests, doc fetches, log processing) to subagents or filtering hooks (e.g. `grep`+`head`) so only matching lines reach context; agent-team cost scaling with team size and teammate duration (~7x in plan mode) +- https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents — context rot (recall accuracy decreasing as the context window fills); compaction should preserve architectural decisions, unresolved bugs, and implementation details while discarding redundant tool output +- https://code.claude.com/docs/en/best-practices — `/clear` and `/compact` usage guidance; performance degrading as the context window fills +- https://code.claude.com/docs/en/sub-agents — subagents run in an isolated context window and return only a summary to the caller +- https://platform.claude.com/docs/en/build-with-claude/vision — current image-token formula `⌈width / 28⌉ × ⌈height / 28⌉` (28x28-pixel patches); base64 images resent in full on every turn; Files API `file_id` keeps payload size flat +- https://www.anthropic.com/engineering/multi-agent-research-system — multi-agent systems measured at ~15x the tokens of a single chat interaction (~4x per individual agent); reserved for high-value, parallelizable, or context-exceeding tasks +- Field evidence 2026-08-21 (dev-loop orchestration run): the coordinator session ended a run at approximately 501k tokens of accumulated context over 593 API calls, with approximately 164M cumulative cache-read tokens billed across the run; a post-run review found approximately 2.2MB of screenshot PNGs had been Read directly into the coordinator's own context over the run instead of delegated to a subagent per directive 3 — a measured instance of that directive's cost when skipped diff --git a/wiki/infrastructure/index.md b/wiki/infrastructure/index.md index fe918e3..bc8d923 100644 --- a/wiki/infrastructure/index.md +++ b/wiki/infrastructure/index.md @@ -20,6 +20,7 @@ Match your situation to a "load when" line; load only matching pages. | [unattended-worker-questions](agent-orchestration/unattended-worker-questions.md) | A worker agent raises a question through its own interactive UI (a numbered chooser, a confirmation/trust/re-auth screen) with no human at that terminal; a worker is flagged stalled with a live terminal and no task-level error; a worker reports a decision it assumed rather than asked; designing the channel a worker uses to ask its coordinator for a decision | | [usage-limit-paused-workers](agent-orchestration/usage-limit-paused-workers.md) | Several workers billed to one account go quiet within minutes of each other while every liveness check passes; a worker's terminal shows a `You've hit your session/weekly/Opus limit · resets …` notice; deciding whether to restart, replace, or wait on a worker with no task-level error; writing the prompt that resumes a worker after a usage window resets | | [worktree-isolated-workers](agent-orchestration/worktree-isolated-workers.md) | Authoring the brief/output contract for parallel workers each confined to its own git worktree; workers stall at the same phase with no task-level error; deciding where shared or produced artifacts live and which direction (read vs write) a worktree guardrail stops; a guardrail escalates on read-only access to another worktree; the isolation guard is a Bash-command hook while workers also edit files with native Edit/Write tools | +| [session-context-token-budget](agent-orchestration/session-context-token-budget.md) | Planning or running long-lived coordinator/worker agent sessions and deciding when to compact or clear context; a run's cost is dominated by cache reads; screenshots or large file reads are entering a long-lived session; choosing slot counts / per-phase token budgets for an orchestrated run | ## ci-cd From be6569f181f86e9f671955bcfd1a6e4d655cca06 Mon Sep 17 00:00:00 2001 From: Younggi Choi <74581798+choiyounggi@users.noreply.github.com> Date: Fri, 21 Aug 2026 23:52:36 +0900 Subject: [PATCH 2/3] feat(orchestrate): add token-report.sh for per-session token usage audit Ships token-report.sh: reports per-session and total token usage (calls, output tokens, cache hit %, final/peak context) from Claude Code session transcript JSONL, so an orchestration run can audit its own token efficiency at teardown. POSIX sh + jq, fail-fast validation (exit 2 usage, exit 4 malformed LO_CTX_WARN), with a synthetic-fixture bats suite. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01PucyFRX7yfYniePZa9oVaA --- skills/orchestrate/scripts/token-report.sh | 202 ++++++++++++++++++ tests/fixtures/token-report/edge/empty.jsonl | 0 .../token-report/edge/zero-usage.jsonl | 2 + tests/fixtures/token-report/session-a.jsonl | 5 + tests/fixtures/token-report/session-b.jsonl | 2 + tests/token-report.bats | 107 ++++++++++ 6 files changed, 318 insertions(+) create mode 100755 skills/orchestrate/scripts/token-report.sh create mode 100644 tests/fixtures/token-report/edge/empty.jsonl create mode 100644 tests/fixtures/token-report/edge/zero-usage.jsonl create mode 100644 tests/fixtures/token-report/session-a.jsonl create mode 100644 tests/fixtures/token-report/session-b.jsonl create mode 100644 tests/token-report.bats diff --git a/skills/orchestrate/scripts/token-report.sh b/skills/orchestrate/scripts/token-report.sh new file mode 100755 index 0000000..34c3e28 --- /dev/null +++ b/skills/orchestrate/scripts/token-report.sh @@ -0,0 +1,202 @@ +#!/bin/sh +# token-report.sh — per-session + total token usage report from Claude Code +# session transcript JSONL files (Claude Code writes these under +# ~/.claude/projects//). Read at orchestration run teardown to +# audit a run's token efficiency: calls, output tokens, cache hit %, and +# final/peak context per session. +# +# usage: token-report.sh [--cwd ] ... +# a .jsonl transcript, or a directory (its *.jsonl +# files, non-recursive). At least one is required. +# An ABSOLUTE file-or-dir is used as-is; a RELATIVE +# one is resolved against the --cwd mapping below (or +# taken as relative to the caller's cwd if --cwd was +# not given). +# --cwd an absolute project/worktree path, mapped to +# ~/.claude/projects/ — the directory Claude +# Code writes that project's session transcripts +# under. is with every `/` +# and `.` replaced by `-`. Typical use at teardown: +# `token-report.sh --cwd "$PWD" .` +# +# exit 0 report printed: table on stdout, peak-context warnings on stderr +# exit 2 usage error — no sources given, a source path that does not +# exist, or (when --cwd is given) its mapped directory does not +# exist (stderr names the attempted path) +# exit 4 malformed LO_CTX_WARN (non-numeric or <= 0) — refused, never +# silently defaulted +# exit 127 jq not found +# +# metrics (one row per session = one .jsonl source file): +# calls count of JSONL lines carrying a `.message.usage` object. A +# line that fails to parse as JSON, or parses but has no +# `.message.usage`, is skipped — not an error. +# out sum of output_tokens across those lines +# hit% sum(cache_read_input_tokens) / +# sum(input_tokens + cache_read_input_tokens + +# cache_creation_input_tokens) * 100, one decimal; +# "0.0" when the denominator is 0 +# final_ctx input+cache_read+cache_creation of the LAST usage line +# peak_ctx max input+cache_read+cache_creation across usage lines +# TOTAL row: calls/out are sums across sessions; hit% is recomputed from +# the summed token counts (not averaged); final_ctx/peak_ctx are the MAX +# across sessions (not summed). +# +# env: +# LO_CTX_WARN peak-context warning threshold (default 300000). After the +# table, every session whose peak_ctx >= this threshold gets +# one `warn: ...` line on stderr. Empty/unset = default; a +# non-numeric or <= 0 value is refused at exit 4 rather than +# silently falling back to the default. +set -eu + +JQ=$(command -v jq) || { echo "token-report: jq not found" >&2; exit 127; } + +usage() { echo "usage: token-report.sh [--cwd ] ..." >&2; } + +# ---- --cwd: map an absolute project path to its transcript directory ------ +cwd_map="" +if [ "${1:-}" = "--cwd" ]; then + if [ $# -lt 2 ]; then usage; exit 2; fi + proj="$2"; shift 2 + escaped=$(printf '%s' "$proj" | tr '/.' '-') + cwd_map="$HOME/.claude/projects/$escaped" + if [ ! -d "$cwd_map" ]; then + echo "token-report: --cwd mapped directory not found: '$cwd_map' (from '$proj')" >&2 + exit 2 + fi +fi + +if [ $# -lt 1 ]; then usage; exit 2; fi + +# ---- LO_CTX_WARN: default 300000; refuse (exit 4) rather than silently +# default on a non-numeric or non-positive value ---------------------------- +warn_threshold="${LO_CTX_WARN:-}" +if [ -z "$warn_threshold" ]; then + warn_threshold=300000 +else + case "$warn_threshold" in + *[!0-9]*) + echo "token-report: invalid LO_CTX_WARN '$warn_threshold' (must be a positive integer)" >&2 + exit 4 ;; + esac + if [ "$warn_threshold" -le 0 ]; then + echo "token-report: invalid LO_CTX_WARN '$warn_threshold' (must be > 0)" >&2 + exit 4 + fi +fi + +# ---- resolve each positional arg to one or more .jsonl source files ------- +# $1 = raw arg -> stdout: one resolved path per line. Exits 2 if it does not +# exist. A directory yields its immediate *.jsonl files (sorted, non-recursive). +resolve_source() { + arg="$1" + case "$arg" in + /*) path="$arg" ;; + *) if [ -n "$cwd_map" ]; then path="$cwd_map/$arg"; else path="$arg"; fi ;; + esac + if [ ! -e "$path" ]; then + echo "token-report: source not found: '$path'" >&2 + exit 2 + fi + if [ -d "$path" ]; then + find "$path" -maxdepth 1 -type f -name '*.jsonl' | sort + else + printf '%s\n' "$path" + fi +} + +sources=$( + for arg in "$@"; do + resolve_source "$arg" + done +) + +# ---- per-session metrics ---------------------------------------------- +# $1 = file path -> stdout: one TSV line "calls out in cread ccreate final_ctx peak_ctx". +# jq confirms/parses each line and extracts the 4 usage fields (skipping a +# line that isn't valid JSON, or lacks .message.usage, rather than failing); +# awk does the integer aggregation (sums, running max, last-line capture). +session_metrics() { + "$JQ" -R -r ' + (fromjson? // empty) as $obj + | ($obj.message.usage? // empty) as $u + | [($u.input_tokens // 0), ($u.cache_read_input_tokens // 0), + ($u.cache_creation_input_tokens // 0), ($u.output_tokens // 0)] + | @tsv + ' "$1" | awk -F'\t' ' + { + in_s = $1 + 0; cr = $2 + 0; cc = $3 + 0; out = $4 + 0 + ctx = in_s + cr + cc + calls++; out_sum += out; in_sum += in_s; cr_sum += cr; cc_sum += cc + final_ctx = ctx + if (ctx > peak_ctx) peak_ctx = ctx + } + END { + printf "%d\t%d\t%d\t%d\t%d\t%d\t%d\n", calls + 0, out_sum + 0, in_sum + 0, cr_sum + 0, cc_sum + 0, final_ctx + 0, peak_ctx + 0 + } + ' +} + +# $1 = numerator $2 = denominator -> "N.N" (one decimal; "0.0" when denom is 0) +pct1() { + "$JQ" -nr --argjson n "$1" --argjson d "$2" ' + if $d == 0 then "0.0" + else + (($n * 1000 / $d) | round) as $tenths + | ($tenths / 10 | floor) as $whole + | ($tenths - ($whole * 10)) as $frac + | "\($whole).\($frac)" + end + ' +} + +printf 'SESSION\tCALLS\tOUT\tHIT%%\tFINAL_CTX\tPEAK_CTX\n' + +calls_total=0; out_total=0; in_total=0; cr_total=0; cc_total=0 +final_max=0; peak_max=0 +warnings="" + +if [ -n "$sources" ]; then + while IFS= read -r file; do + [ -n "$file" ] || continue + name=$(basename "$file") + case "$name" in *.jsonl) name="${name%.jsonl}" ;; esac + + metrics=$(session_metrics "$file") + calls=$(printf '%s' "$metrics" | cut -f1) + out=$(printf '%s' "$metrics" | cut -f2) + in_s=$(printf '%s' "$metrics" | cut -f3) + cr=$(printf '%s' "$metrics" | cut -f4) + cc=$(printf '%s' "$metrics" | cut -f5) + final_ctx=$(printf '%s' "$metrics" | cut -f6) + peak_ctx=$(printf '%s' "$metrics" | cut -f7) + + hit=$(pct1 "$cr" "$((in_s + cr + cc))") + printf '%s\t%d\t%d\t%s\t%d\t%d\n' "$name" "$calls" "$out" "$hit" "$final_ctx" "$peak_ctx" + + calls_total=$((calls_total + calls)) + out_total=$((out_total + out)) + in_total=$((in_total + in_s)) + cr_total=$((cr_total + cr)) + cc_total=$((cc_total + cc)) + [ "$final_ctx" -gt "$final_max" ] && final_max=$final_ctx + [ "$peak_ctx" -gt "$peak_max" ] && peak_max=$peak_ctx + + if [ "$peak_ctx" -ge "$warn_threshold" ]; then + warnings="${warnings}warn: $name peak context $peak_ctx >= $warn_threshold — consider phase-boundary /compact or delegating large reads +" + fi + done <&2 +fi + +exit 0 diff --git a/tests/fixtures/token-report/edge/empty.jsonl b/tests/fixtures/token-report/edge/empty.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/token-report/edge/zero-usage.jsonl b/tests/fixtures/token-report/edge/zero-usage.jsonl new file mode 100644 index 0000000..6f0e5e0 --- /dev/null +++ b/tests/fixtures/token-report/edge/zero-usage.jsonl @@ -0,0 +1,2 @@ +{"type":"summary"} +{"message":{"role":"assistant","content":"hi"}} diff --git a/tests/fixtures/token-report/session-a.jsonl b/tests/fixtures/token-report/session-a.jsonl new file mode 100644 index 0000000..cb59a69 --- /dev/null +++ b/tests/fixtures/token-report/session-a.jsonl @@ -0,0 +1,5 @@ +{"message":{"usage":{"input_tokens":100,"cache_read_input_tokens":0,"cache_creation_input_tokens":900,"output_tokens":50}}} +{"message":{"usage":{"input_tokens":100,"cache_read_input_tokens":800,"cache_creation_input_tokens":100,"output_tokens":150}}} +{"type":"summary"} +not valid json at all +{"message":{"usage":{"input_tokens":200,"cache_read_input_tokens":1800,"cache_creation_input_tokens":0,"output_tokens":300}}} diff --git a/tests/fixtures/token-report/session-b.jsonl b/tests/fixtures/token-report/session-b.jsonl new file mode 100644 index 0000000..d48dda2 --- /dev/null +++ b/tests/fixtures/token-report/session-b.jsonl @@ -0,0 +1,2 @@ +{"message":{"usage":{"input_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":1000,"output_tokens":100}}} +{"message":{"usage":{"input_tokens":0,"cache_read_input_tokens":1000,"cache_creation_input_tokens":0,"output_tokens":100}}} diff --git a/tests/token-report.bats b/tests/token-report.bats new file mode 100644 index 0000000..83a3aa1 --- /dev/null +++ b/tests/token-report.bats @@ -0,0 +1,107 @@ +#!/usr/bin/env bats +# Tests for token-report.sh — per-session + total token usage report from +# Claude Code session transcript JSONL files (calls, output tokens, cache hit +# %, final/peak context), used at orchestration run teardown. + +setup() { + TR="${BATS_TEST_DIRNAME}/../skills/orchestrate/scripts/token-report.sh" + FIX="${BATS_TEST_DIRNAME}/fixtures/token-report" + TAB="$(printf '\t')" + # Isolate from ambient orchestration env (mirrors launch-session.bats). + unset LO_CTX_WARN +} + +# --- normal ------------------------------------------------------------ + +@test "prints 2 session rows + TOTAL with hand-computed sums" { + run sh "$TR" "$FIX" + [ "$status" -eq 0 ] + printf '%s\n' "$output" | grep -qE "^session-a${TAB}3${TAB}500${TAB}65\.0${TAB}2000${TAB}2000\$" + printf '%s\n' "$output" | grep -qE "^session-b${TAB}2${TAB}200${TAB}50\.0${TAB}1000${TAB}1000\$" + printf '%s\n' "$output" | grep -qE "^TOTAL${TAB}5${TAB}700${TAB}60\.0${TAB}2000${TAB}2000\$" +} + +@test "skips a non-usage line and an invalid-JSON line without crashing (session-a has 5 lines, 3 usage)" { + run sh "$TR" "$FIX/session-a.jsonl" + [ "$status" -eq 0 ] + printf '%s\n' "$output" | grep -qE "^session-a${TAB}3${TAB}500${TAB}65\.0${TAB}2000${TAB}2000\$" +} + +# --- error (exit 2) ----------------------------------------------------- + +@test "no args -> exit 2" { + run sh "$TR" + [ "$status" -eq 2 ] +} + +@test "nonexistent source path -> exit 2" { + run sh "$TR" "$FIX/does-not-exist.jsonl" + [ "$status" -eq 2 ] +} + +@test "--cwd given with no following value -> exit 2" { + run sh "$TR" --cwd + [ "$status" -eq 2 ] +} + +@test "--cwd whose mapped directory does not exist -> exit 2, stderr names the attempted path" { + fakehome="$BATS_TEST_TMPDIR/fakehome-missing" + run env HOME="$fakehome" sh "$TR" --cwd "/Users/nope/project" . + [ "$status" -eq 2 ] + printf '%s\n' "$output" | grep -qF -- "-Users-nope-project" +} + +# --- error (exit 4) ----------------------------------------------------- + +@test "LO_CTX_WARN=abc (non-numeric) -> exit 4" { + run env LO_CTX_WARN=abc sh "$TR" "$FIX" + [ "$status" -eq 4 ] +} + +@test "LO_CTX_WARN=0 -> exit 4 (boundary: not just non-numeric, <= 0 also refused)" { + run env LO_CTX_WARN=0 sh "$TR" "$FIX" + [ "$status" -eq 4 ] +} + +# --- boundary ------------------------------------------------------------ + +@test "jsonl with zero usage lines -> zero row, exit 0 (boundary)" { + run sh "$TR" "$FIX/edge/zero-usage.jsonl" + [ "$status" -eq 0 ] + printf '%s\n' "$output" | grep -qE "^zero-usage${TAB}0${TAB}0${TAB}0\.0${TAB}0${TAB}0\$" +} + +@test "empty file -> zero row, exit 0 (boundary)" { + run sh "$TR" "$FIX/edge/empty.jsonl" + [ "$status" -eq 0 ] + printf '%s\n' "$output" | grep -qE "^empty${TAB}0${TAB}0${TAB}0\.0${TAB}0${TAB}0\$" +} + +@test "peak-context warning fires on stderr when LO_CTX_WARN is below the fixture's peak" { + run env LO_CTX_WARN=1500 sh "$TR" "$FIX/session-a.jsonl" + [ "$status" -eq 0 ] + printf '%s\n' "$output" | grep -qF "warn: session-a peak context 2000 >= 1500" +} + +@test "no warning printed when LO_CTX_WARN is above the fixture's peak" { + run env LO_CTX_WARN=3000 sh "$TR" "$FIX/session-a.jsonl" + [ "$status" -eq 0 ] + ! printf '%s\n' "$output" | grep -q "warn:" +} + +@test "a relative source with no --cwd resolves against the caller's own cwd" { + cd "$FIX" + run sh "$TR" "session-a.jsonl" + [ "$status" -eq 0 ] + printf '%s\n' "$output" | grep -qE "^session-a${TAB}3${TAB}500${TAB}65\.0${TAB}2000${TAB}2000\$" +} + +@test "--cwd maps an absolute project path to ~/.claude/projects/ and resolves a relative source against it" { + fakehome="$BATS_TEST_TMPDIR/fakehome-ok" + projdir="$fakehome/.claude/projects/-Users-x-project" + mkdir -p "$projdir" + cp "$FIX/session-a.jsonl" "$projdir/lo-1.jsonl" + run env HOME="$fakehome" sh "$TR" --cwd "/Users/x/project" . + [ "$status" -eq 0 ] + printf '%s\n' "$output" | grep -qE "^lo-1${TAB}3${TAB}500${TAB}65\.0${TAB}2000${TAB}2000\$" +} From a7e5d36a3e36b41dfd61cba39097a133192c69fa Mon Sep 17 00:00:00 2001 From: Younggi Choi <74581798+choiyounggi@users.noreply.github.com> Date: Sat, 22 Aug 2026 00:45:03 +0900 Subject: [PATCH 3/3] feat(orchestrate): direct token-efficient behavior in SKILL.md and templates Wire t1's token-budget wiki page and t3's token-report.sh into the orchestrate skill's own instructions: a coordinator token-budget section plus a post-teardown token-report audit step in SKILL.md, a bounded-output and visual-delegation rule in session-prompt.md's subagent protocol, and a token-hygiene constraint line in brief.md. Each cites wiki/infrastructure/agent-orchestration/session-context-token-budget.md. Sharpens the byte-cksum guard in tests/send-prompt.bats that reddened from the session-prompt.md addition, and adds tests/orchestrate-token-budget.bats with dedicated doc-gates (positive assertion + negative control) for the SKILL.md and brief.md changes. --- skills/orchestrate/SKILL.md | 23 ++++++ skills/orchestrate/templates/brief.md | 3 +- .../orchestrate/templates/session-prompt.md | 8 ++ tests/orchestrate-token-budget.bats | 81 +++++++++++++++++++ tests/send-prompt.bats | 8 +- 5 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 tests/orchestrate-token-budget.bats diff --git a/skills/orchestrate/SKILL.md b/skills/orchestrate/SKILL.md index f156396..641f595 100644 --- a/skills/orchestrate/SKILL.md +++ b/skills/orchestrate/SKILL.md @@ -31,6 +31,20 @@ even if they can't re-read the config. A role is a tool injected into one step, never a loop: do not map a role to an implement/verify-loop tool or another orchestrator (that nests loops); there is no `implement` role. +## Coordinator token budget +Accumulated context, not turn count, is what drives a run's cost — every call +resends the full history at the cached rate, so cost per turn grows as the run +goes on. Three things the coordinator itself controls: (a) never `Read` a +screenshot or other image directly into this session — delegate the visual +check to a subagent that returns a text verdict instead; (b) bound every pane +capture and status read (`tail -N` on tmux panes, one-line `jq` filters on +status JSON) instead of pulling full output into context; (c) at Gate 1 and +Gate 2 — the two points where a human is already present — if the run has gone +long, tell the user this is a safe `/compact` point (the coordinator cannot +compact its own session). Basis: +`wiki/infrastructure/agent-orchestration/session-context-token-budget.md` +(directives 2–4). + ## Preflight Run `${CLAUDE_PLUGIN_ROOT}/hooks/preflight.sh` to resolve git/tmux/jq paths and surface any missing CLI. **git, tmux, and jq are all required** — if any is @@ -699,6 +713,15 @@ final step is the same: `LO_RUN_ID= scripts/safe-cleanup.sh teardown artifacts to `archive--/`. A run that ends without teardown is exactly the leak `list-orphans --stale` exists to find. +After teardown, audit the run's token efficiency: `sh {SKILL}/scripts/token-report.sh +--cwd .` for the coordinator's own session, plus one more `--cwd +` per run worktree (each worker's transcripts live under its own +cwd). Put the table and any `warn:` lines in the final run report — a +peak-context warning is the signal to split tasks or delegate more next run. +Exit 0 is the report, exit 2 a usage error, exit 4 a malformed `LO_CTX_WARN`. +Basis: `wiki/infrastructure/agent-orchestration/session-context-token-budget.md` +(directive 1). + ## Re-entry (resume) On re-invocation with no context, measure real state first: `git worktree list`, each `.orchestration/status/*.json` phase, and which `briefs/plans/reviews/` diff --git a/skills/orchestrate/templates/brief.md b/skills/orchestrate/templates/brief.md index e3c2be9..4521b45 100644 --- a/skills/orchestrate/templates/brief.md +++ b/skills/orchestrate/templates/brief.md @@ -46,7 +46,8 @@ specific tags below as authority. inherits them: knowledge=, tacit=, plan= --> {e.g. docs/specs to read, how to explore; DB read-only if any; resolved roles — knowledge/tacit/plan} - {local rules; surgical changes only on shared files} + {local rules; surgical changes only on shared files} + token hygiene: bound tool output (tail/head, ranged reads); delegate visual checks to a subagent — see wiki/infrastructure/agent-orchestration/session-context-token-budget.md diff --git a/skills/orchestrate/templates/session-prompt.md b/skills/orchestrate/templates/session-prompt.md index eb934d9..5244f43 100644 --- a/skills/orchestrate/templates/session-prompt.md +++ b/skills/orchestrate/templates/session-prompt.md @@ -211,3 +211,11 @@ and report exactly once: [3] Forbidden — do not call any agent by name other than `test-quality-auditor`. Others (e.g. a code-reviewer) may not exist in the user's environment and will fail silently. + +[4] Token hygiene — bound tool output before it reaches your context: pipe long + command output through `tail`/`head`, read big files by range + (offset/limit) instead of in full, and prefer a filename-only search mode + over one that returns full matched content. Delegate any screenshot or + visual verification to a subagent that returns a text verdict — an image + `Read` directly into this session is re-billed in full on every later turn. + See wiki/infrastructure/agent-orchestration/session-context-token-budget.md. diff --git a/tests/orchestrate-token-budget.bats b/tests/orchestrate-token-budget.bats new file mode 100644 index 0000000..e2e5250 --- /dev/null +++ b/tests/orchestrate-token-budget.bats @@ -0,0 +1,81 @@ +#!/usr/bin/env bats +# Doc gates for t2-skill-token-directives: SKILL.md and the brief template +# must direct token-efficient coordinator/worker behavior, each citing the +# governing wiki page +# (wiki/infrastructure/agent-orchestration/session-context-token-budget.md). +# session-prompt.md's item [4] is already pinned by the byte-level cksum +# guard in tests/send-prompt.bats, so it is not duplicated here. +# +# Each gate is paired with a negative control (checks-that-cannot-pass, +# wiki/testing/quality/checks-that-cannot-pass.md): a fixture with the +# asserted span stripped, shown to fail the same check. + +setup() { + REPO_ROOT="${BATS_TEST_DIRNAME}/.." + SKILL="${REPO_ROOT}/skills/orchestrate/SKILL.md" + BRIEF="${REPO_ROOT}/skills/orchestrate/templates/brief.md" +} + +# Collapses embedded newlines to a single space so a substring assertion +# survives prose hard-wrapped across physical lines (same technique as +# tests/orchestrate-teardown-contract.bats). +normalize_ws() { + printf '%s' "$1" | tr '\n' ' ' | tr -s ' ' +} + +flat_skill() { + normalize_ws "$(cat "$SKILL")" +} + +WIKI_SLUG='wiki/infrastructure/agent-orchestration/session-context-token-budget.md' + +# --- doc-gate 1: Coordinator token budget section (D1) --------------------- + +@test "doc-gate: Coordinator token budget section exists and cites the wiki page" { + flat="$(flat_skill)" + printf '%s' "$flat" | grep -qF "## Coordinator token budget" + printf '%s' "$flat" | grep -qF "$WIKI_SLUG" + printf '%s' "$flat" | grep -qF "delegate the visual check to a subagent" + printf '%s' "$flat" | grep -qF "/compact" +} + +@test "doc-gate can fail: a fixture without the Coordinator token budget section does not match" { + fixture="${BATS_TEST_TMPDIR}/no-token-budget.md" + grep -v 'Coordinator token budget' "$SKILL" > "$fixture" + flat="$(normalize_ws "$(cat "$fixture")")" + count="$(printf '%s' "$flat" | grep -cF "## Coordinator token budget" || true)" + [ "$count" -eq 0 ] +} + +# --- doc-gate 2: teardown token audit step (D2) ----------------------------- + +@test "doc-gate: End-of-run contract runs token-report.sh after teardown, citing the wiki page" { + flat="$(flat_skill)" + printf '%s' "$flat" | grep -qF "token-report.sh" + printf '%s' "$flat" | grep -qF -- "--cwd ." + printf '%s' "$flat" | grep -qF -- "--cwd" + printf '%s' "$flat" | grep -qF "$WIKI_SLUG" + printf '%s' "$flat" | grep -qF "Exit 0 is the report, exit 2 a usage error, exit 4 a malformed" +} + +@test "doc-gate can fail: a fixture without the token-report teardown step does not match" { + fixture="${BATS_TEST_TMPDIR}/no-token-report.md" + grep -v 'token-report.sh' "$SKILL" > "$fixture" + flat="$(normalize_ws "$(cat "$fixture")")" + count="$(printf '%s' "$flat" | grep -cF "token-report.sh" || true)" + [ "$count" -eq 0 ] +} + +# --- doc-gate 3: brief.md token-hygiene constraint (D4) -------------------- + +@test "doc-gate: brief.md constraints carry the token-hygiene line citing the wiki page" { + grep -qF 'token hygiene' "$BRIEF" + grep -qF "$WIKI_SLUG" "$BRIEF" +} + +@test "doc-gate can fail: a fixture without the token-hygiene line does not match" { + fixture="${BATS_TEST_TMPDIR}/brief-no-hygiene.md" + grep -v 'token hygiene' "$BRIEF" > "$fixture" + count="$(grep -cF 'token hygiene' "$fixture" || true)" + [ "$count" -eq 0 ] +} diff --git a/tests/send-prompt.bats b/tests/send-prompt.bats index 8bf99ba..064f090 100644 --- a/tests/send-prompt.bats +++ b/tests/send-prompt.bats @@ -481,9 +481,13 @@ tpl_sections_single_line() { # protocol every worker depends on. cksum is POSIX, so this holds on both CI # runners. If this block is ever intentionally changed, update this number in # the SAME commit and say so in the PR. + # Bumped from 2594177116/1010 when rule [4] gained the token-hygiene item + # (t2-skill-token-directives): bound tool output + delegate visual checks to + # a subagent, citing wiki/infrastructure/agent-orchestration/session-context- + # token-budget.md. run sh -c "sed -n '/^## Subagent usage protocol/,\$p' '$TPL' | cksum" [ "$status" -eq 0 ] - [ "$output" = "2594177116 1010" ] + [ "$output" = "4167093106 1552" ] } @test "template: negative control — the byte-identity guard fails on a reworded block" { @@ -492,7 +496,7 @@ tpl_sections_single_line() { sed 's/you MUST call the/you MAY call the/' "$TPL" > "$reworded" ! cmp -s "$TPL" "$reworded" # the edit really changed something run sh -c "sed -n '/^## Subagent usage protocol/,\$p' '$reworded' | cksum" - [ "$output" != "2594177116 1010" ] + [ "$output" != "4167093106 1552" ] } @test "template: the Orca prompt set is byte-identical" {