diff --git a/CHANGELOG.md b/CHANGELOG.md index ee3350c..4299f7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,11 @@ method, the rubric and the tables. It started from five misfires reported from o ### Fixed +- **dojo-tally 0.1.2** — a workflow agent that answers through StructuredOutput counts as returned. Its last turn + has no text, so the tally called every such agent "failed — it returned an empty answer": on 2026-10-27 a run of + 16 healthy scout, verify and synthesize agents read "returned 2 · failed 16". The arguments of the agent's last + accepted StructuredOutput call are now its answer, judged like text and handed to `/tally refute`'s judge; the + call itself no longer counts as a read, so a clean verdict handed back after reading nothing is still `silent`. - **wisdom-garden 1.2.1** — `/garden-score` reads the memory store from `/memory` when that folder holds `MEMORY.md` (the root otherwise, so older layouts keep working); `compressions/` and git still come from the root. Before this, a repo with its store in `memory/` scored 0 active seeds and 0 stale reads. diff --git a/plugins/dojo-tally/.claude-plugin/plugin.json b/plugins/dojo-tally/.claude-plugin/plugin.json index 5e142ff..387989e 100644 --- a/plugins/dojo-tally/.claude-plugin/plugin.json +++ b/plugins/dojo-tally/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "dojo-tally", - "version": "0.1.1", + "version": "0.1.2", "description": "Count returns, not dispatches: when a fan-out settles, the model hears what came back (returned, failed, silent, unchecked), and /tally refute sends a report to a judge.", "author": { "name": "Dojo Genesis", diff --git a/plugins/dojo-tally/README.md b/plugins/dojo-tally/README.md index c894752..ab409a9 100644 --- a/plugins/dojo-tally/README.md +++ b/plugins/dojo-tally/README.md @@ -39,6 +39,10 @@ What it does: - **Done means done.** An agent that ends a turn but is still held by the engine (parked on background work of its own) is waiting, not returned. Nothing about it is reported until the engine says it is done, and then its last answer is the one judged. +- **Structured answers.** An agent run with an output schema (a workflow's `agent()` with `schema`) answers + by calling StructuredOutput, so its last turn has no text. The arguments of its last accepted + StructuredOutput call are its answer: judged like text, and what `/tally refute` hands the judge. A + rejected try is not an answer, and the call itself is not counted as looking at anything. - **Classification**, first match wins: - `failed`: an error, a refusal, an interrupt, an empty answer, or an agent the engine stopped; - `silent`: a clean verdict ("no defects found", "looks good") from an agent that looked at nothing; @@ -79,6 +83,8 @@ What it does: - Only the last five waves are kept; `/tally refute` names agents by their number in the session. - The bare `/refute` is only a shortcut: when another plugin holds the name (dojo-protocol's `refute` skill does, so the whole suite installed means no `/refute` from this plugin), Claude Code keeps it there. `/tally refute ` always works. - Teammates (agents that idle between messages) are not tallied. +- A structured answer is kept in memory between the StructuredOutput call and the end of that agent's turn, + so a hot reload in that window loses it and the agent is counted as an empty answer. - A settle note arrives a few seconds after the last agent ends: the mod first asks the engine whether that agent is done. - A `/tally refute` judge runs on the model you pick and costs what that model costs. diff --git a/plugins/dojo-tally/hooks/register.ts b/plugins/dojo-tally/hooks/register.ts index cacb535..d0b3686 100644 --- a/plugins/dojo-tally/hooks/register.ts +++ b/plugins/dojo-tally/hooks/register.ts @@ -12,9 +12,15 @@ import type { TallyAgent, TallyJudge, TallyOutcome, TallyState } from '../types' // What counts as the agent having looked at something (the `reads` count, which the `silent` test needs): any tool call // that is not a write, so Read/Grep/Glob/Bash/Web*/LSP and also every mcp__ tool, Skill and Agent. Bookkeeping that -// looks at nothing (a todo list, loading a tool's schema) does not count. +// looks at nothing (a todo list, loading a tool's schema, handing back a structured answer) does not count. const WRITE_TOOLS = new Set(['Edit', 'Write', 'NotebookEdit', 'MultiEdit']) -const BOOKKEEPING = new Set(['TodoWrite', 'ToolSearch', 'ExitPlanMode']) +const BOOKKEEPING = new Set(['TodoWrite', 'ToolSearch', 'ExitPlanMode', 'StructuredOutput']) + +// An agent run with an output schema (a workflow's agent() with `schema`) answers by calling StructuredOutput, and its +// run ends on that call's result: its last turn has no visible text, so turn.complete's `answer` is "". The call's +// arguments are the answer. Keys tool.call carries that are not the tool's own arguments: +const STRUCTURED = 'StructuredOutput' +const RESERVED_KEYS = new Set(['tool', 'tool_use_id', 'agentId', 'consent', 'requestMeta']) const KEEP_WAVES = 5 const ANSWER_CLIP = 8000 @@ -65,9 +71,10 @@ type SpawnInfo = { id: string; label: string; type: string; model: string; isWor // How an agent ended. `byReconcile` marks an end the mod inferred (no turn.complete was seen), which a real // turn.complete that turns up later may correct. type Ending = { verdict: Verdict; byReconcile: boolean } -// What the hooks share: the options, the turn.completes that came before their agent was registered, and a flag that -// keeps two reconcile passes from overlapping. One per load of the module. -type Ctx = { cfg: Config; early: Map; isReconciling: boolean } +// What the hooks share: the options, the turn.completes that came before their agent was registered, each agent's last +// accepted StructuredOutput (JSON, until its turn ends), and a flag that keeps two reconcile passes from overlapping. +// One per load of the module. +type Ctx = { cfg: Config; early: Map; structured: Map; isReconciling: boolean } // `$` may only be handed to a function declared at the top of this file, and `$.env.get` takes string literals. type EnvReader = { env: { get: (name: string) => Promise } } @@ -123,6 +130,20 @@ function labelOf(e: SpawnEvent): string { return 'agent' } +// The tool's own arguments, as JSON: what the agent handed back. +function payloadOf(e: Record): string { + return JSON.stringify(Object.fromEntries(Object.entries(e).filter(([k]) => !RESERVED_KEYS.has(k)))) +} + +// A turn with no text whose agent handed back a structured answer returned that answer. Text, when there is any, +// stays the answer; either way the kept payload is used up by the end. +function withStructured(ctx: Ctx, id: string, d: Done): Done { + const payload = ctx.structured.get(id) + if (payload === undefined) return d + ctx.structured.delete(id) + return d.answer.trim() === '' ? { ...d, answer: payload } : d +} + function claimsCheck(text: string): boolean { for (const p of CHECK_CLAIM) { for (const m of text.matchAll(new RegExp(p.source, 'gi'))) { @@ -730,6 +751,7 @@ export const register: Register = (on, options) => { judgeModel: options.judge_model === 'sonnet' ? 'sonnet' : 'opus', }, early: new Map(), + structured: new Map(), isReconciling: false, } let tick: { cancel: () => void } | undefined @@ -765,13 +787,25 @@ export const register: Register = (on, options) => { return next(e) }) - // Counts after the call settles. A call that came back denied or errored was still attempted. + // Counts after the call settles. A call that came back denied or errored was still attempted. A StructuredOutput the + // engine accepted is kept as the agent's answer (a rejected try is not: the agent tries again or has none). on('tool.call', async ($, e, next) => { const r = await next(e) const id = e.agentId if (id !== undefined) { try { - if (!(await standDown($))) await countCall($, id, String(e.tool), r.deny !== undefined || r.isError === true) + if (await standDown($)) return r + const tool = String(e.tool) + const failed = r.deny !== undefined || r.isError === true + await countCall($, id, tool, failed) + if (tool === STRUCTURED && !failed) { + ctx.structured.delete(id) // re-set, so the oldest is the first out + ctx.structured.set(id, payloadOf(e as Record)) + if (ctx.structured.size > EARLY_CAP) { + const oldest = ctx.structured.keys().next().value + if (oldest !== undefined) ctx.structured.delete(oldest) + } + } } catch { // counting is best effort } @@ -801,7 +835,7 @@ export const register: Register = (on, options) => { const id = e.agentId if (id !== undefined) { try { - if (!(await standDown($))) await onDone($, ctx, id, e) + if (!(await standDown($))) await onDone($, ctx, id, withStructured(ctx, id, e)) } catch { // best effort } diff --git a/plugins/dojo-tally/tests/tally.test.ts b/plugins/dojo-tally/tests/tally.test.ts index 62c968a..b5ca117 100644 --- a/plugins/dojo-tally/tests/tally.test.ts +++ b/plugins/dojo-tally/tests/tally.test.ts @@ -148,6 +148,81 @@ test('a failed agent says why: error, refusal, empty answer', async ($, on) => { expect(row).toMatch('#3 three (general-purpose): failed — it returned an empty answer') }) +// ----------------------------------------------------------------------------------------------- structured answers + +// A workflow agent given an output schema answers through the StructuredOutput tool, and its run ends on that call's +// result: the turn's final visible text is "" and the report is the call's input. Hand-written from wf_73711b2d-734 +// (2026-10-27): each transcript ends tool_use StructuredOutput → tool_result "Structured output provided successfully" +// with no text after it, the journal holds a `result` line per agent, and the tally called all 16 "an empty answer". +const flow = (agentIndex: number) => ({ subagentType: 'workflow-subagent', workflow: { runId: 'wf_73711b2d-734', agentIndex } }) as any + +test('a workflow agent that answers through StructuredOutput has returned, and its payload is the report', async ($, on) => { + const seen = spawner(on) + mock.env(on, {}) + tick(on, { now: 1000 }) + const sess = mock.session(on) + standIn(on) + const ui = watchUi(on) + await start($) + await spawn($, 'scout:livekit', flow(1)) + await spawn($, 'verify:livekit', flow(2)) + await spawn($, 'synthesize', flow(3)) + await spawn($, 'builder') // a plain Agent-tool builder: it ends with text + await call($, 'a1', 'WebFetch', { url: 'https://api.github.com/repos/livekit/livekit' }) + await bash($, 'a1', 'gh api repos/livekit/livekit --jq .license.spdx_id') + await call($, 'a1', 'Write', { file_path: '/work/docs/research/L7-livekit.md', content: '# LiveKit' }) + await call($, 'a1', 'StructuredOutput', { lane: 'livekit', report: '/work/docs/research/L7-livekit.md', claims: ['LiveKit server is Apache-2.0'] }) + await read($, 'a2') + await bash($, 'a2', 'curl -s https://api.github.com/repos/livekit/livekit') + // a first try the schema rejected (the stand-in errors on command 'fail'), then the one that took + await call($, 'a2', 'StructuredOutput', { command: 'fail', verdicts: 'not an array' }) + await call($, 'a2', 'StructuredOutput', { verdicts: [{ claim: '1. LiveKit server is Apache-2.0', verdict: 'confirmed', source_url: 'https://api.github.com/repos/livekit/livekit' }] }) + await read($, 'a3') + await call($, 'a3', 'StructuredOutput', { summary: 'Eight lanes in; three claims not confirmed.', not_confirmed: 3 }) + await read($, 'a4') + for (const id of ['a1', 'a2', 'a3']) await complete($, '', { agentId: id }) + await complete($, 'Built the scaffold: 4 files under /work/app.', { agentId: 'a4' }) + + expect(ui.toast[0]).toBe('wave settled — dispatched 4 · returned 4 · failed 0 · silent 0 · unchecked 0') + const row = texts(sess)[0] as string + expect(row).toMatch('[dojo-tally] Fan-out settled: dispatched 4 · returned 4 · failed 0') + expect(row).not.toMatch('empty answer') + // the payload is what a judge is handed, the one that took and not the rejected try + const spawns = seen.length + await run($, 'tally', 'refute 2') + expect(seen.length).toBe(spawns + 1) + const judged = seen[seen.length - 1]?.prompt ?? '' + expect(judged).toMatch('"verdict":"confirmed"') + expect(judged).not.toMatch('not an array') +}) + +test('StructuredOutput looks at nothing: a clean verdict handed back with no reads is silent; a rejected one alone is still empty', async ($, on) => { + mock.env(on, {}) + tick(on, { now: 1000 }) + const sess = mock.session(on) + standIn(on) + spawner(on) + const ui = watchUi(on) + await start($) + await spawn($, 'review:read', flow(1)) + await spawn($, 'review:idle', flow(2)) + await spawn($, 'review:rejected', flow(3)) + await read($, 'a1') + await call($, 'a1', 'StructuredOutput', { verdict: 'No defects found.' }) + await call($, 'a2', 'StructuredOutput', { verdict: 'No defects found.' }) + await read($, 'a3') + await call($, 'a3', 'StructuredOutput', { command: 'fail', verdict: 'No defects found.' }) + for (const id of ['a1', 'a2', 'a3']) await complete($, '', { agentId: id }) + + expect(ui.toast[0]).toBe('wave settled — dispatched 3 · returned 1 · failed 1 · silent 1 · unchecked 0') + const row = texts(sess)[0] as string + expect(row).toMatch('#2 review:idle (workflow-subagent): silent — clean verdict after reading nothing (rule 9)') + expect(row).toMatch('#3 review:rejected (workflow-subagent): failed — it returned an empty answer') + expect(row).not.toMatch('#1 review:read') + const out = (await run($, 'tally')).text as string + expect(flat(out)).toMatch('#2 review:idle workflow-subagent inherit silent 0 reads · 0 writes · 0 bash') +}) + test('"Tests pass." with no Bash is unchecked; with a Bash call it is returned', async ($, on) => { mock.env(on, {}) tick(on, { now: 1000 }) diff --git a/plugins/dojo-tally/tests/test_plugin_shape.py b/plugins/dojo-tally/tests/test_plugin_shape.py index f417c7a..ff14990 100644 --- a/plugins/dojo-tally/tests/test_plugin_shape.py +++ b/plugins/dojo-tally/tests/test_plugin_shape.py @@ -41,7 +41,7 @@ def test_description_is_the_promise_and_within_budget(self): def test_required_fields(self): m = self.manifest self.assertEqual(m["name"], NAME) - self.assertEqual(m["version"], "0.1.1") + self.assertEqual(m["version"], "0.1.2") self.assertEqual(m["author"], {"name": "Dojo Genesis", "email": "cruz@trespiesdesign.com"}) self.assertEqual(m["homepage"], "https://dojogenesis.com/suite#" + NAME) self.assertEqual(m["repository"], "https://github.com/DojoGenesis/plugins")