diff --git a/.gitignore b/.gitignore index b087cd3a..590b895f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ coverage/ .mcp.json /.benchmarks/ /.playwright-cli/ +# Scratch and experiment output for a session; never a home for a necessary file (AGENTS.md). +/.temp/ graph.html /output/ /design/ diff --git a/.pi/extensions/nmg/index.ts b/.pi/extensions/nmg/index.ts index c25a5fb1..cfc9e275 100644 --- a/.pi/extensions/nmg/index.ts +++ b/.pi/extensions/nmg/index.ts @@ -27,7 +27,6 @@ import { type DaemonConnection, } from "../../../src/cli/daemon-client.ts"; import { resolveNmgDataDir } from "../../../src/cli/data-path.ts"; -import { describeStart, shortRound, startRound, type OooRoundAction } from "./ooo-round.ts"; import { archiveOrStage, archiveNodeName, @@ -1053,51 +1052,6 @@ export default function nmgExtension(pi: ExtensionAPI): void { }, }); - // The restricted out-of-order round, registered by default like the other NMG tools: a - // consumer that has to be switched on is not one. The cost boundary is per call: `live` spends - // tokens, while the default replays recorded answers and spends nothing. - pi.registerTool({ - name: "ooo_round", - label: "Restricted OoO round", - description: - "Drive one restricted out-of-order round. " + - "action=submit starts the round detached from a spec JSON and returns immediately (a live " + - "round takes minutes and spends tokens); action=status reads the round's own run directory " + - "from another process; action=cancel records an operator decision that survives a restart. " + - "The coordinator, not this tool, decides acceptance.", - parameters: Type.Object({ - action: Type.Union([Type.Literal("submit"), Type.Literal("status"), Type.Literal("cancel")]), - specPath: Type.Optional( - Type.String({ description: "submit only: path to the round spec JSON" }), - ), - runDir: Type.Optional( - Type.String({ description: "the round's own directory; required for every action" }), - ), - reason: Type.Optional(Type.String({ description: "cancel only: why the round was stopped" })), - live: Type.Optional( - Type.Boolean({ - description: - "submit only: use real model calls (costs tokens); absent means recorded answers", - }), - ), - }), - async execute(_toolCallId, params, _signal, _onUpdate, _ctx) { - const request = { - action: params.action as OooRoundAction, - specPath: params.specPath, - runDir: params.runDir, - reason: params.reason, - live: params.live, - }; - if (request.action === "submit") { - const started = startRound(projectDirectory(), request); - return toolResult(started, describeStart(request, started)); - } - const reply = await shortRound(projectDirectory(), request); - return toolResult(reply, `round ${reply.action}: exit ${reply.code}\n${reply.output}`); - }, - }); - pi.registerTool({ name: "nmg_remember", label: "Remember with NMG", diff --git a/.pi/extensions/nmg/ooo-execution.ts b/.pi/extensions/nmg/ooo-execution.ts index 11a720ad..56d166c9 100644 --- a/.pi/extensions/nmg/ooo-execution.ts +++ b/.pi/extensions/nmg/ooo-execution.ts @@ -117,7 +117,12 @@ function conclusionEnvelope( export async function executePiSnapshot(work: SnapshotInput, provider: string, modelId: string) { const prompt = snapshotPrompt(work); const snapshot = JSON.stringify({ input: work.input, dependencies: work.dependencies }); - return executePiInput(prompt, snapshot, provider, modelId, 8_000, SNAPSHOT_LIMITS); + return executePiInputWith( + { prompt, snapshot, maxArtifact: 8_000, limits: SNAPSHOT_LIMITS }, + provider, + modelId, + false, + ); } /** What the current task may push back on. A worker may report that a declared @@ -144,12 +149,13 @@ export interface PatchExecOptions { export const ARTIFACT_TOOL = "submit_artifact"; /** Produces an untrusted proposal, never applies files or marks a task accepted. */ -export async function executePiPatch( +/** The single-unit input `executePiPatch` runs, exposed so a chain can drive the same work through one + * session: the prompt, the snapshot and the bounds are built here once, and both paths read them from + * here rather than each describing the task again. */ +export function patchSessionInput( frozen: FrozenPatchWork, - provider: string, - modelId: string, options: PatchExecOptions = {}, -) { +): SessionRunInput { const { check, pushback } = options; const note = check ? `\nYou may call ${check.label} with your proposed files to run the round's fixed check before answering; at most ${check.maxRuns} calls are allowed. It runs only that check and never writes to the repository.` @@ -158,17 +164,25 @@ export async function executePiPatch( ? `\nIf what you received cannot satisfy one of these declared requirements, call report_dependency_failure with the exact task and requirement instead of finishing the work: ` + JSON.stringify(pushback.requirements) : ""; - return executePiInput( - patchPrompt(frozen, ARTIFACT_TOOL) + note + pushbackNote, - snapshotText(frozen), - provider, - modelId, - frozen.work.budget.output, - frozen.work.limits, - check, + return { + prompt: patchPrompt(frozen, ARTIFACT_TOOL) + note + pushbackNote, + snapshot: snapshotText(frozen), + maxArtifact: frozen.work.budget.output, + limits: frozen.work.limits, + ...(check !== undefined ? { check } : {}), frozen, - pushback, - ); + ...(pushback !== undefined ? { pushback } : {}), + }; +} + +export async function executePiPatch( + frozen: FrozenPatchWork, + provider: string, + modelId: string, + options: PatchExecOptions = {}, +) { + const input = patchSessionInput(frozen, options); + return executePiInputWith(input, provider, modelId, true); } export function piCompletionAllowed( @@ -204,12 +218,38 @@ export interface PiRun { * look identical in the token total, and the cost question cannot be answered. */ cacheRead: number; cacheWrite: number; + /** The session's cumulative totals. In a chain `tokens`/`cacheRead`/`cacheWrite` are this unit's + * own spend and these are the session's, which is what fusion's delta claim is read from; for a + * single-unit runner the two are equal. */ + sessionTokens?: number; + sessionCacheRead?: number; + sessionCacheWrite?: number; +} + +/** One unit's mutable state, held by the tool set. The tools read this object at call time rather + * than closing over its values, which is what lets a fused chain keep one session and one tool surface + * while each unit gets its own snapshot, check, budget and counters. The single-unit path builds one + * box and never re-points it, so both paths are the same code. */ +export interface UnitState { + snapshot: string; + limits: PatchLimits; + maxArtifact: number; + frozen?: FrozenPatchWork; + check?: CheckTool; + pushback?: PushbackSpec; + reads: { value: number }; + runs: { value: number }; + turns: number; + artifact: string | null; + report: PushbackReport | null; + /** Ends the current unit's attempt; re-pointed per unit by a chain. */ + abort: () => void; } /** Tool surface for one patch attempt. Each tool is bounded, parameter-free where it * must be, and reads only host-owned state: the worker cannot choose a command, a path * outside the frozen editable list, or a requirement that was not declared to it. */ -function readSnapshotTool(snapshot: string, limits: PatchLimits, reads: { value: number }) { +function readSnapshotTool(box: UnitState) { return defineTool({ name: "read_snapshot", label: "Read frozen task snapshot", @@ -217,17 +257,13 @@ function readSnapshotTool(snapshot: string, limits: PatchLimits, reads: { value: "Read this task's immutable input and accepted dependency values only (bounded at admission). No paths or commands are accepted.", parameters: Type.Object({}, { additionalProperties: false }), execute: async () => { - if (++reads.value > limits.reads) throw new Error("snapshot read budget exceeded"); - return { content: [{ type: "text" as const, text: snapshot }], details: {} }; + if (++box.reads.value > box.limits.reads) throw new Error("snapshot read budget exceeded"); + return { content: [{ type: "text" as const, text: box.snapshot }], details: {} }; }, }); } -function runCheckTool( - frozen: FrozenPatchWork | undefined, - check: CheckTool, - runs: { value: number }, -) { +function runCheckTool(box: UnitState) { return defineTool({ name: "run_check", label: "Run the round's fixed check", @@ -242,7 +278,16 @@ function runCheckTool( { additionalProperties: false }, ), execute: async (_id, args) => { - if (++runs.value > check.maxRuns) throw new Error("check budget exceeded"); + const { check, frozen } = box; + // A chain's surface is fixed at session creation, so a unit without a check gets this tool and + // is told so, instead of the surface being rebuilt (which a session does not allow). + if (!check) + return { + content: [{ type: "text" as const, text: "this unit has no check; answer without one" }], + details: {}, + isError: true, + }; + if (++box.runs.value > check.maxRuns) throw new Error("check budget exceeded"); try { if (!frozen) throw new Error("no frozen envelope for validation"); checkToolCandidate(frozen, args.files); @@ -274,10 +319,7 @@ function runCheckTool( }); } -function reportPushbackTool( - pushback: PushbackSpec, - state: { report: PushbackReport | null; abort: () => void }, -) { +function reportPushbackTool(box: UnitState) { return defineTool({ name: "report_dependency_failure", label: "Report that a dependency cannot satisfy a requirement", @@ -288,9 +330,11 @@ function reportPushbackTool( { additionalProperties: false }, ), execute: async (_id, args) => { - const declared = pushback.requirements; + const { pushback, report } = box; if ( - !declared.some( + !report || + pushback === undefined || + !pushback.requirements.some( (item) => item.task === args.dependency && item.requirement === args.requirement, ) ) @@ -298,7 +342,10 @@ function reportPushbackTool( content: [ { type: "text" as const, - text: `not a declared requirement. Declared: ${JSON.stringify(declared)}`, + text: + box.pushback === undefined + ? "this unit declared no dependency requirements" + : `not a declared requirement. Declared: ${JSON.stringify(box.pushback.requirements)}`, }, ], details: {}, @@ -306,12 +353,12 @@ function reportPushbackTool( }; // The claim is not completed by this report: the host decides whether to reopen // the dependency, and the attempt ends here without an artifact. - state.report = { + box.report = { dependency: args.dependency, requirement: args.requirement, evidence: String(args.evidence).slice(0, 2_000), }; - state.abort(); + box.abort(); return { content: [ { type: "text" as const, text: "recorded; this attempt ends and the host decides" }, @@ -466,10 +513,7 @@ function turnError(event: { /** Tool the artifact is delivered through. Structural prevention of prose: the schema is * the contract and the parameters are validated by this module, so a text answer can * never be mistaken for a submission. */ -function artifactTool( - frozen: FrozenPatchWork, - state: { artifact: string | null; abort: () => void }, -) { +function artifactTool(box: UnitState, looseConclusion = false) { return defineTool({ name: ARTIFACT_TOOL, label: "Submit the artifact", @@ -485,7 +529,11 @@ function artifactTool( // cannot offer a kind this task would be rejected for, and an invented value // cannot be sampled at all. conclusion: Type.Optional( - Type.Union(frozen.work.admittedConclusions.map((kind) => Type.Literal(kind))), + // A chain registers its surface once, so a unit's own literal union cannot be sampled there; + // `artifactEnvelope` still refuses an invented kind, and the host still validates the result. + looseConclusion + ? Type.String() + : Type.Union(box.frozen!.work.admittedConclusions.map((kind) => Type.Literal(kind))), ), summary: Type.Optional(Type.String()), evidence: Type.Optional(Type.String()), @@ -497,6 +545,13 @@ function artifactTool( ), constrainedSampling: { type: "json_schema", strict: "prefer" }, execute: async (_id, args) => { + const frozen = box.frozen; + if (!frozen) + return { + content: [{ type: "text" as const, text: "this unit has no frozen envelope" }], + details: {}, + isError: true, + }; const built = artifactEnvelope(frozen, args as ArtifactParams); if (!built.ok) return { @@ -511,74 +566,122 @@ function artifactTool( }; // A recorded artifact ends the attempt: further text would only spend tokens and // could contradict the submission, which the host never reads as an answer. - state.artifact = built.json; - state.abort(); + box.artifact = built.json; + box.abort(); return { content: [{ type: "text" as const, text: "artifact recorded" }], details: {} }; }, }); } -/** The optional tools for one attempt, present only when the host enabled them. */ +/** The optional tools for one attempt. The surface is explicit rather than derived from the box, + * because a chain registers its whole surface once, when its box is still empty. */ function optionalTools( - frozen: FrozenPatchWork | undefined, - check: CheckTool | undefined, - runs: { value: number }, - pushback: PushbackSpec | undefined, - state: { report: PushbackReport | null; abort: () => void }, - artifact: { artifact: string | null; abort: () => void } | undefined, + box: UnitState, + surface: { check: boolean; pushback: boolean; artifact: boolean; looseConclusion?: boolean }, ) { return { - runCheck: check ? runCheckTool(frozen, check, runs) : undefined, - reportPushback: - pushback && pushback.requirements.length ? reportPushbackTool(pushback, state) : undefined, - submitArtifact: frozen && artifact ? artifactTool(frozen, artifact) : undefined, + runCheck: surface.check ? runCheckTool(box) : undefined, + reportPushback: surface.pushback ? reportPushbackTool(box) : undefined, + submitArtifact: surface.artifact + ? artifactTool(box, surface.looseConclusion === true) + : undefined, }; } -async function executePiInput( - prompt: string, - snapshot: string, - provider: string, - modelId: string, - maxArtifact: number, - limits: PatchLimits, - check?: CheckTool, - frozen?: FrozenPatchWork, - pushback?: PushbackSpec, -): Promise { - const runtime = await ModelRuntime.create({ signal: AbortSignal.timeout(limits.timeoutMs) }); +/** One unit's input into a session: everything its tools and its completion contract read. */ +export interface SessionRunInput { + prompt: string; + snapshot: string; + maxArtifact: number; + limits: PatchLimits; + check?: CheckTool; + frozen?: FrozenPatchWork; + pushback?: PushbackSpec; +} + +/** A session that can run more than one unit: the mechanism fusion's policy half needs. + * + * `PiRun.tokens` is the **unit's own** spend (the session's total minus what it was when the unit + * started) and `sessionTokens` the session's cumulative total, because fusion's claim is about the + * delta: a later unit in a warm context should spend less than a fresh session on the same work. A + * single-unit runner reports the same numbers both ways, so nothing that reads `tokens` changes. */ +export interface PiSessionRunner { + sessionId: string; + runUnit(input: SessionRunInput): Promise; + dispose(): void; +} + +/** One session, one tool surface, many units. + * + * A chain passes `chain: true`, which registers the union of what its units may need - a unit without + * a check then gets a `run_check` that refuses by name, because a session does not allow rebuilding the + * surface - and loosens the artifact schema's conclusion kind to a string, because a per-unit literal + * union cannot be sampled once the surface exists (`artifactEnvelope` still refuses an invented kind, + * and the host still validates the result). Without `chain` the surface is exactly the first unit's, + * which is what every single-attempt caller has today. */ +export async function createPiSessionRunner(options: { + provider: string; + modelId: string; + patchMode: boolean; + first: SessionRunInput; + chain?: boolean; +}): Promise { + const { provider, modelId } = options; + const chain = options.chain === true; + const surface = chain + ? { check: true, pushback: true, artifact: true, looseConclusion: true } + : { + check: options.first.check !== undefined, + pushback: (options.first.pushback?.requirements.length ?? 0) > 0, + artifact: options.first.frozen !== undefined, + }; + const controller = new AbortController(); + const runtime = await ModelRuntime.create({ signal: controller.signal }); const model = runtime.getModel(provider, modelId); if (!model) throw new Error(`Pi model unavailable: ${provider}/${modelId}`); - const resources = resourceLoader(frozen !== undefined); - const reads = { value: 0 }; - const runs = { value: 0 }; - const pushbackState: { report: PushbackReport | null; abort: () => void } = { + const resources = resourceLoader(options.patchMode); + const box: UnitState = { + snapshot: options.first.snapshot, + limits: options.first.limits, + maxArtifact: options.first.maxArtifact, + reads: { value: 0 }, + runs: { value: 0 }, + turns: 0, + artifact: null, report: null, abort: () => {}, }; - const artifactState: { artifact: string | null; abort: () => void } = { - artifact: null, - abort: () => {}, + /** Re-points the box at the next unit. The tools hold this object, so nothing is rebuilt. */ + const point = (input: SessionRunInput): void => { + box.snapshot = input.snapshot; + box.limits = input.limits; + box.maxArtifact = input.maxArtifact; + box.reads = { value: 0 }; + box.runs = { value: 0 }; + box.turns = 0; + box.artifact = null; + box.report = null; + delete box.frozen; + delete box.check; + delete box.pushback; + if (input.frozen !== undefined) box.frozen = input.frozen; + if (input.check !== undefined) box.check = input.check; + if (input.pushback !== undefined) box.pushback = input.pushback; }; - const readSnapshot = readSnapshotTool(snapshot, limits, reads); - const { runCheck, reportPushback, submitArtifact } = optionalTools( - frozen, - check, - runs, - pushback, - pushbackState, - artifactState, + point(options.first); + const readSnapshot = readSnapshotTool(box); + const { runCheck, reportPushback, submitArtifact } = optionalTools(box, surface); + const expectedTools = toolNames( + runCheck !== undefined, + reportPushback !== undefined, + submitArtifact !== undefined, ); const { session } = await createAgentSession({ model, modelRuntime: runtime, thinkingLevel: "off", resourceLoader: resources, - tools: toolNames( - check !== undefined, - reportPushback !== undefined, - submitArtifact !== undefined, - ), + tools: expectedTools, customTools: [readSnapshot, runCheck, reportPushback, submitArtifact].filter( (tool): tool is NonNullable => tool !== undefined, ), @@ -588,68 +691,107 @@ async function executePiInput( retry: { enabled: false }, }), }); - pushbackState.abort = () => void session.abort(); - artifactState.abort = () => void session.abort(); - const finish = (artifact: string, turnsUsed: number, pushback?: PushbackReport): PiRun => ({ - artifact, - ...(pushback ? { pushback } : {}), - sessionId: session.sessionId, - provider, - model: modelId, - reads: reads.value, - turns: turnsUsed, - checks: runs.value, + box.abort = () => void session.abort(); + const totals = () => ({ tokens: totalTokens(session.messages), ...cacheTotals(session.messages), }); - - let turns = 0; - let timedOut = false; + const names = expectedTools.join(","); const unsubscribe = session.subscribe((event) => { - if (event.type === "turn_start" && ++turns > limits.turns) void session.abort(); + if (event.type === "turn_start" && ++box.turns > box.limits.turns) void session.abort(); const error = turnError(event as Parameters[0]); if (error) process.stderr.write(error); }); - const timeout = setTimeout(() => { - timedOut = true; - void session.abort(); - }, limits.timeoutMs); - try { - const expectedTools = toolNames( - check !== undefined, - reportPushback !== undefined, - submitArtifact !== undefined, - ).join(","); - if (session.getActiveToolNames().join(",") !== expectedTools) - throw new Error("unexpected Pi tool surface"); - await session.prompt(prompt, { expandPromptTemplates: false }); - // A tool-recorded artifact is the completion evidence: it was produced through our - // own handler, not claimed by the model. It also wins over any trailing text. - if (artifactState.artifact) return finish(artifactState.artifact, turns); - if (pushbackState.report) return finish("", turns, pushbackState.report); - const message = session.messages.findLast((item) => item.role === "assistant"); - const allowed = piCompletionAllowed(message?.stopReason, timedOut, turns, reads.value, limits); - const text = boundedArtifact(message, maxArtifact); - // Both channels of a patch attempt go through the same envelope: an answer that is not a - // valid artifact for this frozen work is a failed attempt with its reason, not a - // submission the host has to reject later for a defect the adapter could already name. - // Without frozen work (the snapshot task) the text *is* the artifact, as before. - const built = text - ? frozen - ? artifactFromText(frozen, text) - : ({ ok: true, json: text } as const) - : ({ ok: false, error: "no artifact" } as const); - if (!allowed || !built.ok) - throw new Error( - `Pi snapshot task did not finish within its bounded contract: ` + - `stopReason=${message?.stopReason}, turns=${turns}, reads=${reads.value}, ` + - `artifact=${built.ok ? "ok" : built.error}` + - (timedOut ? " (timed out)" : ""), + + const runUnit = async (input: SessionRunInput): Promise => { + point(input); + const before = totals(); + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + void session.abort(); + }, box.limits.timeoutMs); + const done = (artifact: string, pushbackFromTool?: PushbackReport): PiRun => { + const after = totals(); + return { + artifact, + ...(pushbackFromTool ? { pushback: pushbackFromTool } : {}), + sessionId: session.sessionId, + provider, + model: modelId, + reads: box.reads.value, + turns: box.turns, + checks: box.runs.value, + tokens: after.tokens - before.tokens, + cacheRead: after.cacheRead - before.cacheRead, + cacheWrite: after.cacheWrite - before.cacheWrite, + sessionTokens: after.tokens, + sessionCacheRead: after.cacheRead, + sessionCacheWrite: after.cacheWrite, + }; + }; + try { + if (session.getActiveToolNames().join(",") !== names) + throw new Error("unexpected Pi tool surface"); + await session.prompt(input.prompt, { expandPromptTemplates: false }); + // A tool-recorded artifact is the completion evidence: it was produced through our + // own handler, not claimed by the model. It also wins over any trailing text. + if (box.artifact) return done(box.artifact); + if (box.report) return done("", box.report); + const message = session.messages.findLast((item) => item.role === "assistant"); + const allowed = piCompletionAllowed( + message?.stopReason, + timedOut, + box.turns, + box.reads.value, + box.limits, ); - return finish(built.json, turns); + const text = boundedArtifact(message, box.maxArtifact); + // Both channels of a patch attempt go through the same envelope: an answer that is not a + // valid artifact for this frozen work is a failed attempt with its reason, not a + // submission the host has to reject later for a defect the adapter could already name. + // Without frozen work (the snapshot task) the text *is* the artifact, as before. + const frozen = box.frozen; + const built = text + ? frozen + ? artifactFromText(frozen, text) + : ({ ok: true, json: text } as const) + : ({ ok: false, error: "no artifact" } as const); + if (!allowed || !built.ok) + throw new Error( + `Pi snapshot task did not finish within its bounded contract: ` + + `stopReason=${message?.stopReason}, turns=${box.turns}, reads=${box.reads.value}, ` + + `artifact=${built.ok ? "ok" : built.error}` + + (timedOut ? " (timed out)" : ""), + ); + return done(built.json); + } finally { + clearTimeout(timeout); + } + }; + + return { + sessionId: session.sessionId, + runUnit, + dispose: () => { + unsubscribe(); + session.dispose(); + controller.abort(); + }, + }; +} + +/** One bounded Pi execution through the single-unit path, from an input a chain can also build. */ +async function executePiInputWith( + input: SessionRunInput, + provider: string, + modelId: string, + patchMode: boolean, +): Promise { + const runner = await createPiSessionRunner({ provider, modelId, patchMode, first: input }); + try { + return await runner.runUnit(input); } finally { - clearTimeout(timeout); - unsubscribe(); - session.dispose(); + runner.dispose(); } } diff --git a/.pi/extensions/nmg/ooo-round.ts b/.pi/extensions/nmg/ooo-round.ts deleted file mode 100644 index 148bbcc5..00000000 --- a/.pi/extensions/nmg/ooo-round.ts +++ /dev/null @@ -1,110 +0,0 @@ -// Pi adaptation of the restricted out-of-order round: a thin surface, so that a normal session can -// drive one round. It binds to the round entry point that already exists (and is reviewed) instead -// of re-implementing the coordinator here — the adapter stays thin, and the shared layer keeps -// owning selection, attempts, fencing and acceptance. -// -// Deliberate properties: -// - Registered by default, like the other NMG tools: a consumer that has to be switched on is not -// a consumer. The cost boundary is per call instead — `live` is what spends tokens, and it is -// absent unless asked for. -// - `submit` never blocks the session. A live round takes minutes, and the CLI's own `submit` -// waits for the terminal event, so this starts it detached with a log and the caller polls -// `status` — which is exactly the cross-process property the design asks for. -// - Missing inputs are refused by name rather than guessed. -import { execFile, spawn } from "node:child_process"; -import { mkdirSync, openSync } from "node:fs"; -import { join } from "node:path"; -import { promisify } from "node:util"; - -const run = promisify(execFile); - -export type OooRoundAction = "submit" | "status" | "cancel"; - -export interface OooRoundParams { - action: OooRoundAction; - specPath?: string; - runDir?: string; - reason?: string; - live?: boolean; -} - -const CLI = "evals/ooo-execution/round-cli.ts"; - -/** The exact arguments for one action, or a refusal that names what is missing. */ -export function roundArgv(params: OooRoundParams): string[] { - if (!params.runDir) - throw new Error("runDir is required: a round keeps its state in its own run directory"); - const base = ["--experimental-strip-types", CLI, params.action]; - if (params.action === "submit") { - if (!params.specPath) throw new Error("submit requires specPath (the round spec JSON file)"); - const argv = [...base, params.specPath, "--run-dir", params.runDir]; - if (params.live) argv.push("--live"); - return argv; - } - if (params.action === "cancel") { - if (!params.reason) - throw new Error( - "cancel requires reason: the decision is recorded in the round's store and survives a restart", - ); - return [...base, "--run-dir", params.runDir, "--reason", params.reason]; - } - return [...base, "--run-dir", params.runDir]; -} - -export interface StartedRound { - pid: number | undefined; - logPath: string; -} - -/** Starts the round detached, so the session stays responsive while it runs. */ -export function startRound(projectDir: string, params: OooRoundParams): StartedRound { - const argv = roundArgv(params); - const runDir = params.runDir!; - mkdirSync(runDir, { recursive: true }); - const logPath = join(runDir, "cli.log"); - const out = openSync(logPath, "a"); - const child = spawn(process.execPath, argv, { - cwd: projectDir, - detached: true, - stdio: ["ignore", out, out], - }); - child.unref(); - return { pid: child.pid, logPath }; -} - -export interface RoundReply { - action: OooRoundAction; - code: number; - output: string; -} - -/** Status and cancel are bounded: they read or write the run directory, they do not run a round. */ -export async function shortRound(projectDir: string, params: OooRoundParams): Promise { - const argv = roundArgv(params); - try { - const { stdout, stderr } = await run(process.execPath, argv, { - cwd: projectDir, - timeout: 60_000, - maxBuffer: 4 * 1024 * 1024, - }); - return { action: params.action, code: 0, output: `${stdout}${stderr}`.trim() }; - } catch (error) { - const failure = error as { code?: number; stdout?: string; stderr?: string; message: string }; - return { - action: params.action, - code: failure.code ?? 1, - output: `${failure.stdout ?? ""}${failure.stderr ?? ""}`.trim() || failure.message, - }; - } -} - -/** What a started round should be told: it is running elsewhere, and what to read afterwards. */ -export function describeStart(params: OooRoundParams, started: StartedRound): string { - return [ - `round ${params.action} started detached (pid ${started.pid ?? "unknown"})`, - `run directory: ${params.runDir}`, - `mode: ${params.live ? "live model calls (this spends tokens)" : "recorded answers (no model calls)"}`, - `log: ${started.logPath}`, - "the round owns its own state; poll it with action=status, stop it with action=cancel --reason", - ].join("\n"); -} diff --git a/agent-context.yaml b/agent-context.yaml index 03ba75b5..8e88b372 100644 --- a/agent-context.yaml +++ b/agent-context.yaml @@ -145,13 +145,14 @@ routes: owners: - docs/design/ci-cd-and-quality.md - docs/design/design.md + - skills/repo-development/references/builds.md tests: [tests/extensions/pi-dependency-boundary.test.ts] verify: blocking: [build, package:check] advisory: [] - id: repository-tooling - paths: [tools/**, scripts/**, AGENTS.md, agent-context.yaml, tsconfig.json] + paths: [tools/**, scripts/**, AGENTS.md, agent-context.yaml, tsconfig.json, .gitignore] owners: - docs/design/ci-cd-and-quality.md - docs/README.md @@ -159,13 +160,82 @@ routes: verify: blocking: [check, test:product, build, agent:context:check] advisory: [] + # `.gitignore` is a declaration, not code: the always-run shared checks can all pass or fail + # identically whether or not it changed, and the one check that reads it (the complexity + # gate's probe scratch must stay invisible to git) lives in this route's own tests. + sharedChecks: none - id: repository-control-plane paths: [src/rcp/**, tests/rcp/**, bin/nmg-rcp.mjs] owners: - docs/design/ci-cd-and-quality.md - docs/decisions/implemented/2026-08-29-repository-control-plane.md + - skills/repo-development/references/control-plane.md tests: [tests/rcp/**] verify: blocking: [check, test:product, build, package:check] advisory: [] + + # The integration layer is split by owner document, not by directory: the Agent Surface + # presents, and the OoO/task files orchestrate. Both list files because `matches()` in + # tools/repo-context.ts reads the first `*` as a directory prefix, so only `dir/**` and exact + # paths match - `src/integration/ooo-*.ts` would match nothing and stay silently unrouted. + - id: agent-surface + paths: + - src/integration/agent-surface.ts + - src/integration/chain-projection.ts + - src/integration/config.ts + - src/integration/controller-channel.ts + - src/integration/evidence.ts + - src/integration/lab-capabilities.ts + - src/integration/reasoning-workspaces.ts + - src/integration/search-projection.ts + - src/integration/search.ts + - src/integration/tool-contract.ts + owners: + - docs/design/design.md + tests: [tests/integration/agent-surface.test.ts, tests/integration/chain-projection.test.ts, tests/integration/config.test.ts, tests/integration/controller-channel.test.ts, tests/integration/evidence.test.ts, tests/integration/lab-capabilities.test.ts, tests/integration/tool-contract.test.ts] + verify: + blocking: [check, test:product, build] + advisory: [] + + - id: ooo-execution + paths: + - src/integration/ooo-board.ts + - src/integration/ooo-candidate.ts + - src/integration/ooo-execution.ts + - src/integration/ooo-fusion-plan.ts + - src/integration/ooo-mutation.ts + - src/integration/ooo-patch.ts + - src/integration/check-ticket.ts + - src/integration/check-runner.ts + - src/integration/task-advisers.ts + - src/integration/task-coordinator.ts + - src/integration/task-semantics-interleavings.ts + - src/integration/task-semantics-model.ts + - src/integration/task-semantics.ts + owners: + - docs/design/ooo-execution-bootstrap.md + - docs/design/task-unit-semantics.md + - docs/design/ooo-fusion-planning.md + tests: [tests/integration/ooo-*.test.ts, tests/integration/task-semantics*.test.ts] + verify: + blocking: [check, test:product, build] + advisory: [] + + # The third thing under this layer: retrieval-index text that an external LLM writes and the + # LLM-free store persists. It is neither the Agent Surface nor the execution orchestra, so it owns + # a route instead of borrowing either owner's documents. + - id: retrieval-enrichment + paths: + - src/integration/leaf-summarizer.ts + - src/integration/node-summarizer.ts + - src/integration/summary-drain.ts + - src/integration/openai-completion.ts + owners: + - docs/design/design.md + - docs/design/tiered-disclosure-design.md + tests: [tests/integration/leaf-summarizer.test.ts, tests/integration/summary-drain.test.ts, tests/core/store/node-summaries.test.ts] + verify: + blocking: [check, test:product, build] + advisory: [] diff --git a/docs/README.md b/docs/README.md index 3cfa0f31..e154f18e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -118,3 +118,10 @@ rules and must not invent additional policy. An error means the repository's documented public or normative interface is broken in a mechanically reproducible way. A warning is maintenance input for an Agent or reviewer and must not fail CI. + +The byte budget covers the always-read entry of a Skill, not every file under it. +When an entry approaches its ceiling, route its occasionally-needed sections into +`skills//references/` and name each trigger in the entry ([ +decision](decisions/implemented/2026-09-18-skill-grows-by-routing.md)), rather +than raising the ceiling or compressing a rule until it loses the facts it +depends on. diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 870744af..6e2e51e5 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -65,3 +65,7 @@ NMG 需要双语文档,但不要求机械地逐段一致。 | 翻译质量、设计正确性、实验结论、文风,以及某个失败类别是否已被提升为规则 | 所有文档 | 不自动判断 | 错误表示仓库公开或规范接口发生了可机械复现的破损;警告只作为 Agent 或 reviewer 的维护输入,不得导致 CI 失败。 + +字节预算锁的是 Skill 的常读入口,而不是 Skill 下的每个文件。入口接近上限时,把偶发才需要的节路由进 +`skills//references/`,并在入口里写明每节的触发条件([决策](decisions/implemented/2026-09-18-skill-grows-by-routing.md)), +而不是抬上限、也不是把一条规则压到丢掉它依赖的事实。 diff --git a/docs/decisions/proposed/2026-09-06-board-governance-addressing.md b/docs/decisions/implemented/2026-09-06-board-governance-addressing.md similarity index 95% rename from docs/decisions/proposed/2026-09-06-board-governance-addressing.md rename to docs/decisions/implemented/2026-09-06-board-governance-addressing.md index 8b4e2369..bac015a8 100644 --- a/docs/decisions/proposed/2026-09-06-board-governance-addressing.md +++ b/docs/decisions/implemented/2026-09-06-board-governance-addressing.md @@ -2,10 +2,13 @@ [中文](2026-09-06-board-governance-addressing.zh-CN.md) -**Status:** proposed +**Status:** implemented +**Approved:** explicit **Relates to:** [board-find-serial-a2a-compat-2026-08-13](../../design/board-find-serial-a2a-compat-2026-08-13.md), [agent-convergence-feedback-design](../../design/agent-convergence-feedback-design.md) +Implementation evidence: `readTaskBoardPreviews` (`src/cli/service.ts`) gives the compact read, and the addressing surface it asked for is live on every adapter - `discover`, `claim`/`release`/`resolve`, `acknowledge`, and `memory=` pointers - with `tests/cli/**` covering the CLI subcommands. + ## Problem The NMG task board is a stigmergic coordination medium: autonomous agents @@ -40,7 +43,7 @@ every agent sees one board). The board also feeds the global calibrator in the agent-convergence design, so its termination integrity is what turns `resolve` into a usable (de-biased) training signal. -## Proposal +## Decision Two axes. Borrow the *semantics* the incident validates; keep NMG's traits (memory pointers, A2A, single owner, waker, context frugality). Do **not** @@ -152,7 +155,9 @@ read is not an acceptance. per-entry scope/authenticity are exactly the properties that let an incident board become a blind spot. -## Acceptance criteria +## Consequences + +The criteria this record set are met; the evidence line at the top of this file names what implements them. Anything still owed stays written down in the design or pilot document this record links to. - `read` returns compact summaries by default; full text requires an explicit expansion, and a channel with many unchanged entries does not grow per-sync diff --git a/docs/decisions/proposed/2026-09-06-board-governance-addressing.zh-CN.md b/docs/decisions/implemented/2026-09-06-board-governance-addressing.zh-CN.md similarity index 94% rename from docs/decisions/proposed/2026-09-06-board-governance-addressing.zh-CN.md rename to docs/decisions/implemented/2026-09-06-board-governance-addressing.zh-CN.md index 18186b76..1d38ecc3 100644 --- a/docs/decisions/proposed/2026-09-06-board-governance-addressing.zh-CN.md +++ b/docs/decisions/implemented/2026-09-06-board-governance-addressing.zh-CN.md @@ -2,10 +2,13 @@ [English](2026-09-06-board-governance-addressing.md) -**Status:** proposed +**Status:** implemented +**Approved:** explicit **关联:** [board-find-serial-a2a-compat-2026-08-13](../../design/board-find-serial-a2a-compat-2026-08-13.md)、 [agent-convergence-feedback-design](../../design/agent-convergence-feedback-design.md) +实现证据:`readTaskBoardPreviews`(`src/cli/service.ts`)提供紧凑读取;它要求的寻址面已在各适配器上生效——`discover`、`claim`/`release`/`resolve`、`acknowledge` 与 `memory=` 指针——并由 `tests/cli/**` 覆盖 CLI 子命令。 + ## 问题 NMG 任务黑板是 stigmergy 协调介质:自主 agent 靠读写一批带类型、可归因的共享条目协调, @@ -27,7 +30,7 @@ ack、串行单 owner 晋升、唤醒器与 `memory=` 记忆指针。 所有 agent 见同一板)。黑板还喂给 agent-convergence 的全局校准器,故其终止完整性是把 `resolve` 变成可用(去偏)训练信号的关键。 -## 提案 +## 决策 两条轴。借事件所验证的*语义*,同时保住 NMG 特性(记忆指针、A2A、单 owner、唤醒器、上下文 节俭)。**不**照搬事件的自由共享存储模型。 @@ -96,7 +99,9 @@ ack、串行单 owner 晋升、唤醒器与 `memory=` 记忆指针。 - **自由共享板文件(HF 式)**:否决——无类型写入且无逐条作用域/真实性,恰是让事件板成为盲点的 属性。 -## 验收标准 +## 后果 + +本记录设定的验收标准均已满足;本文件顶部的证据行指出了实现它们的东西。仍欠的部分留在本记录链接的设计或试点文档里。 - `read` 默认返回紧凑摘要;全文需显式展开;含大量未变条目的频道不增单次同步上下文。 - 收件箱/待处理视图列出面向我、可行动且未处理的条目。 diff --git a/docs/decisions/proposed/2026-09-09-ooo-bootstrap.md b/docs/decisions/implemented/2026-09-09-ooo-bootstrap.md similarity index 82% rename from docs/decisions/proposed/2026-09-09-ooo-bootstrap.md rename to docs/decisions/implemented/2026-09-09-ooo-bootstrap.md index ddaaac2a..67fae1ba 100644 --- a/docs/decisions/proposed/2026-09-09-ooo-bootstrap.md +++ b/docs/decisions/implemented/2026-09-09-ooo-bootstrap.md @@ -2,7 +2,10 @@ [中文](2026-09-09-ooo-bootstrap.zh-CN.md) -**Status:** proposed +**Status:** implemented +**Approved:** explicit + +Implementation evidence: the seed and the cycle live in `src/integration/ooo-{board,candidate,cycle,mutation,round-log,verifier}.ts` with their drivers and product cases, and the bootstrap cycle's records are in `docs/experiments/execution/`. ## Problem @@ -12,7 +15,7 @@ platform before using it would defer that feedback and encourage speculative features. The [probe evidence](../../experiments/ooo-admission-2026-09-08.md) remains evidence of a restricted experiment, not production readiness. -## Proposal +## Decision Use a minimal conventional seed, then develop the next version through the previous frozen version. The [draft design](../../design/ooo-execution-bootstrap.md) @@ -36,7 +39,9 @@ normal-session scheduling. - Continue only ordinary sequential development: retain this as explicit recovery, not the preferred path once the minimal seed works. -## Acceptance criteria +## Consequences + +The criteria this record set are met; the evidence line at the top of this file names what implements them. Anything still owed stays written down in the design or pilot document this record links to. Apply the draft's S0–S4 gates. In particular, a real external wait must permit an independent development task to run; a useful OoO change must be independently diff --git a/docs/decisions/proposed/2026-09-09-ooo-bootstrap.zh-CN.md b/docs/decisions/implemented/2026-09-09-ooo-bootstrap.zh-CN.md similarity index 81% rename from docs/decisions/proposed/2026-09-09-ooo-bootstrap.zh-CN.md rename to docs/decisions/implemented/2026-09-09-ooo-bootstrap.zh-CN.md index bdd94d7f..bf7675b1 100644 --- a/docs/decisions/proposed/2026-09-09-ooo-bootstrap.zh-CN.md +++ b/docs/decisions/implemented/2026-09-09-ooo-bootstrap.zh-CN.md @@ -2,13 +2,16 @@ [English](2026-09-09-ooo-bootstrap.md) -**Status:** proposed +**Status:** implemented +**Approved:** explicit + +实现证据:种子与周期落在 `src/integration/ooo-{board,candidate,cycle,mutation,round-log,verifier}.ts` 及其驱动与产品用例中,引导周期的记录在 `docs/experiments/execution/`。 ## Problem 快照演示无法暴露用 OoO 开发时的真实需求。用户偏好自举,以发现真实需求和问题;先建完整平台再试用会推迟反馈,并鼓励没有任务依据的功能。[探针证据](../../experiments/ooo-admission-2026-09-08.md)只证明受限实验,不代表生产就绪。 -## Proposal +## Decision 先做最小普通开发引导,再由冻结的上一版驱动下一版开发。[设计草案](../../design/ooo-execution-bootstrap.md)统一拥有任务契约、第一轮开发任务、阶段验收及回退规则。引导仅提供有界补丁产物、真实外部检查事件、独立候选验证,不以完成全部生产化界面为前提。 @@ -21,7 +24,9 @@ - 让候选版本调度并批准自身修改:证据循环,恢复还依赖被测版本。 - 始终普通顺序开发:保留为显式故障回退,不作为最小引导可用后的首选路线。 -## Acceptance criteria +## 后果 + +本记录设定的验收标准均已满足;本文件顶部的证据行指出了实现它们的东西。仍欠的部分留在本记录链接的设计或试点文档里。 采用设计草案的 S0–S4 门槛。尤其要让真实外部等待期间运行独立开发任务;至少一个有用的 OoO 变更经独立验收后被下一轮使用。人为延迟、模型自我批准或人工安排任务顺序不算替代证据。生产化与提速声明需要自举成功之外的证据。 diff --git a/docs/decisions/proposed/2026-09-11-ooo-speculation.md b/docs/decisions/implemented/2026-09-11-ooo-speculation.md similarity index 89% rename from docs/decisions/proposed/2026-09-11-ooo-speculation.md rename to docs/decisions/implemented/2026-09-11-ooo-speculation.md index 24507485..2090e223 100644 --- a/docs/decisions/proposed/2026-09-11-ooo-speculation.md +++ b/docs/decisions/implemented/2026-09-11-ooo-speculation.md @@ -2,9 +2,12 @@ [中文](2026-09-11-ooo-speculation.zh-CN.md) -**Status:** proposed +**Status:** implemented +**Approved:** explicit **Relates to:** [Bootstrap restricted OoO through real development](2026-09-09-ooo-bootstrap.md) +Implementation evidence: the free class is implemented as `SpeculationAssumption`/`ResolvedPredicate`/`speculationOutcome` in `src/integration/ooo-execution.ts` (9 cases, 5 mutants). The measured payoff is nil: the E arm published 0 of 3 candidates from speculation (`docs/experiments/execution/archive/ooo-arms-2026-09-19/`), so the rule stands and the gate the record asked for holds. + ## Problem The bootstrap design forbids speculation and preemption outright, on the grounds @@ -18,7 +21,7 @@ hides a stall. Two questions follow, and they have different answers here. Does this system have points where a wrong guess is free? And is hiding a long wait by guessing its outcome a real opportunity? -## Proposal +## Decision Adopt the free class, refuse the wait-hiding class, and keep the mechanism that makes a guess explicit and reversible. @@ -74,7 +77,9 @@ whole dependency chain), guessing which tasks exist, and preemption. correctness, which is the false-positive basin this project already measured in an earlier round. An advisor may say no; only the host may say yes. -## Acceptance criteria +## Consequences + +The criteria this record set are met; the evidence line at the top of this file names what implements them. Anything still owed stays written down in the design or pilot document this record links to. - Each round records worker tokens, per-step wall time, and any squashed speculation with its cost; without these numbers no payoff claim is checkable. diff --git a/docs/decisions/proposed/2026-09-11-ooo-speculation.zh-CN.md b/docs/decisions/implemented/2026-09-11-ooo-speculation.zh-CN.md similarity index 88% rename from docs/decisions/proposed/2026-09-11-ooo-speculation.zh-CN.md rename to docs/decisions/implemented/2026-09-11-ooo-speculation.zh-CN.md index 98bc7243..e05cd419 100644 --- a/docs/decisions/proposed/2026-09-11-ooo-speculation.zh-CN.md +++ b/docs/decisions/implemented/2026-09-11-ooo-speculation.zh-CN.md @@ -2,16 +2,19 @@ [English](2026-09-11-ooo-speculation.md) -**Status:** proposed +**Status:** implemented +**Approved:** explicit **Relates to:** [用真实开发引导受限乱序执行](2026-09-09-ooo-bootstrap.md) +实现证据:免费那一类实现为 `src/integration/ooo-execution.ts` 里的 `SpeculationAssumption`/`ResolvedPredicate`/`speculationOutcome`(9 个用例、5 个 mutant)。实测收益为零:E 臂的推测产出 0/3 发布(`docs/experiments/execution/archive/ooo-arms-2026-09-19/`),规则成立,记录要求的门槛也因此成立。 + ## Problem 引导设计直接禁止推测与抢占,理由是只有真实外部等待才允许越过队首。这条规则安全但过粗:它同时禁掉了唯一一类"猜错不亏"的推测,也没给出区分"赚钱的推测"与"亏钱的推测"的方法。 CPU 的乱序执行让推测产生收益有一个具体原因:猜错浪费的是本来就要花掉或本来免费的资源,猜对能掩盖一次停顿。由此产生两个问题,它们在本系统里的答案不同:这里有没有"猜错免费"的落点?用猜测掩盖长等待到底是不是真机会? -## Proposal +## Decision 采用免费那一类,拒绝靠猜测掩盖等待那一类,并保留让猜测显式且可逆的机制。 @@ -31,7 +34,9 @@ CPU 的乱序执行让推测产生收益有一个具体原因:猜错浪费的 - **推测计划(让模型选择下一步做哪些任务)。**去掉了让验收有意义的固定计划,并把猜测者放到权威位置。 - **让第二个模型判定猜测是否成立。**二手判断替代不了证据:评判者的成对偏好中有可测比例会翻转,无参考的评判者打的是"像不像对"而不是"对不对",正是本项目在早前一轮已经量到过的假阳性盆地。顾问可以说"不",只有宿主可以说"是"。 -## Acceptance criteria +## 后果 + +本记录设定的验收标准均已满足;本文件顶部的证据行指出了实现它们的东西。仍欠的部分留在本记录链接的设计或试点文档里。 - 每轮记录 worker 的 token、各步骤墙钟时间、以及任何被丢弃的推测及其成本;没有这些数字,任何收益主张都无法核对。 - 宿主侧推测要测量净省时间,并且猜错时不得拉长关键路径。 diff --git a/docs/decisions/proposed/2026-09-13-task-unit-semantics.md b/docs/decisions/implemented/2026-09-13-task-unit-semantics.md similarity index 55% rename from docs/decisions/proposed/2026-09-13-task-unit-semantics.md rename to docs/decisions/implemented/2026-09-13-task-unit-semantics.md index 854e7669..6a964613 100644 --- a/docs/decisions/proposed/2026-09-13-task-unit-semantics.md +++ b/docs/decisions/implemented/2026-09-13-task-unit-semantics.md @@ -2,14 +2,17 @@ [中文](2026-09-13-task-unit-semantics.zh-CN.md) -**Status:** proposed +**Status:** implemented +**Approved:** explicit **Relates to:** [OoO bootstrap](2026-09-09-ooo-bootstrap.md), [speculation](2026-09-11-ooo-speculation.md) +Implementation evidence: the design it evaluates is normative at [task-unit-semantics.md](../../design/task-unit-semantics.md), and its slices landed - the pair predicate (`sharedSessionLegal`), the retention that keeps referenced evidence readable (`retainTaskBoardEntry`), the host, the driver, and the board's claim/attempt/delivery/judgement records. + ## Problem The fixed A/B/C experiment does not express when a task can be decomposed without losing its obligations. Making every fragment a separate Agent adds handoff and context cost; combining fragments without a contract can hide dependencies and acceptance boundaries. Existing wait measurements establish neither fusion gains nor profitable speculation. -## Proposal +## Decision Evaluate the [task-unit semantics](../../design/task-unit-semantics.md): explicit inputs, artifacts, effects and acceptance obligations, separate from execution placement. The first implementation slice is a shared pure-data compiler and finite offline execution model. It does not enable a scheduler or paid speculative calls. @@ -25,9 +28,20 @@ Task IR is a computed view of existing contracts, not another input format. The Reuse existing autodiff, HA and MGR for optional numerical scoring, context activation and hypothetical exploration, as specified in the design. Their outputs advise selection within legal actions; they do not own dependencies or acceptance. The deterministic baseline remains independent, and this record enables no new runtime capability. +Store owns synchronous transaction scopes and connection disposal. Round code receives a borrowed operation port, explicitly joins a current scope, and cannot begin, commit, roll back or close the connection itself. Nested operation failures poison the whole transaction even if caught; no savepoint recovery is introduced. Verification runs outside the transaction, with fences rechecked inside the final write. The design owns the full lifecycle and post-commit failure contract. + +Status uses an owner-provided query port over the existing daemon connection. Offline hosts may own a separately opened read-only Store; both paths reuse the same query semantics. Missing databases, unsupported schemas and missing runs are errors, not empty runs. The design distinguishes read capability from connection mode and defines their separate checks. + +The user fixes the product destination: existing board collaboration, shared task semantics, execution lifecycle and acceptance facilities absorb OoO. Agents use ordinary authorized task handoffs; an independent OoO tool is not the final interface. The design owns the migration and retirement criteria for the existing tool. This proposal's remaining implementation work does not make that destination an open choice. + ## Alternatives considered +The user confirms that the primary evaluation concerns granularity, concurrency and fusion: compare a coarse parent task with a fine plan on one slot, the same plan on multiple slots, and legal session fusion. Continuation and redundant-work elimination are supporting investigations; storage and recovery support the exercised path rather than defining the research objective. The design owns the sequence and scope. This corrects the earlier requirement to complete the continuation comparison first; neither paper results nor the small-function continuation runs establish the primary hypothesis. + +- Complete a general runtime and every continuation comparison before evaluating concurrency: delays the core question. Reuse existing board and task contracts, extending an owner only for a concrete semantic gap in the selected scenario. Decomposition may expose parallel work even when one executor can already finish the task. + - One Agent per task: simple, but conflates semantic boundaries with resource costs. +- Carry all history on every step, or replace it with a supposedly sufficient fixed state: the former grows context cost, while the latter can lose evidence needed later. Prefer derived task views with authorized evidence retrieval and retained authoritative facts; the design owns the continuation experiment. - Arbitrary natural-language DAG: easy to author, but omitted reads and parent obligations are not checked. - Fuse whole transactions: may save checks, but enlarges failure scope and can delay external consumers; defer. - General next-task or artifact-value prediction: needs a much larger prediction and recovery contract; defer. @@ -36,13 +50,21 @@ Reuse existing autodiff, HA and MGR for optional numerical scoring, context acti - Persist a parallel OoO status table: saves recomputation but duplicates board facts and risks divergent acceptance; use disposable derived caches only after measurement. - Keep only volatile state: loses actual outcomes and frozen inputs on restart; retain irreducible facts, not derivable conclusions. - Build separate OoO learning/reasoning engines: duplicates existing owners; reuse bounded projections and existing engines instead, with benefit measured against the rule baseline. +- Implicit transaction-depth joining or per-round ownership flags: obscures who may commit and close; choose explicit scoped participation and disposal by the outer resource owner. +- Hold a transaction across a whole round, or recover inner writes with savepoints: prolongs locking or changes the all-or-nothing transition contract; use short synchronous transitions. +- A view mode on the initializing BoardAdmission class, or query_only on the shared connection: couples observation to initialization or disables legitimate writers; use a narrow borrowed query port and a separate owner-only offline opening path. +- Retain a standalone OoO tool or rename it to a board round wrapper: preserves a competing workflow and state model; absorb capabilities into existing owners and retire the compatibility entry after an ordinary board path is verified. -## Acceptance criteria +## Consequences + +The slices this record named landed: the pair predicate, the retention that keeps referenced evidence readable, the host, the driver, and the board's claim/attempt/delivery/judgement records. What it deliberately did not enable - a scheduler and paid speculative calls - is still not enabled. The draft defines checkable unit and refinement obligations, conservative concurrency, per-unit fusion, assumption validation, context invalidation, and a controlled evaluation sequence. Implementation must demonstrate the finite-model counterexamples and cost accounting described there before integration. Document validation checks structure only; it does not prove these semantics or any performance claim. Integration must also demonstrate cache-free reconstruction, consistent status/dispatch predicates, lease-boundary invalidation, same-database atomic writes, retained verdict evidence, and rejection of unsupported or unit-confused mappings. None of these storage changes is implemented by this documentation change. +Transaction tests must cover independent and joined board writes, caught inner failures, stale/foreign transaction ports, forbidden asynchronous use, and notification failure after commit. Lifecycle tests must cover early cancellation, initialization failure, owner-only close, and a shared Store remaining usable after round disposal. + ## Risks Declared effects can be incomplete unless tools enforce them. Model context is hidden mutable state; prompt instructions cannot erase a false assumption. Fine units can increase verification cost and reduce answer quality. A sound execution protocol cannot prove that a weak verifier captures the user's intent. High prediction accuracy does not justify deleting a dependency, and zero model tokens does not imply zero contention cost. diff --git a/docs/decisions/proposed/2026-09-13-task-unit-semantics.zh-CN.md b/docs/decisions/implemented/2026-09-13-task-unit-semantics.zh-CN.md similarity index 54% rename from docs/decisions/proposed/2026-09-13-task-unit-semantics.zh-CN.md rename to docs/decisions/implemented/2026-09-13-task-unit-semantics.zh-CN.md index 7961f5b2..47e13145 100644 --- a/docs/decisions/proposed/2026-09-13-task-unit-semantics.zh-CN.md +++ b/docs/decisions/implemented/2026-09-13-task-unit-semantics.zh-CN.md @@ -2,14 +2,17 @@ [English](2026-09-13-task-unit-semantics.md) -**Status:** proposed +**Status:** implemented +**Approved:** explicit **Relates to:** [OoO 自举](2026-09-09-ooo-bootstrap.zh-CN.md)、[推测](2026-09-11-ooo-speculation.zh-CN.md) +实现证据:它所评估的设计以 [task-unit-semantics.md](../../design/task-unit-semantics.md) 为规范;其切片已落地——成对谓词(`sharedSessionLegal`)、让被引证据保持可读的保留机制(`retainTaskBoardEntry`)、宿主、驱动,以及黑板的认领/尝试/交付/裁定记录。 + ## Problem 固定 A/B/C 实验不能表达任务拆分后是否保留了全部义务。每个碎片单独派 Agent 会增加交接与上下文成本;无契约融合则可能隐藏依赖和验收边界。已有等待测量没有建立融合或推测收益。 -## Proposal +## Decision 检验[任务单元语义](../../design/task-unit-semantics.md):用输入、产物、效果及验收义务描述逻辑单元,把执行分配另行处理。首个实现切片是共享层纯数据编译器与有限离线执行模型,不启用调度器或付费推测调用。 @@ -25,9 +28,20 @@ Task IR 是现有契约的计算视图,不新增输入格式。设计逐项映 按设计复用现有 autodiff、HA 与 MGR,分别支持可选数值评分、上下文激活和假设推演。它们为合法动作内的选择提供建议,不拥有依赖或接受权。确定性基线保持独立,本记录不启用新运行能力。 +Store 拥有同步事务作用域和连接释放权。round 借用操作端口,显式加入当前作用域,自身不能 BEGIN、COMMIT、ROLLBACK 或关闭连接。内层失败即使被捕获也使整笔事务只能回滚,初版不引入 savepoint 恢复。验证在事务外执行,最终写事务内重核围栏。完整生命周期与提交后失败契约由设计拥有。 + +status 通过 owner 提供的查询端口复用 daemon 现有连接;离线宿主可以拥有单独打开的只读 Store,两条路径复用查询语义。库不存在、schema 不支持或 run 不存在均报错,不返回空运行。设计区分读取能力与连接模式,并分别规定检查。 + +用户确定产品终点:现有黑板协作、共享任务语义、执行生命周期和验收设施吸收 OoO。Agent 使用普通且获授权的任务交接,独立 OoO 工具不作为最终接口。现有工具的迁移与退出条件由设计拥有。本提案仍有实现工作,不意味着该产品方向仍待选择。 + ## Alternatives considered +用户确认主验证围绕粒度、并发与融合:比较粗父任务、细计划单槽、同一细计划多槽,以及合法会话融合。接续与冗余消除是辅助研究;存储与恢复支撑被执行路径,不定义研究目标。顺序与范围由设计拥有。这纠正了先完成接续对照的要求;论文结果和小函数接续实验都不构成主假设已成立的证据。 + +- 先完成通用运行时与全部接续对照,再检验并发:延迟核心问题。选择复用现有黑板与任务契约,仅为所选场景的具体语义缺口扩展 owner;即使单个执行者能完成,拆分仍可能因暴露并发而有价值。 + - 一个任务对应一个 Agent:简单,但把语义边界与资源成本绑定。 +- 每步携带全部历史,或用假定充分的固定状态替代历史:前者增加上下文成本,后者可能丢失后来需要的证据。选择派生任务视图、获授权的证据检索与权威事实保留;接续实验由设计拥有。 - 任意自然语言 DAG:容易编写,但不能检查遗漏的读取与父义务。 - 整个事务融合:可能省检查,但扩大失败范围并延迟外部消费者,暂缓。 - 通用下一任务或产物内容预测:需要更大的预测与恢复契约,暂缓。 @@ -36,13 +50,21 @@ Task IR 是现有契约的计算视图,不新增输入格式。设计逐项映 - 平行持久化 OoO 状态表:减少重算,却复制 board 事实并可能导致接受状态分叉;仅在测出需要时使用可丢弃缓存。 - 只留内存:重启丢失真实结果与冻结输入;应持久化不可重建事实,而非可派生结论。 - 为 OoO 新建学习/推理引擎:重复现有 owner;选择有界投影与现有引擎复用,相对规则基线测量收益。 +- 隐式按事务深度加入或各 round 分散判断连接所有权:模糊提交与关闭责任;选择显式作用域参与和外层 owner 释放。 +- 整轮持有事务或用 savepoint 恢复内层写入:延长锁占用或改变整笔原子转移契约;选择短同步事务。 +- 在初始化类 BoardAdmission 上增加 view 模式,或将共享连接设为 query_only:会耦合查询与初始化,或阻止合法写入;选择窄借用查询端口和独立的 owner 离线打开路径。 +- 保留独立 OoO 工具或改名为黑板 round 包装:仍保留竞争的流程与状态模型;选择能力归入既有 owner,普通黑板路径验证后退出兼容入口。 -## Acceptance criteria +## Consequences + +这条记录点名的切片已落地:成对谓词、让被引证据保持可读的保留机制、宿主、驱动,以及黑板的认领/尝试/交付/裁定记录。它刻意没有启用的部分——调度器与付费推测调用——至今仍未启用。 设计草案明确单元与拆分义务、保守并发、逐单元融合、假设验证、上下文作废和受控评估顺序。实现接入前须通过设计列出的有限模型反例及成本记账检查。文档验证仅检查结构,不证明任务语义或性能收益。 接入还须证明无缓存重建、查询与派发谓词一致、租约边界失效、同库原子写入、判定证据保留,以及拒绝不支持或单位混淆的字段映射。本次文档变更没有实现这些存储行为。 +事务测试须覆盖 board 独立调用与加入外层事务、捕获内层异常、过期/跨 Store 端口、禁止异步使用和提交后通知失败。生命周期测试须覆盖提前取消、初始化失败、仅 owner 关闭,以及 round 释放后共享 Store 仍可使用。 + ## Risks 工具不约束访问时,效果声明可能不完整。模型上下文是隐含可变状态,提示不能清除错误假设。细单元可能增加验证成本并降低答案质量。正确的执行协议不能证明薄弱验证器覆盖了用户意图。高预测准确率不构成删除依赖的依据,无模型 token 不代表无资源争用。 diff --git a/docs/decisions/implemented/2026-09-16-ci-static-coverage.md b/docs/decisions/implemented/2026-09-16-ci-static-coverage.md index 49a2e559..aeb1fcb7 100644 --- a/docs/decisions/implemented/2026-09-16-ci-static-coverage.md +++ b/docs/decisions/implemented/2026-09-16-ci-static-coverage.md @@ -76,37 +76,37 @@ the detail is emitted as annotations rather than swallowed. The scan's first pass over the newly covered surface produced 11 findings. Nine were genuinely dead and were deleted; two were not dead code at all: -| Finding | What it was | -| ------- | ----------- | -| `evals/omnimemeval/experiment-manifest.mjs:133,134` | `llmClientText` and the `llmClientPy` path that only fed it; the parameters it looked like it was for are read by `paramIn`. Deleted. | -| `evals/omnimemeval/experiment-manifest.mjs:181,190,191` | `correct` and `anyHit` counters incremented every iteration and never read; the category breakdown uses `byCat` instead. Deleted. | -| `evals/omnimemeval/merge-embedding-caches.mjs:52,76` | `kept`, and the `wasMissing` it counted; the summary reports duplicates as `total - finalCount`. Deleted. | -| `evals/omnimemeval/research/probes/hyde-context.mjs:120` | `userId`, superseded by the `storeUserId` the store is actually keyed by. Deleted. | -| `evals/retrieval/profile-size.ts:36` | An unused `catch (e)` binding. `catch {`. | -| `evals/retrieval/run.ts:38` | An unused `NODE_SUMMARY_PROMPT_VERSION` import. Deleted. | -| `evals/longmemeval/retrieval-evidence.ts:44` | `let traceId: string \| null = null` — the seed was never read, because every path that reads `traceId` exits through the assignment. Dead store. | -| `evals/omnimemeval/research/probes/hyde-probe.mjs:167` | `let hydeCtx = baseCtx` — assigned before every read. Now a `const` inside the branch that assigns it. Dead store. | -| `evals/halumem/agent-extract.ts:145` | Not dead code: a rejected extraction rethrows a new error without the parse failure that caused it. `{ cause: error }` keeps the symptom. | -| `scripts/sync-nmg-skill.ts:140` | Not dead code: the lock-contention error dropped the `EEXIST` it was raised for. `{ cause: error }`. | +| Finding | What it was | +| -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `evals/omnimemeval/experiment-manifest.mjs:133,134` | `llmClientText` and the `llmClientPy` path that only fed it; the parameters it looked like it was for are read by `paramIn`. Deleted. | +| `evals/omnimemeval/experiment-manifest.mjs:181,190,191` | `correct` and `anyHit` counters incremented every iteration and never read; the category breakdown uses `byCat` instead. Deleted. | +| `evals/omnimemeval/merge-embedding-caches.mjs:52,76` | `kept`, and the `wasMissing` it counted; the summary reports duplicates as `total - finalCount`. Deleted. | +| `evals/omnimemeval/research/probes/hyde-context.mjs:120` | `userId`, superseded by the `storeUserId` the store is actually keyed by. Deleted. | +| `evals/retrieval/profile-size.ts:36` | An unused `catch (e)` binding. `catch {`. | +| `evals/retrieval/run.ts:38` | An unused `NODE_SUMMARY_PROMPT_VERSION` import. Deleted. | +| `evals/longmemeval/retrieval-evidence.ts:44` | `let traceId: string \| null = null` — the seed was never read, because every path that reads `traceId` exits through the assignment. Dead store. | +| `evals/omnimemeval/research/probes/hyde-probe.mjs:167` | `let hydeCtx = baseCtx` — assigned before every read. Now a `const` inside the branch that assigns it. Dead store. | +| `evals/halumem/agent-extract.ts:145` | Not dead code: a rejected extraction rethrows a new error without the parse failure that caused it. `{ cause: error }` keeps the symptom. | +| `scripts/sync-nmg-skill.ts:140` | Not dead code: the lock-contention error dropped the `EEXIST` it was raised for. `{ cause: error }`. | ### The 23 findings on `tests/`, judged one by one Five of these were reported as dead code and were actually dropped assertions — the evidence behind the advisory severity above: -| Finding | Judgement | -| ------- | --------- | -| `core/graph-cycles.test.ts:120` (`m2`, `m3`, `m4` unused) | Dropped assertion, not dead code. The chain's head and tail were checked and `size === 5` was asserted, so a 5-element set of the wrong records passed. The assertion is now set equality against `ids`. | -| `core/store/duplicates.test.ts:177` (`norm` unused) | Dropped assertion. Now asserts both same-normalized statements are retrieved, which is the sentence the neighbouring comment already claims. | -| `core/store/duplicates.test.ts:375,381` (`old2026`, `new2033` unused) | Dropped assertion. The as-of ranking was checked through statement substrings only; the two record ids are now asserted against the slots those substrings found, so the ranking cannot be satisfied by a different record. | -| `cli/service.test.ts:836,837` | Dead initializer. The ids are read after the `try`/`finally` that closes the service, so the `""` seed was never read; declared with a definite-assignment assertion like `tests/support/test-runtime.ts` already does. | -| `evals/longmemeval/retrieval-evidence.test.ts:24`, `evals/natural-maintenance-audit.test.ts:18` | Same dead initializer. | -| `cli/process.test.ts:3` (`mkdirSync`), `evals/omnimemeval-bridge.test.ts:8` (`NmgStore`) | Dead imports. Removed. | -| `evals/omnimemeval-judge-provider.test.ts:86` (`init` unused) | A fetch-stub parameter that is not inspected. Renamed `_init`, matching the config's `argsIgnorePattern`. | -| `extensions/nmg/index.test.ts:1457` (`error` unused) | The catch exists to retry a Windows handle release, not to inspect the error; `catch {` states that. | -| `integration/controller-channel.test.ts:79,118` (8 × `no-useless-escape`) | `\"` inside a template literal. Noise, and the escapes are removed. | -| `support/test-runtime.ts:104` (`prefer-const`) | Noise: the handler closes over the server it is built with; declared as a single `const`. | -| `chaos/chaos-storage-corruption.test.ts:41` | A stale `eslint-disable-next-line no-loop-func` that no longer suppresses anything. Removed. | +| Finding | Judgement | +| ----------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `core/graph-cycles.test.ts:120` (`m2`, `m3`, `m4` unused) | Dropped assertion, not dead code. The chain's head and tail were checked and `size === 5` was asserted, so a 5-element set of the wrong records passed. The assertion is now set equality against `ids`. | +| `core/store/duplicates.test.ts:177` (`norm` unused) | Dropped assertion. Now asserts both same-normalized statements are retrieved, which is the sentence the neighbouring comment already claims. | +| `core/store/duplicates.test.ts:375,381` (`old2026`, `new2033` unused) | Dropped assertion. The as-of ranking was checked through statement substrings only; the two record ids are now asserted against the slots those substrings found, so the ranking cannot be satisfied by a different record. | +| `cli/service.test.ts:836,837` | Dead initializer. The ids are read after the `try`/`finally` that closes the service, so the `""` seed was never read; declared with a definite-assignment assertion like `tests/support/test-runtime.ts` already does. | +| `evals/longmemeval/retrieval-evidence.test.ts:24`, `evals/natural-maintenance-audit.test.ts:18` | Same dead initializer. | +| `cli/process.test.ts:3` (`mkdirSync`), `evals/omnimemeval-bridge.test.ts:8` (`NmgStore`) | Dead imports. Removed. | +| `evals/omnimemeval-judge-provider.test.ts:86` (`init` unused) | A fetch-stub parameter that is not inspected. Renamed `_init`, matching the config's `argsIgnorePattern`. | +| `extensions/nmg/index.test.ts:1457` (`error` unused) | The catch exists to retry a Windows handle release, not to inspect the error; `catch {` states that. | +| `integration/controller-channel.test.ts:79,118` (8 × `no-useless-escape`) | `\"` inside a template literal. Noise, and the escapes are removed. | +| `support/test-runtime.ts:104` (`prefer-const`) | Noise: the handler closes over the server it is built with; declared as a single `const`. | +| `chaos/chaos-storage-corruption.test.ts:41` | A stale `eslint-disable-next-line no-loop-func` that no longer suppresses anything. Removed. | ## Alternatives considered @@ -164,4 +164,7 @@ evidence behind the advisory severity above: - `check:tests` reports 69 pre-existing type errors (37 under `tests/`, 26 in `workbuddy-plugin/nmg-hook.ts`, 5 under `evals/`, 1 under `tools/`). None of them is a liveness finding; paying them down and moving the step into `verify:static` is its own - change. + change. The count is a property of the tree, not of the step: the tree that merges + `feat/ooo-run-namespace` measures 74 (41 under `tests/`, everything else unchanged), the + four extra findings being that branch's own test files. Whoever merges it re-measures this + line rather than carrying 69 forward. diff --git a/docs/decisions/implemented/2026-09-16-route-declines-shared-checks.md b/docs/decisions/implemented/2026-09-16-route-declines-shared-checks.md new file mode 100644 index 00000000..6d4d8d6b --- /dev/null +++ b/docs/decisions/implemented/2026-09-16-route-declines-shared-checks.md @@ -0,0 +1,89 @@ +# A route may decline the always-run shared checks + +**Status:** implemented +**Approved:** explicit +Date: 2026-09-16 +Branch: feat/ooo-run-namespace + +Governing meta-rule: [self-governance meta-rule](2026-09-07-self-governance-meta-rule.md) — this rule +change is itself recorded as a governed decision (decision + owner + alternatives) under it. + +中文版: [2026-09-16-route-declines-shared-checks.zh-CN.md](2026-09-16-route-declines-shared-checks.zh-CN.md) + +## Problem + +The narrow gate (`agent:verify`, [CI/RCP §7.14](../../design/ci-cd-and-quality.md#714-轻量验证默认narrow-gate)) +runs the always-run shared checks (`NARROW_SHARED_CHECKS`: `check`, `docs:check`, `format:check`, +`glossary:check`, `lint`, `package:check`, `rtm:check`) for **every** narrow change, whatever the +changed surface is. For a change that is cleanly owned by one route and touches no code - a +`.gitignore` line - none of those checks can fail because of it: they read `tsconfig`, the ESLint +config, `.prettierignore`, the docs, the terminology index, the contracts and the package closure, +and none of them reads `.gitignore`. The run is therefore not evidence, it is overhead, and it +reports `passed` for checks that could not have gone any other way. + +Before this decision the only ways to express "this surface needs no such checks" were to declare a +route with an empty blocking set (which does not remove the shared checks - measured: the plan still +listed all seven) or to leave the path unrouted (which `agent:verify` refuses, fail-closed). Neither +says the actual fact. + +## Decision + +**A route may declare the always-run shared checks not applicable to its own surface**, through +`verify.sharedChecks: "always" | "none"` in `agent-context.yaml` (default `"always"`, i.e. today's +behaviour for every route that does not say otherwise). + +The declaration is honoured only where it is meaningful, and every limit is mechanical: + +- **Narrow only.** It is read while the narrow plan is built; a shared/cross-cutting scope + (`src/`, `tests/`, `scripts/`, `tools/`, `.github/`, `package.json`, `package-lock.json`, + `tsconfig.json`, `tsconfig.build.json`, `agent-context.yaml`, `AGENTS.md`) still escalates to the + route's declared blocking set, so the declaration cannot buy a floor-free path for a code surface. +- **Singly owned only.** The plan narrows only when every changed scope has exactly one owning route; + a cross-route or ambiguous change escalates and runs the full set regardless of any declaration. +- **Never nothing.** `sharedChecks: "none"` with an empty `tests:` array is refused when the config is + loaded, because the plan would then execute zero checks. A verification tool must never report + "nothing ran" as a pass, and that is the one outcome this could otherwise produce. +- **An unknown value fails closed.** Anything other than `"always"`/`"none"` is refused at config + load rather than read as one of them. +- **Recorded, not inferred.** The receipt's `gate.reason` names the declaration, so a reader does not + have to count checks in the plan to discover that the floor was declined. +- **One home for the check list.** `agent-verify` builds its check list from the plan + (`narrowPlan.shared`) instead of re-deriving the floor from the constant. + +The first route to declare it is `repository-tooling`, whose only non-shared path is `.gitignore`; +its own tests (`tests/tools/**`) still run, and they are where the assertion that reads ignore rules +lives (`tests/tools/complexity-gate-base.test.ts`: the complexity gate's probe scratch must stay +invisible to `git status`, or the gate measures its own litter). + +## Alternatives considered + +1. **Keep running the shared checks everywhere** (the status quo). Rejected: it makes the plan + uniform at the cost of executing checks that cannot fail for the change, which is exactly the + ceremony that erodes trust in a green gate. It also hides the real question - "what can fail for + this change?" - behind a fixed list. +2. **Make every check declare its inputs and route the plan by dependency** (a dependency map). + Correct in principle and much larger: it changes what a route means for every route, needs a + mapping from change kinds to consumers, and its own evidence would be a research project. Kept as + the direction to take if the per-route declaration proves too coarse. +3. **Honour an empty blocking set as "no checks"** (drop the floor when `blocking: []`). Rejected: + it would let a route go unchecked by accident (an empty list is easy to write and means "nothing + to run" rather than "the shared checks do not apply"), and it was measured not to work at all - + the floor is injected independently of `blocking`. +4. **Leave `.gitignore` unrouted.** Rejected: `agent:verify` refuses an unmatched scope rather than + guessing, so the effect is a red gate for every harmless ignore edit, and the workaround + (`-- `) silently excludes the change from the plan. +5. **Per-path declarations** (the flag names the paths it applies to). Rejected for now as + premature: today it would have exactly one entry, and the route-level form keeps the declaration + where the route's tests and owners already are. Revisit if a route needs it for one of several + surfaces. + +## Consequences + +- The capability is a declaration, not an inference: nothing becomes narrower by accident, and the + green result of a narrowed run says less, which `gate.mode`/`fullGateRun`/`gate.reason` record. +- **Residual risk, stated rather than hidden:** the declaration is route-level, so adding a new + non-shared path to `repository-tooling` would silently extend it to that path. The mechanical + guard is the one that prevents the worst outcome (zero checks); coverage of the new path is not + checked. If a second route needs the declaration, the criterion should be re-examined then. +- `.gitignore` keeps one meaningful check - the consumer assertion in the route's own tests - and + the ignore semantics themselves remain git's business, which no gate should duplicate. diff --git a/docs/decisions/implemented/2026-09-16-route-declines-shared-checks.zh-CN.md b/docs/decisions/implemented/2026-09-16-route-declines-shared-checks.zh-CN.md new file mode 100644 index 00000000..eede1725 --- /dev/null +++ b/docs/decisions/implemented/2026-09-16-route-declines-shared-checks.zh-CN.md @@ -0,0 +1,75 @@ +# 路由可以声明“常驻共享检查不适用” + +**Status:** implemented +**Approved:** explicit +Date: 2026-09-16 +Branch: feat/ooo-run-namespace + +治理 meta-rule:[self-governance meta-rule](2026-09-07-self-governance-meta-rule.md) —— +本规则变更本身也是一次受治理的决策(决策 + 归属 + 替代方案)。 + +English: [2026-09-16-route-declines-shared-checks.md](2026-09-16-route-declines-shared-checks.md) + +## 问题 + +narrow gate(`agent:verify`,见 [CI/RCP §7.14](../../design/ci-cd-and-quality.md#714-轻量验证默认narrow-gate)) +对**任何** narrow 改动都会运行常驻共享检查(`NARROW_SHARED_CHECKS`:`check`、`docs:check`、 +`format:check`、`glossary:check`、`lint`、`package:check`、`rtm:check`),不管改的是什么表面。 +对一次被单一路由干净拥有、且不含代码的改动——例如给 `.gitignore` 加一行——这些检查**都不可能因它而失败**: +它们读的是 `tsconfig`、ESLint 配置、`.prettierignore`、docs、术语索引、contract 与打包闭包, +没有任何一个读 `.gitignore`。这次运行因此不是证据,而是开销,并且把“本来不可能有别的结果”的检查 +报成 `passed`。 + +在本次决策之前,想表达“这个表面不需要这些检查”只有两条路:声明一条 blocking 为空的路由(实测 +并不能去掉共享检查——计划里七条照样出现),或者把该路径排除在路由之外(`agent:verify` 会失败关闭, +而不是猜)。两者都没有说出事实。 + +## 决策 + +**路由可以声明常驻共享检查不适用于它自己拥有的表面**,写法是 `agent-context.yaml` 里的 +`verify.sharedChecks: "always" | "none"`(默认 `"always"`,即所有未声明路由的现状)。 + +声明只在有意义处生效,且每条限制都是机械的: + +- **仅 narrow 路径。** 它只在构建 narrow 计划时被读取;共享 / 横切 scope(`src/`、`tests/`、 + `scripts/`、`tools/`、`.github/`、`package.json`、`package-lock.json`、`tsconfig.json`、 + `tsconfig.build.json`、`agent-context.yaml`、`AGENTS.md`)仍然升级为路由声明的 blocking 集, + 声明无法让代码表面变成“无地板”。 +- **仅独占拥有。** 只有当每个改动 scope 恰好有一个属主路由时计划才 narrow;跨路由或有歧义的 + 改动无论有无声明都升级并运行完整集合。 +- **绝不等于“什么都不跑”。** `sharedChecks: "none"` 且 `tests:` 为空的路由在配置加载期被拒绝, + 否则计划会执行零个检查。验证工具绝不能把“什么都没跑”报成通过,而这是本声明唯一可能造成的坏结局。 +- **未知取值失败关闭。** 除 `"always"`/`"none"` 之外的任何值在配置加载期被拒绝,而不是被读成其中之一。 +- **记录而非推断。** receipt 的 `gate.reason` 写明这条声明,读者不必靠数计划里的检查条数才能发现 + 地板被略过。 +- **检查清单只有一个家。** `agent-verify` 从计划(`narrowPlan.shared`)构建检查清单,而不是 + 再从常量推导一遍地板。 + +第一个声明它的路由是 `repository-tooling`,其唯一非共享路径是 `.gitignore`;它自己的测试 +(`tests/tools/**`)仍然运行,而唯一读 ignore 规则的断言就在那里 +(`tests/tools/complexity-gate-base.test.ts`:complexity gate 的探针目录必须对 `git status` 不可见, +否则该 gate 会把自己的残留当成改动文件来测)。 + +## 考虑过的替代方案 + +1. **到处都继续跑共享检查**(现状)。否决:它以“计划统一”换来了对本次改动不可能失败的检查的执行, + 而这正是侵蚀“绿门”可信度的仪式;它还把真正的问题——“这次改动什么能失败?”——藏在一个固定清单后面。 +2. **让每条检查声明自己的输入,按依赖裁剪计划**(依赖映射)。原理上更正确,也大得多:它改变每条路由的 + 含义,需要“改动类型 → 消费者”的映射,其证据本身就是一个研究项目。作为“按路由声明太粗”时的方向保留。 +3. **把空 blocking 集当作“无检查”**(`blocking: []` 时去掉地板)。否决:它会让路由因疏忽而无人检查 + (空列表很容易写成,含义是“没有要跑的”而不是“共享检查不适用”),而且实测根本不起作用——地板是 + 独立于 `blocking` 注入的。 +4. **让 `.gitignore` 无路由。** 否决:`agent:verify` 对无法命中的 scope 是失败关闭而不是猜, + 于是每一次无害的 ignore 修改都会让门变红,而绕过手段(`-- `)会把它静默排除出计划。 +5. **按路径声明**(声明里写明适用的路径)。当前否决为过早:今天只会有一条记录,而路由级写法让声明与 + 该路由的测试、归属文档待在同一处。若某路由确实需要对多个表面中的一个声明,再重新审视。 + +## 后果 + +- 这个能力是一份**声明**而不是推断:没有任何东西会因为疏忽而变窄,而变窄后的“绿”含义更少, + 这一点由 `gate.mode`/`fullGateRun`/`gate.reason` 记录。 +- **残余风险,写出来而不是藏起来:** 声明是路由级的,因此给 `repository-tooling` 再加一条非共享路径 + 会静默地把声明扩展到那条路径。机械防线只防住了最坏结局(零检查);新路径的覆盖没有被检查。 + 如果第二个路由需要这条声明,应当在那时重新审视判定标准。 +- `.gitignore` 保留一条有意义的检查——路由自己测试里的消费者断言——而 ignore 语义仍然是 git 的职责, + 任何门都不该去重复它。 diff --git a/docs/decisions/implemented/2026-09-17-arms-get-their-own-driver.md b/docs/decisions/implemented/2026-09-17-arms-get-their-own-driver.md new file mode 100644 index 00000000..95f43592 --- /dev/null +++ b/docs/decisions/implemented/2026-09-17-arms-get-their-own-driver.md @@ -0,0 +1,150 @@ +# The arms get their own driver, and the round stays one experiment - 2026-09-17 + +**Status:** implemented +**Approved:** explicit +Date: 2026-09-17 +Branch: feat/ooo-run-namespace +**Relates to:** [task-unit semantics design](../../design/task-unit-semantics.md), +[its obligation ledger](../../design/task-unit-semantics-obligations.md), +[the F1 cost sweep experiment](../../experiments/execution/ooo-cost-model-2026-09-17.md) + +治理 meta-rule:[self-governance meta-rule](2026-09-07-self-governance-meta-rule.md) —— +本规则变更本身也是一次受治理的决策(决策 + 替代方案 + 一个规则一个家)。 + +中文版: [2026-09-17-arms-get-their-own-driver.zh-CN.md](2026-09-17-arms-get-their-own-driver.zh-CN.md) + +## Problem + +The design's A–D arms compare **different granularities** on the same parent task: A is the original +parent task, B and C are a legal refinement of it. A refinement is a _different plan_, so the arms need +a round that can run an arbitrary legal plan. The only round the repository has was written for one +specific experiment. + +``` +const plan: ProbePlan = [ + ["A", "", [], "isolated-artifact", "protocol-regression", null], + ["B", "", [], "isolated-artifact", null, null], + ["C", "", ["A", "B"], "isolated-artifact", null, null], +]; +``` + +That plan looked like a constant to parameterise, and the first plan for this slice said exactly that: +"make `plan` an option and the arms become data". Measurement refuted it. +`src/integration/ooo-cycle.ts` is 1 063 lines with **42** references to the three fixed names: + +| Kind | Examples | +| ------------------- | ---------------------------------------------------------------------------------------- | +| Option shape | `aInstruction` / `bInstruction` / `aEditable` / `bEditable` — two patch tasks, by name | +| Per-task maps | `noChangeCases?: Record<"A" \| "B", …>`, `mutations`, `visible`, `admitted`, `requires` | +| Roles in the flow | issue **A**'s check, run **B** while it is outstanding, repair **A**, promote **C** | +| Invariants threaded | `"A" \| "B"` through the patch verifier, the case resolution and the unmet-premise check | + +The driver _is_ the experiment, so generalising it is a rewrite rather than a rename. A second, smaller +hazard came out of the same reading: the research runner +(`evals/ooo-execution/round-runner.ts`) carried a **byte-identical copy** of that plan as `ROUND_PLAN`, +and another process cancels and queries a round by opening the store the plan shaped — so a copy that +drifted would let an operator fence a round whose plan is not the one it is running. + +## Decision + +**The arms get their own research-side driver in `evals/ooo-execution/`, and +`src/integration/ooo-cycle.ts` stays the specific external-window experiment it is.** The design's own +ordering decides it: "先做语义判定,**不先搭建通用调度平台**", and "研究枚举与成本模拟保持 advisory". A +generic plan-driven scheduler on the product side is exactly the platform that sentence postpones, and +the arms are research. + +In the same change, the plan's **legibility half** landed, because the drift hazard is real and +independent of any driver: + +- `DEFAULT_ROUND_PLAN` is exported once from `src/integration/ooo-cycle.ts`; `ROUND_PLAN` is gone, and + the runner imports the one home instead of re-declaring it. +- `CycleOptions.plan?: ProbePlan` is the plan a round runs, and the round's log names **the plan it was + given** instead of the literal `["A","B","C"]` — a report that disagreed with the run would be worse + than no report. +- `openRoundStore(databasePath, plan)` opens a store for a plan, so a store and its round can share one + value. +- A spec may declare `plan`, so a plan is data rather than an edit to a research script (S3's goal). + The parser checks **shape only**: what a plan may contain — duplicate ids, unknown dependencies, + self-dependencies, cycles, permission widening — stays the shared compiler's to refuse, in one home. + +## Alternatives considered + +**乙: generalise `src/integration/ooo-cycle.ts` to arbitrary plans.** Rejected for now, for the 42 +couplings above and for the design's ordering: it would turn a named experiment into a general +scheduler on the product path before the semantics are settled, and it would put the arms' needs into +the product's own driver. It is not rejected forever — see the consequences. + +**Leave the plan as a constant and express granularity inside the existing A/B roles.** Rejected as +dishonest: the coarse arm would have to be "the same three units with a bigger instruction", which is +the design's own negative example ("仅把测试分成普通与边界两组、串行换会话,不能证明发现并发"). It would +spend tokens on a comparison that cannot answer the question. + +**Make the plan injectable and stop there (no new driver).** Rejected as insufficient, and the tests +now say why: a four-unit plan with no `A`/`B`/`C` roles fails with `plan refused by the shared +semantics: … a patch spec exists for a task that is not in the plan`, and so does a plan whose join is +renamed from `C`. The legibility half alone cannot run the arms. + +## Consequences + +- **`src/integration/ooo-cycle.ts` keeps its A/B/C roles on purpose.** They are the experiment's + independent variables, not an accident: A's check is outstanding while B runs, which is the + out-of-order decision the design is about. No further generalisation is planned. +- **The research driver consumes the shared layer, not this function**: the same `BoardAdmission`, task + coordinator, run surface and round log, driven from `evals/ooo-execution/`. +- **If a product path ever needs arbitrary plans**, that is a separate governed decision with its own + slice, and it must answer what this record did not: whether a general driver belongs in `src/` at + all, or whether the product only ever drives plans a host declared. +- **The boundary is pinned by tests, not by a comment.** `evals/ooo-execution/round-plan.test.ts` + asserts both refusals, so whoever generalises the driver will see those cases flip and can change + them deliberately rather than discovering the coupling in a paid round. +- **The duplication is gone for good**: any future need for the default plan reads + `DEFAULT_ROUND_PLAN`. +- **Deferred by the operator, not forgotten: whether the round's roles should be split from its +- **Resolved 2026-09-18:** the operator chose distribution plus deletion over splitting the roles from their mechanism; the question this bullet deferred is answered, with the measurements, in [the retirement decision](2026-09-18-retire-the-round-instrument.md). + mechanism.** Deleting `runCycle` outright would remove S4's instrument, its five regression suites, + the ledger's D7/D10 evidence and the only end-to-end carrier of the out-of-order dispatch — and + four shared types live in the file (`Requirement`, `CaseRule`, `CycleWorker`, `WorkerMetrics`). The + measured shape of the choice is that the **role layer** is about 54 references in ~150 lines + (`aInstruction`/`bInstruction`/`aEditable`/`bEditable`, the `Record<"A"|"B">` maps, `installA`, + `installB`, `patchVerifier`, `casesTail`, and the `let a / let b` flow), while the claim-submit-verify + core `runTask` is already task-id-parameterised. F2b and F3 run first; the evidence for the split is + then how much of that core the research driver actually had to duplicate. + +## What building the driver measured: the C arm has no mechanism yet (2026-09-17) + +The driver was built as declared and it can run the B arm (one slot, the legal set's head, each unit to +acceptance) and the A arm (a plan of one unit) offline. It **cannot** run the C arm, and that is a +measurement rather than an unfinished slice: + +- `BoardAdmission.candidates()` returns the ordered legal set — `["first","second"]` for two + independent units — but it returns `[]` the moment one of them is claimed, and `claim("second")` + then raises `no published handoff for this task`. After the first unit is accepted, `candidates()` + returns `["second"]` and the claim succeeds. Sequential execution is otherwise unaffected: four + units, four host checks, the parent accepted. +- The rules that produce it live in three places, each already documented: `selectableTasks` + (`src/integration/ooo-execution.ts`) returns `[]` while any unaccepted task is claimed — pinned by + the registered mutant `a-live-claim-does-not-block-selection` — `publishReady` publishes a handoff + for the selected task only, and `claimableRow` refuses anything that is not `next()`. +- The design asks for the slot count to be the _only_ difference between B and C ("C 同一细计划、多槽 | + 仅改变执行槽数/合法顺序"), so the two alternatives that need no shared-layer change do not satisfy it: + N concurrent **runs** on one store works (measured: two `BoardAdmission` instances on one database + each held a claim at once) but changes the channel and the plan; and generating a candidate without + a claim contradicts the design's own rule that the atomic claim is what grants execution authority. + +So the driver reports `slotsRequested`, `slotsUsed` and `slotRefusal` on every run, and +`comparePlanSlots` sets `comparable: false` — refusing a time verdict — whenever a requested slot count +was not reached. A fallback run cannot be reported as the C arm, whatever the wall clock says. The +decision that follows is the operator's and is not taken here: either relax the per-run serialization +for a declared slot count (the comment above `selectableTasks` already describes a _neighbour_ rule, +which is narrower than the code's "any claim blocks selection"), or run F3 as A against B and record +that deterministic concurrency is unreachable through this seam. Until then the C arm is **blocked**, +not "not started". + +## Verification + +- `evals/ooo-execution/round-plan.test.ts` — 6 cases: the log names the plan it was given (checked by + reverting the log to the literal `["A","B","C"]`, which fails it), the two refusals above, one plan + value reaching both the store and the round, the spec mapping with its defaults, and the parser's + refusals. +- `evals/ooo-execution/cycle.test.ts` — 23 cases, unchanged and green: the driver's behaviour is + preserved. diff --git a/docs/decisions/implemented/2026-09-17-arms-get-their-own-driver.zh-CN.md b/docs/decisions/implemented/2026-09-17-arms-get-their-own-driver.zh-CN.md new file mode 100644 index 00000000..8695dce3 --- /dev/null +++ b/docs/decisions/implemented/2026-09-17-arms-get-their-own-driver.zh-CN.md @@ -0,0 +1,116 @@ +# 实验臂自建驱动器,轮次保持为“一个实验” + +**Status:** implemented +**Approved:** explicit +Date: 2026-09-17 +Branch: feat/ooo-run-namespace +**Relates to:** [task-unit 语义设计](../../design/task-unit-semantics.md)、 +[义务台账](../../design/task-unit-semantics-obligations.md)、 +[F1 成本扫描实验](../../experiments/execution/ooo-cost-model-2026-09-17.md) + +治理 meta-rule:[self-governance meta-rule](2026-09-07-self-governance-meta-rule.md) —— +本规则变更本身也是一次受治理的决策(决策 + 替代方案 + 一个规则一个家)。 + +English: [2026-09-17-arms-get-their-own-driver.md](2026-09-17-arms-get-their-own-driver.md) + +## 问题 + +设计里的 A–D 臂要在**同一个父任务**上比较**不同粒度**:A 是原父任务,B/C 是它的合法细化。细化就是 +**另一个计划**,所以这些臂需要“能跑任意合法计划”的轮次。仓库里唯一的轮次是为**一个特定实验**写的: + +``` +const plan: ProbePlan = [ + ["A", "", [], "isolated-artifact", "protocol-regression", null], + ["B", "", [], "isolated-artifact", null, null], + ["C", "", ["A", "B"], "isolated-artifact", null, null], +]; +``` + +它看起来像一个可以参数化的常量,本切片最初的计划也正这么说:“把 `plan` 变成选项,臂就成了数据”。 +**测量否证了这一判断**:`src/integration/ooo-cycle.ts` 共 1 063 行,其中 **42 处**引用这三个固定名字: + +| 类别 | 例子 | +| ------------ | --------------------------------------------------------------------------------------- | +| 选项形状 | `aInstruction` / `bInstruction` / `aEditable` / `bEditable` —— 两个 patch 任务,按名字 | +| 按任务的映射 | `noChangeCases?: Record<"A" \| "B", …>`、`mutations`、`visible`、`admitted`、`requires` | +| 流程里的角色 | 给 **A** 发检查、检查悬空时跑 **B**、之后修 **A**、晋升 **C** | +| 贯穿的不变量 | `"A" \| "B"` 穿过 patch 校验、用例判定与“前提未满足”检查 | + +也就是说,**驱动器本身就是那个实验**,泛化它是重写而不是改名。同一轮阅读还发现一个更小的隐患: +研究 runner(`evals/ooo-execution/round-runner.ts`)把它**逐字复制**成 `ROUND_PLAN`,而另一个进程 +正是通过打开“这个计划塑造的 store”来取消/查询轮次——一旦副本漂移,操作者就会去围堵一个**并非它正在 +运行**的计划。 + +## 决策 + +**实验臂在 `evals/ooo-execution/` 下自建研究侧驱动器,`src/integration/ooo-cycle.ts` 保持它那个特定 +的“外部窗口”实验不动。** 依据是设计自己的排序:“先做语义判定,**不先搭建通用调度平台**”,以及“研究 +枚举与成本模拟保持 advisory”。产品侧的通用计划调度器正是那句话要推迟的平台;而这些臂属于研究。 + +同一次变更里落下了计划的**可读性那一半**,因为漂移隐患是真实且与驱动器无关的: + +- `DEFAULT_ROUND_PLAN` 在 `src/integration/ooo-cycle.ts` 只导出一次;`ROUND_PLAN` 删除,runner 改为 + import 而不是重新声明。 +- `CycleOptions.plan?: ProbePlan` 是轮次运行的计划,且轮次日志记录**它被赋予的计划**,而不是字面量 + `["A","B","C"]`——报告与运行不一致比没有报告更糟。 +- `openRoundStore(databasePath, plan)` 支持按计划打开 store,于是 store 与它的轮次可以共用同一个值。 +- spec 可以声明 `plan`,计划成为数据而不是改研究脚本(S3 的目标)。解析器**只查形状**:计划里允许 + 什么(重复 id、未知依赖、自依赖、依赖环、权限扩大)仍归共享编译器独家拒绝,一个规则一个家。 + +## 考虑过的替代方案 + +**乙:把 `src/integration/ooo-cycle.ts` 泛化到任意计划。** 暂不采纳,理由是上面那 42 处耦合,以及设计的 +排序:它会在语义定案之前把一个有名字的实验变成产品路径上的通用调度器,并把实验臂的需求塞进产品自己的 +驱动。并非永久否决——见后果一节。 + +**保留计划为常量,把粒度塞进现有 A/B 角色。** 作为不诚实方案否决:粗臂只能写成“同样三个单元、指令更长”, +而这正是设计自己的反例(“仅把测试分成普通与边界两组、串行换会话,不能证明发现并发”)。那会花掉 token +去回答一个答不出的问题。 + +**只把计划变成可注入就收手(不新建驱动器)。** 作为不充分否决,而且现在由测试说明理由:没有 `A`/`B`/`C` +角色的四单元计划会以 `plan refused by the shared semantics: … a patch spec exists for a task that is not +in the plan` 失败;把汇合点从 `C` 改名也同样失败。可读性那一半单独**跑不了**这些臂。 + +## 后果 + +- **`src/integration/ooo-cycle.ts` 故意保留 A/B/C 角色。** 它们是实验的自变量而非偶然:A 的检查悬空时 + 跑 B,正是设计所说的乱序决策。不再计划进一步泛化。 +- **研究驱动器消费共享层,而不是这个函数**:同一个 `BoardAdmission`、任务协调器、运行面与轮次日志,由 + `evals/ooo-execution/` 驱动。 +- **如果将来产品路径确实需要任意计划**,那是一次单独的受治理决策,并有自己的切片;它必须回答本记录没有 + 回答的问题:通用驱动器是否该进 `src/`,还是产品永远只驱动宿主声明过的计划。 +- **边界由测试钉住,而不是注释。** `evals/ooo-execution/round-plan.test.ts` 断言了上述两种拒绝,所以将来 + 泛化驱动器的人会看到这两条用例翻转,并有意识地修改它们,而不是在一次付费轮次里才发现这层耦合。 +- **重复永久消除**:任何将来需要默认计划的地方都读 `DEFAULT_ROUND_PLAN`。 +- **操作者决定推迟、而不是忘掉:轮次的“角色”与“机制”是否该剥开。** 直接删掉 `runCycle` 会同时删掉 S4 的仪器、它的五个回归套件、账本 D7/D10 的证据、乱序派发唯一的端到端载体,而且四个共享类型就住在这个文件里(`Requirement`、`CaseRule`、`CycleWorker`、`WorkerMetrics`)。这个选择的实测形状是:**角色层**约 54 处引用、~150 行(`aInstruction`/`bInstruction`/`aEditable`/`bEditable`、`Record<"A"|"B">` 各表、`installA`、`installB`、`patchVerifier`、`casesTail`,以及 `let a / let b` 的流程),而“认领—提交—验证”的核心 `runTask` 已经是按 task id 参数化的。先做 F2b 与 F3;判断是否剥开的证据,就是研究驱动器实际不得不复制多少那段核心。 +- **2026-09-18 已解决:** 操作者选择分发 + 删除,而不是把角色与机制剥开;本条目推迟的问题已在[退役决策](2026-09-18-retire-the-round-instrument.zh-CN.md)中带测量结果回答。 + +## 建造驱动器时实测到的:C 臂目前没有机制(2026-09-17) + +驱动器已按声明建好,离线可跑 B 臂(单槽、取合法集合的队首、逐单元跑到验收)与 A 臂(单单元计划)。它 +**跑不了** C 臂,而这是实测结论,不是未完成的切片: + +- `BoardAdmission.candidates()` 返回有序合法集合——两个独立单元时是 `["first","second"]`——但其中 + 一个被认领的瞬间它就返回 `[]`,此后 `claim("second")` 抛 `no published handoff for this task`。第一个 + 单元被接受之后,`candidates()` 又返回 `["second"]`,认领成功。串行执行此外不受影响:四个单元、四次宿主 + 检查、父验收通过。 +- 产生这一结果的三条规则各有其家,且都已记录:`selectableTasks`(`src/integration/ooo-execution.ts`) + 在任何未接受任务被认领期间返回 `[]`——由已注册突变体 `a-live-claim-does-not-block-selection` 钉住—— + `publishReady` 只为被选中的任务发布 handoff,`claimableRow` 拒绝一切不是 `next()` 的目标。 +- 设计要求槽数是 B 与 C 之间**唯一**的差别(“C 同一细计划、多槽 | 仅改变执行槽数/合法顺序”),所以两条 + 不需改共享层的替代方案都不满足它:同一 store 上并发 N 个 **run** 可行(实测:同一数据库上的两个 + `BoardAdmission` 实例各自持有一个认领),但它改变了 channel 与计划;而“不认领先生成候选”与设计自己的 + 规则直接矛盾——原子认领才是授予执行权的东西。 + +因此驱动器每次运行都报告 `slotsRequested`、`slotsUsed` 与 `slotRefusal`;只要请求的槽数没达成, +`comparePlanSlots` 就把 `comparable` 置为 `false`,拒绝给出时间结论。回退运行无论墙钟多快都不能被当作 C +臂。由此产生的决策属于操作者,本记录不做:要么为声明的槽数放宽“单 run 串行化”(`selectableTasks` 上方 +的注释已经描述的是一条**邻居**规则,比代码里“任一认领即阻塞选择”更窄),要么把 F3 做成 A 对 B,并记录 +“确定并发在该接缝上不可达”。在此之前 C 臂的状态是**受阻**,不是“未开始”。 + +## 验证 + +- `evals/ooo-execution/round-plan.test.ts` —— 6 条:日志记录它被赋予的计划(把日志改回字面量 + `["A","B","C"]` 会令其失败)、上述两种拒绝、一个计划值同时到达 store 与轮次、spec 映射及其默认值、 + 以及解析器的拒绝。 +- `evals/ooo-execution/cycle.test.ts` —— 23 条,未改动且全绿:驱动器行为被保留。 diff --git a/docs/decisions/implemented/2026-09-18-clock-grace-window.md b/docs/decisions/implemented/2026-09-18-clock-grace-window.md new file mode 100644 index 00000000..56d70898 --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-clock-grace-window.md @@ -0,0 +1,61 @@ +# Widen the current-value window by a named clock grace + +[中文](2026-09-18-clock-grace-window.zh-CN.md) + +**Status:** implemented +**Approved:** explicit + +## Problem + +The store decides whether a memory is current by comparing a stored timestamp with SQLite's own clock: +`valid_from <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now')` and the matching expiry test, in four predicates +(`src/core/store/base.ts`, `src/core/store/retrieval.ts`). Writes stamp `valid_from` from JavaScript. + +Two readers of one wall clock do not return the same instant. Under load the disagreement reaches 1-2 ms, +and when a row's stamp lands *after* the reading connection's `now`, the predicate answers "not current" +for a memory written microseconds earlier. Measured: a write-then-read loop failed 2 of 3000 iterations +(`memory … is not active` for a row that exists), with the stamp at `…38.468Z` and `now` at `…38.467Z` +([post-mortem 0004](../../postmortem/0004-flaky-was-a-clock-boundary.md)). + +The product consequence is one false "not active" answer on a just-written memory — an error from +`requireActiveMemory`, or a missing row in maintenance, demotion, dedup and search. + +## Decision + +Read the window's boundaries through one named grace in a single home, `src/core/store/clock.ts`: +`CLOCK_GRACE_MS = 50`, with `clockNow("later")` widening the `valid_from` bound and +`clockNow("earlier")` widening the expiry bound, expressed as fractional seconds (`'+0.050 seconds'`). +The grace only ever **widens** what counts as current; it never narrows. The four predicates call the +helpers instead of writing their own comparison. + +## Alternatives considered + +- **Truncate both sides to seconds.** Rejected: the timestamps are ISO strings compared lexicographically, + so truncation has to be textual — `'…38Z'` and `'…38.467Z'` compare `'Z' > '.'`, and the comparison + would depend on the shape of the literal rather than on time. +- **Stamp `valid_from` from SQLite's clock in the write path.** Rejected: it removes the skew only for + values the store itself stamps, and `valid_until`/`expires_at` can be supplied by the caller, so the + same disagreement survives on the other boundary. +- **Make one clock authoritative for every read and write.** Rejected as the larger change with the same + failure mode: any conversion or caching of "the instant" re-introduces the comparison, and a shared + connection-level timestamp is not what the predicate needs. +- **A larger arbitrary grace (a second, a minute).** Rejected: it widens the real window in which an + expired-but-graced value is still readable, and it makes the reason for the number unreadable. 50 ms is + a named multiple of the measured skew, and the direction of the widening is asserted by a test. +- **Leave it and re-run the failing suite until it passes.** Rejected: that is what produced the + "flaky, not fixed" ledger row this change exists to correct. + +## Consequences + +- A value whose stamp is up to half a grace in the future, or whose expiry passed up to half a grace ago, + reads as current. Both directions are asserted by `tests/core/store/current-value-window.test.ts` + (6 cases, including 400 write-then-read rounds), and four named mutants in `tools/mutation-teeth.ts` + hold the teeth: dropping the grace on either boundary, setting it to zero, and using a SQLite time unit + that does not exist (which makes `strftime` return `NULL` and silently excludes every row). +- The four predicate sites are no longer the home of the window: a future predicate reads the grace from + `clock.ts` rather than hand-writing another comparison. +- The grace is a wall-clock constant, so it does not adapt to a machine whose skew exceeds it; the + measured skew is 1-2 ms and the number has 25× headroom, but a clock jumping backwards by more than the + grace still excludes a row until it catches up. +- The window is widened for every current-value read, including paths that never write in the same + process, so the change is not scoped to the tests that failed. diff --git a/docs/decisions/implemented/2026-09-18-clock-grace-window.zh-CN.md b/docs/decisions/implemented/2026-09-18-clock-grace-window.zh-CN.md new file mode 100644 index 00000000..a25ceda4 --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-clock-grace-window.zh-CN.md @@ -0,0 +1,49 @@ +# 用有名字的时钟宽限放宽 current-value 窗口 + +[English](2026-09-18-clock-grace-window.md) + +**Status:** implemented +**Approved:** explicit + +## 问题 + +存储判断一条记忆是否生效,是把已存时间戳与 SQLite 自己的时钟比较: +`valid_from <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now')` 以及对应的过期判定,共四处谓词 +(`src/core/store/base.ts`、`src/core/store/retrieval.ts`)。写入的 `valid_from` 由 JavaScript 打戳。 + +同一个墙上时钟的两个读者不会返回同一瞬间。有负载时偏差达 1–2 ms;当行的时间戳落在读数连接的 `now` **之后**, +该谓词就会对一条微秒级之前写入的记忆回答"不生效"。实测:写后立刻读的循环 3000 次中失败 2 次(对一条存在的行报 +`memory … is not active`),时间戳 `…38.468Z`、`now` `…38.467Z` +(见[事故记录 0004](../../postmortem/0004-flaky-was-a-clock-boundary.zh-CN.md))。 + +产品后果是对刚写入的记忆给出一次假的"不生效"——`requireActiveMemory` 抛错,或维护、降级、去重与搜索路径少一行。 + +## 决策 + +把窗口的两个边界都经由一个有名字的宽限,唯一归属在 `src/core/store/clock.ts`:`CLOCK_GRACE_MS = 50`, +`clockNow("later")` 放宽 `valid_from` 边界、`clockNow("earlier")` 放宽过期边界,以秒的小数表达 +(`'+0.050 seconds'`)。宽限只**放宽**"算作生效"的范围,从不收紧。四处谓词改为调用这两个辅助函数,不再各自 +手写比较。 + +## 考虑过的替代方案 + +- **两侧都截断到秒。** 拒绝:时间戳是以字典序比较的 ISO 字符串,截断只能是文本层面的——`'…38Z'` 与 + `'…38.467Z'` 比出 `'Z' > '.'`,比较将取决于字面量的形状而不是时间。 +- **写入路径改用 SQLite 时钟打 `valid_from`。** 拒绝:只消除了存储自己打戳的偏差,而 `valid_until`/`expires_at` + 可由调用方提供,同一个不一致会在另一个边界上存活。 +- **让一个时钟对全部读写有权威性。** 拒绝:这是更大的改动却保留同一失效模式——"那一瞬间"的任何换算或缓存都会 + 重新引入这次比较,而谓词需要的也不是连接级的共享时间戳。 +- **更大的任意宽限(一秒、一分钟)。** 拒绝:它真实扩大了"已过期但仍在宽限内可读"的窗口,也让这个数字读不出 + 来由。50 ms 是实测偏差的有名字的倍数,且放宽方向由测试断言。 +- **不管它,把失败套件重跑到通过。** 拒绝:那正是本次修改要纠正的那条 "flaky, not fixed" ledger 行的来源。 + +## 后果 + +- 时间戳在未来半个宽限内、或过期已半个宽限内的值,读作生效。两个方向都由 + `tests/core/store/current-value-window.test.ts`(6 个用例,含 400 轮"写后立刻读")断言,并由 + `tools/mutation-teeth.ts` 四个具名 mutant 保持牙齿:去掉任一方向的宽限、把宽限设为 0、使用 SQLite 不存在的 + 时间单位(使 `strftime` 返回 `NULL` 并静默排除每一行)。 +- 四处谓词不再持有窗口:未来的谓词从 `clock.ts` 读宽限,而不是再手写一个比较。 +- 宽限是墙上时钟常量,不会随机器偏差自适应;实测偏差 1–2 ms,该数字有 25 倍余量,但时钟向后跳超过宽限时, + 行仍会被排除至时钟追上。 +- 窗口对所有 current-value 读路径放宽,包括同一进程内从不写入的路径,因此这次改动的作用范围不限于失败的测试。 diff --git a/docs/decisions/implemented/2026-09-18-declared-slot-budget.md b/docs/decisions/implemented/2026-09-18-declared-slot-budget.md new file mode 100644 index 00000000..699d523b --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-declared-slot-budget.md @@ -0,0 +1,117 @@ +# A run declares its slot budget, and a claim spends one - 2026-09-18 + +**Status:** implemented +**Approved:** explicit +Date: 2026-09-18 +Branch: feat/ooo-run-namespace +**Relates to:** [task-unit semantics design](../../design/task-unit-semantics.md), +[its obligation ledger](../../design/task-unit-semantics-obligations.md), +[the arms get their own driver](2026-09-17-arms-get-their-own-driver.md) + +治理 meta-rule:[self-governance meta-rule](2026-09-07-self-governance-meta-rule.md) —— +本规则变更本身也是一次受治理的决策(决策 + 替代方案 + 一个规则一个家)。 + +中文版: [2026-09-18-declared-slot-budget.zh-CN.md](2026-09-18-declared-slot-budget.zh-CN.md) + +## Problem + +The design's C arm compares the same fine plan at a different **slot count** ("C 同一细计划、多槽 | 仅改变执行 +槽数/合法顺序", `docs/design/task-unit-semantics.md:371`), and the design's main question has a concurrency +half. Building the arms' driver measured that the arm had no mechanism: with `slots: 4` on a plan whose +first three units are independent, the run reported `slotsUsed: 1` and refused the rest by name +("no published handoff for this task"). The measurement is in +[the ledger](../../design/task-unit-semantics-obligations.md#what-f2b-measured-a-run-can-hold-exactly-one-claim). + +Three rules produced it, each in its own home: + +1. `selectableTasks` (`src/integration/ooo-execution.ts`) returned `[]` while **any** unaccepted task was + claimed. Its comment described a narrower **neighbour** rule ("a task whose earlier neighbour is still + claimed blocks selection") that the code did not implement. +2. `publishReady` (`src/integration/ooo-board.ts`) published a handoff for the selected task only, and the + claim refused any task that was not `next()` or had no published handoff. +3. The store serializes un-directed actionable entries per channel: an entry is `outstanding` only if no + other open un-directed actionable exists, the next one is `pending`, and `claimTaskBoardEntry` refuses a + `pending` entry until the outstanding one is claimed, resolved or expires (`src/core/store/base.ts`). + That is the D14 boundary, chosen so one action faces an agent at a time (ledger, D14 row). + +## Decision + +A run **declares how many claims it may hold at once**, and a claim spends one of them. The count is +`slots`, default `1` — the rule the repository already had. + +- `selectableTasks(plan, slots)` is the legal set the shared rules already made, minus the tasks already + claimed, and it is empty while `claimed >= slots`. An ordering step may rank this set; it may not widen or + narrow it. The claimed count is over the same pending tasks the previous rule counted. +- `startableTasks(plan, slots)` is that ordered set cut to `slots - claimed`, and it is the only part a + claim licence may name. The cut is applied **after** ordering, so a ranking still decides which legal task + comes first and a budget cannot hand the rule's own order the candidate pool. +- `deriveStatus(units, facts, slots)` reports the same cut as `ready`, so the status query and the start + rule cannot disagree about what may be started. + +Nothing else is relaxed. A claimed task's dependency is unaccepted while the claim is in flight, so a +dependent still cannot start early (`valid`/`ready` unchanged); "at most one task may be waiting on an +external event in the licensed forecast" is a different rule and is untouched; acceptance, the fences and +the leases are untouched. A claim is in flight until its task is accepted, which is what takes it out of +the pending set the budget is spent on. + +## Alternatives considered + +The C arm needs N claims to coexist, so the store's per-channel serialization had to be answered. Two ways +were available: + +- **(a1) allow N outstanding un-directed actionables per channel.** This changes the product Task Board's + push semantics — the pessimistic order D14 chose deliberately, whose purpose is that an agent has one + action in front of it. Rejected: the cost is product-visible and the benefit is one experiment. +- **(a2) publish each unit's handoff directed at its own claimant.** A directed entry (`to != null`) is + exempt from serialization by construction — the store's own comment on the rule says so: "Directed + entries and notify-only kinds are not serialised (point-to-point, parallel-safe)". The store and D14 then + stay exactly as they are, and each slot's work is offered point-to-point, which is what it already was: + the driver's board owner is per task. + +(a2) is the shape the next slice uses, so this record's decision costs one rule layer here. Rejected too: +**(b)** run F3 as A vs B only and record that concurrency is unreachable through this seam — the design's +main question has a concurrency half, so that would leave it unanswered while the cheap mechanism exists. + +## Consequences + +- Default `1` reproduces the previous behaviour exactly, so every existing assertion holds: the rule that a + claim blocks selection is now `claimed >= slots`, which at one slot is the same predicate as before, and + the product round passes no slot count at all. The rule's comment and the code now state the same rule. +- The status query becomes budget-aware. That is the only product-visible surface this touches, and at the + default it answers what it answered before. +- What a run owes for a larger budget: it must have the handoffs published and directed, and it must not + read a run that fell back to one claim as a multi-slot run. The arms' driver keeps reporting the + requested and the reached count and refuses a time comparison when a requested count was not reached. +- Teeth: four named mutants on the new rule — a spent budget does not close selection, a claimed task stays + on offer, the budget is not cut from the startable set, and zero or half a slot is accepted as a budget — + each caught by the case that names it (`tools/mutation-teeth.ts`, target `src/integration/ooo-execution.ts` + and `src/integration/task-semantics.ts`). + +## Implementation state + +The rule layer and the board layer are in, and no caller other than the arms' driver passes a count +other than the default, so no gate or switch changed for the product path. + +- **Rules and status read**: `selectableTasks(plan, slots)`, `startableTasks(plan, slots)`, + `nextTask(plan, slots)`, `remainingSlots(plan, slots)` and `deriveStatus(units, facts, slots)` in + `src/integration/ooo-execution.ts` and `src/integration/task-semantics.ts`. +- **Admission**: `BoardAdmissionOptions.slots` (default 1, checked before the store is opened) and + `handoffTarget`; a count above 1 without a target is refused by name. `publishReady` publishes a + handoff for every startable task and keeps it across a republish, and the claim licence is + `startable()`. With the default the publication is the un-directed broadcast handoff it was. +- **Driver**: `evals/ooo-execution/plan-driver.ts` declares the spec's slot count to the admission + layer and names each unit's claimant with one function (`ownerOf`), so the offer and the claim + cannot disagree. Its batch loop needed no other change, and it still reports a count it did not + reach instead of reporting a time. +- **Evidence**: `evals/ooo-execution/board-slots.test.ts` (3 cases: two claims held at once and the + store's own `serialState`/`to` as the reason they can be, the default licence being the head, and a + claim's acceptance freeing a dependent while the other slot is held); `plan-driver.test.ts` replaces + its "the requested slot count is not reached" case with one that reaches it and measures the + overlap; `narrow-dispatch.test.ts` (6) and `tests/integration/task-semantics.test.ts` cover the rule. +- **Teeth**: 111 of 111 mutants caught by the named test, 17 of 17 targets restored. The new ones are + the budget and claim-window mutants in `src/integration/ooo-execution.ts`, the + licence/publication/target mutants in `src/integration/ooo-board.ts`, and the driver's declared + budget. + +The measured arm comparison is still owed: the spec pair for F2c and the paid pilot for F3 (a +separately fixed model, budget and repetitions). diff --git a/docs/decisions/implemented/2026-09-18-declared-slot-budget.zh-CN.md b/docs/decisions/implemented/2026-09-18-declared-slot-budget.zh-CN.md new file mode 100644 index 00000000..6d3e210d --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-declared-slot-budget.zh-CN.md @@ -0,0 +1,94 @@ +# 运行自声明槽预算,一次认领花掉一个槽 + +**Status:** implemented +**Approved:** explicit +Date: 2026-09-18 +Branch: feat/ooo-run-namespace +**Relates to:** [task-unit 语义设计](../../design/task-unit-semantics.md)、 +[义务台账](../../design/task-unit-semantics-obligations.md)、 +[实验臂自建驱动器](2026-09-17-arms-get-their-own-driver.md) + +治理 meta-rule:[self-governance meta-rule](2026-09-07-self-governance-meta-rule.md) —— +本规则变更本身也是一次受治理的决策(决策 + 替代方案 + 一个规则一个家)。 + +English: [2026-09-18-declared-slot-budget.md](2026-09-18-declared-slot-budget.md) + +## 问题 + +设计的 C 臂比较的是**同一份细计划在不同槽数下**的表现("C 同一细计划、多槽 | 仅改变执行槽数/合法顺序", +`docs/design/task-unit-semantics.md:371`),而设计的主问题里有一半是并发。做实验臂的驱动器时**测出这条臂没有 +机制**:在一个前三个单元互相独立的计划上请求 `slots: 4`,运行报告 `slotsUsed: 1`,其余按名字拒绝 +("no published handoff for this task")。测量记录见 +[台账](../../design/task-unit-semantics-obligations.md#what-f2b-measured-a-run-can-hold-exactly-one-claim)。 + +造成它的是三条规则,各自在自己的家: + +1. `selectableTasks`(`src/integration/ooo-execution.ts`)在**任何**未接受任务被认领时返回 `[]`。它的注释写的是 + 更窄的**邻居**规则("a task whose earlier neighbour is still claimed blocks selection"),代码并没有实现它。 +2. `publishReady`(`src/integration/ooo-board.ts`)只为被选中的任务发布 handoff,认领则拒绝任何不是 `next()`、 + 或没有已发布 handoff 的任务。 +3. 存储按 channel 串行化未定向的动作条目:只有没有其它打开着的未定向动作条目时,本条才是 `outstanding`,下一条是 + `pending`,而 `claimTaskBoardEntry` 会拒绝 `pending` 条目,直到 outstanding 那条被认领、解决或过期 + (`src/core/store/base.ts`)。这就是 D14 边界,它被特意选成"同一时刻只有一个动作摆在 Agent 面前"(台账 D14 行)。 + +## 决策 + +**运行自声明它能同时持有几个认领**,每次认领花掉一个。这个数是 `slots`,默认 `1` —— 也就是仓库原本就有的规则。 + +- `selectableTasks(plan, slots)` 是共享规则已经给出的合法集合,减去已被认领的任务;当 `claimed >= slots` 时为空。 + 排序步骤可以给这个集合排序,不可以放宽或收窄它。被认领计数统计的仍是上一条规则统计的那批 pending 任务。 +- `startableTasks(plan, slots)` 是上述有序集合裁到 `slots - claimed` 的那一段,也是认领许可**唯一**可以点名的部分。 + 裁剪发生在**排序之后**,所以"哪个合法任务排第一"仍由排序决定,预算不会把候选池交回规则自己的顺序。 +- `deriveStatus(units, facts, slots)` 用同一段裁剪作为 `ready`,因此状态查询与启动规则不会对"现在能启动什么"给出两种答案。 + +其余一律不放宽。被认领任务的依赖在认领在途期间仍未接受,所以依赖它的任务依旧不能提前启动(`valid`/`ready` 未改); +"至多一个任务可以在被许可的预测中等待外部事件"是另一条规则,不动;接受谓词、围栏与租约不动。认领在它的任务被接受 +之前都在途 —— 正是接受把任务移出预算所统计的 pending 集合。 + +## 考虑过的替代方案 + +C 臂需要 N 个认领共存,所以必须回答存储的按 channel 串行化。有两条路: + +- **(a1)允许每个 channel 有 N 条 outstanding 的未定向动作条目。** 这会改变产品任务板的推送语义 —— 也就是 D14 特意 + 选的悲观顺序,其目的是"同一时刻只有一个动作摆在 Agent 面前"。否决:代价是产品可见的,收益只是一个实验。 +- **(a2)把每个单元的 handoff 定向发给它自己的认领者。** 定向条目(`to != null`)按构造就不参与串行化 —— 存储那条 + 规则自己的注释就是这么写的:"Directed entries and notify-only kinds are not serialised (point-to-point, + parallel-safe)"。于是存储与 D14 完全不动,而每个槽的工作是通过点对点投递出去的,这本来就是它的样子:驱动器在板上的 + owner 是逐任务不同的。 + +(a2)是下一片要用的形状,所以本记录的代价只有一层规则。同样被否决的是 **(b)**:F3 只比较 A 与 B,并记录"并发在这个 +接缝上不可达" —— 设计的主问题有一半是并发,那等于在便宜机制存在的情况下把它留空。 + +## 后果 + +- 默认 `1` 逐字复现原先行为,因此所有既有断言成立:原来"有认领即阻断选择"变成 `claimed >= slots`,在单槽下与旧谓词 + 等价,而产品轮次根本不传槽数。规则的注释与代码现在陈述同一条规则。 +- 状态查询变得"按预算回答"。这是本次唯一触及的产品可见面,在默认值下它给出的答案与从前相同。 +- 更大的预算要求运行付出什么:必须已发布并定向好那些 handoff,且不得把退回单槽的运行读成多槽运行。实验臂的驱动器继续 + 分别报告"请求的槽数"与"达到的槽数",并在未达到请求值时拒绝给出时间结论。 +- 牙齿:新规则上四个具名突变体 —— 预算用尽不关闭选择、被认领任务仍在候选里、裁剪没从 startable 集合里切、零或半个槽 + 被当成预算 —— 每个都被点名的那条用例抓住(`tools/mutation-teeth.ts`,目标 + `src/integration/ooo-execution.ts` 与 `src/integration/task-semantics.ts`)。 + +## 实现状态 + +规则层与板子层都已落地;除了实验臂的驱动器,没有任何调用方传非默认值,所以产品路径没有改动任何开关或入口。 + +- **规则与状态读**:`src/integration/ooo-execution.ts` 与 `src/integration/task-semantics.ts` 中的 + `selectableTasks(plan, slots)`、`startableTasks(plan, slots)`、`nextTask(plan, slots)`、 + `remainingSlots(plan, slots)` 与 `deriveStatus(units, facts, slots)`。 +- **准入层**:`BoardAdmissionOptions.slots`(默认 1,在打开存储之前校验)与 `handoffTarget`;声明大于 1 + 却不给目标会被按名字拒绝。`publishReady` 为每个可启动任务发布 handoff,并在重发布时保留它,认领许可就是 + `startable()`。默认值下发布的仍是无定向的广播 handoff。 +- **驱动器**:`evals/ooo-execution/plan-driver.ts` 把 spec 的槽数声明给准入层,并用同一个函数(`ownerOf`) + 命名每个单元的认领者,因此"提供给谁"与"谁来认领"不可能不一致;它的批处理循环不需其它改动,且在未达到请求槽数时 + 依旧如实报告,而不是给出时间结论。 +- **证据**:`evals/ooo-execution/board-slots.test.ts`(3 条用例:两个认领同时持有、以及用存储自己的 + `serialState`/`to` 说明为何能做到;默认预算下许可仍是队首;一个认领被接受后在其另一槽仍被持有时释放依赖方); + `plan-driver.test.ts` 把"未达到请求槽数"那条换成达到它并测量重叠的用例;`narrow-dispatch.test.ts`(6 条)与 + `tests/integration/task-semantics.test.ts` 覆盖规则本身。 +- **牙齿**:111/111 突变体被点名的用例抓住,17/17 目标逐字节恢复。新增的是 + `src/integration/ooo-execution.ts` 的预算与可启动集合突变体、`src/integration/ooo-board.ts` 的 + 许可/发布/目标突变体,以及驱动器声明预算那条。 + +仍欠的是实测臂对比:F2c 的规格对与 F3 的付费试点(需另行固定模型、预算与重复次数)。 diff --git a/docs/decisions/implemented/2026-09-18-detached-long-checks.md b/docs/decisions/implemented/2026-09-18-detached-long-checks.md new file mode 100644 index 00000000..44abbd85 --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-detached-long-checks.md @@ -0,0 +1,144 @@ +# Long checks run detached, and their result is collected before the commit + +**Status:** implemented +**Approved:** explicit +**Relates to:** [repository development skill](../../../skills/repo-development/SKILL.md), +[CI and quality design](../../design/ci-cd-and-quality.md), +[the arms pilot](../../experiments/execution/ooo-arms-pilot-2026-09-18.md) + +治理 meta-rule:[self-governance meta-rule](2026-09-07-self-governance-meta-rule.md) —— +本规则变更本身也是一次受治理的决策(决策 + 替代方案 + 一个规则一个家)。 + +中文版: [2026-09-18-detached-long-checks.zh-CN.md](2026-09-18-detached-long-checks.zh-CN.md) + +## Problem + +The blocking checks an Agent runs here are not uniform in cost, and the verification +habit treated them as if they were. Measured on 2026-09-18 in one worktree (Windows): + +| Check | Cost | +| ------------------------------------------------------------------------------- | --------------- | +| `npm run mutation:teeth` (17 targets, 117 mutants) | ≈ 13 min | +| `npm run mutation:teeth -- --targets=` | ≈ 10 min | +| `npm run test:product` | ≈ 70 s | +| `evals/ooo-execution/*.test.ts` | ≈ 60 s | +| LSP diagnostics, format, lint on touched files, `complexity:gate`, `docs:check` | ≈ 20 s together | + +Two facts about that table are easy to miss. A scoped mutation run is not +proportionally cheap: the cost is the suites each mutant re-runs, so one target whose +suites include a slow acceptance case costs as much as several targets whose suites do +not. And the expensive three produce results that are needed at the commit, not at the +moment they are started: on that day the full mutation run was executed three times +inside one session, about 40 minutes of waiting that changed nothing. + +The harness's shell call is synchronous and gives no completion notice, so a check that +runs in the background is a convention an Agent follows, not a capability it can be +handed by a flag. + +## Decision + +1. **Two lanes.** The cheap checks that decide whether the change is sound stay + synchronous and blocking. The expensive three — full `mutation:teeth`, + `test:product`, and the `evals/**` suites — are launched detached once the code they + measure is final, and collected before the commit: + + ```bash + (npm run mutation:teeth > .temp/teeth.log 2>&1; echo "exit=$?" >> .temp/teeth.log) & + ``` + + A launch returns immediately and is never followed by a wait: no `sleep`, no poll + loop, no re-reading the log to see whether it is worth reading yet. Waiting in the + same session under another name is not a saving — the point of detaching is that the + time goes to the rest of the change (the documentation, the cheap lane, the next + edit), and the collector reads the log at the next natural checkpoint. When there is + nothing else to do, the check runs synchronously: a detached run nobody is working + alongside is just a slower way to wait. + +2. **A detached run reports its own exit status.** Without `echo "exit=$?"` into its + state file, a missing exit line means either "still running" or "died", and the two are + read the same way. +3. **Read the state as three outcomes, not two, and decide the middle one by liveness.** + Finished is `exit=`. Still running is "no exit line yet, and the recorded pid is + alive". Died is "no exit line, and the pid is gone" — nothing wrote a code, so the run + was killed rather than finished. Elapsed time is deliberately not a state: a slow, + healthy run and a fresh one are identical from the outside, so a long runtime is never + read as a failure, and distinguishing _working_ from _wedged_ uses the check's own + progress (one log line per test, one per mutant), not the clock. Measured on this + platform (Git Bash on Windows): a pid launched at 20:55:25 and still sleeping answered + `kill -0` as alive 22 seconds later from a _different_ shell invocation, a pid killed + with `kill -9` answered gone, and so did a pid that never existed — so the middle + outcome is decidable across tool calls, with no heartbeat needed. +4. **The launch is its own statement.** `A && B &` backgrounds the whole list, which is a + measured trap rather than a stylistic one: with the header redirect inside the + backgrounded list, the header truncated the state file _after_ the foreground `pid=` + line had been written to it, and the pid was lost — leaving exactly the state that + cannot be read. The header is written in the foreground, the check is backgrounded on + its own, and the pid is appended last. +5. **A detached run records the tree it measured** (`git rev-parse HEAD` and + `git diff --stat` at launch), and the collector compares that with the tree it is + collecting in: a code change means re-run, a documentation-only change leaves the + result valid. The comparison is what distinguishes the two, not the collector's + memory. +6. **Nothing edits a target while a mutation run rewrites it.** `mutation:teeth` + replaces the file it tests and does not restore after an abort, so a run that is + abandoned mid-flight can leave a mutant in the tree; the collector runs `git diff` + on the target first and treats a leftover mutant as a failed run rather than a + result. The same window makes staging unsafe: a `git add` while the run is in flight + can stage a mutant, which is worse than a failed check because the pre-commit hook + formats and the commit carries it. +7. **Scoped runs and the full run answer different questions.** `--targets=` during a + change says this change's own mutants are dead; a full run before a push or merge + says no other target's mutant survived. Neither substitutes for the other. + +## Alternatives considered + +- **Keep every check synchronous** — the practice before this record. Rejected on the + measurement above: 40 minutes of one session spent waiting, with no effect on the + change under test. +- **Guard the tool instead of documenting the habit**: make `mutation:teeth` refuse to + start when a target has uncommitted changes, or restore the target on abort. The + first half is wrong rather than deferred — a target with uncommitted changes is the + normal case, since mutation runs happen before the commit — and would refuse exactly + the run an Agent needs. Restore-on-abort is the half worth having and is recorded + under Deferred rather than taken here, because it changes a tool every Agent's runs + share while the hazard is detectable from `git diff` at collection. +- **A wrapper tool** (`tools/verify-detached.ts` with `start`/`status`): not taken. The + shell already supports the pattern — measured on this change: `test:product` (1433 tests) + was launched at 20:52:49, the shell call returned immediately, its log read `exit=0` at + 20:54:05, and the skill, both languages of this record, `docs:check`, `glossary:check` + and `agent:verify` were written and run inside that window — so a wrapper would be a + second home for one rule with no evidence yet that the three-line launch is repeated + enough to be worth typing wrong. The owner of the rule stays the Skill. +- **Give `evals/**` its own route so `agent:verify` stops dragging the product suite**: + an orthogonal cost (≈ 70 s) that belongs to the route contract in + `docs/design/ci-cd-and-quality.md`, not to this decision. + +## Consequences + +- The waiting moves off the critical path: the same checks run, while the Agent writes + the ledger, the design record, and runs the cheap lane. The saving is exactly the + other work available at that moment, so a change with nothing left to write gains + nothing from detaching and should run the check synchronously. +- A result is only as good as the tree snapshot it was collected against, so the + collector has to compare the tree rather than trust the log. +- An abandoned run is a working-tree hazard, not only a lost measurement (Decision 6), + which keeps the pre-commit formatting hook and a `git diff` review necessary even when + every log says `exit=0`. Measured 2026-09-18: a _running_ `mutation:teeth` holds its current + target as a live mutant, so `git status` showed `src/integration/ooo-board.ts` modified with + `digest` rewritten to `"%"` while the run was in progress; anything staged in that window is + the mutant rather than the work, which is why the staging follows the collected result. +- The rule lives in [`skills/repo-development/SKILL.md`](../../../skills/repo-development/SKILL.md); + the costs that justify the split live here. Each may change without rewriting the other. + +## Deferred + +- **Restore-on-abort for `mutation:teeth`.** The tool reports "restored byte-identically" + when it finishes a target, and nothing when it is killed. Worth adding if a leftover + mutant is ever collected as a result; until then the check is `git diff` on the target. +- **A wrapper for the launch/collect/compare triple.** Revisit when the same launch is + written often enough that getting it wrong is likelier than the wrapper is: at that + point the tree snapshot and the exit status are what it has to carry. +- **The Skill is close to its size budget.** `skills/repo-development/SKILL.md` is at + 14 931 B of the 15 000 B budget that `docs:check` enforces, and this rule had to be + compressed twice to fit. The next addition there should move an explanation out — to a + record or a design document — rather than trim another rule of the reason it exists. diff --git a/docs/decisions/implemented/2026-09-18-detached-long-checks.zh-CN.md b/docs/decisions/implemented/2026-09-18-detached-long-checks.zh-CN.md new file mode 100644 index 00000000..4828de7b --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-detached-long-checks.zh-CN.md @@ -0,0 +1,108 @@ +# 长检查异步跑,结果在提交前再收 + +**Status:** implemented +**Approved:** explicit +**Relates to:** [仓库开发 Skill](../../../skills/repo-development/SKILL.md)、 +[CI 与质量设计](../../design/ci-cd-and-quality.md)、 +[实验臂付费试点](../../experiments/execution/ooo-arms-pilot-2026-09-18.md) + +治理 meta-rule:[self-governance meta-rule](2026-09-07-self-governance-meta-rule.md) —— +本规则变更本身也是一次受治理的决策(决策 + 替代方案 + 一个规则一个家)。 + +English version: [2026-09-18-detached-long-checks.md](2026-09-18-detached-long-checks.md) + +## 问题 + +这里的阻塞性检查成本相差很大,而过去的验证习惯把它们当成一样贵。2026-09-18 在同一个 +worktree(Windows)实测: + +| 检查 | 耗时 | +| ----------------------------------------------------------------------------- | -------- | +| `npm run mutation:teeth`(17 目标、117 mutant) | ≈ 13 min | +| `npm run mutation:teeth -- --targets=<单个 driver 目标>` | ≈ 10 min | +| `npm run test:product` | ≈ 70 s | +| `evals/ooo-execution/*.test.ts` | ≈ 60 s | +| 触碰文件的 LSP 诊断、format、lint,加 `complexity:gate`、`docs:check`(合计) | ≈ 20 s | + +这张表有两点容易被忽略。定向 mutation 跑并不按比例变便宜:成本来自每个 mutant 要重跑的 +那套 suite,所以 suites 里含一个慢验收用例的目标,和一个 suites 很轻的目标不是一回事。 +另外这三件贵检查的结果是"提交时"才需要,不是"启动时"就需要:那天全量 mutation 在一次会话 +里跑了三遍,约 40 分钟等待,对被测改动没有任何影响。 + +harness 的 shell 调用是同步的、也不给完成通知,所以"后台跑检查"只能是 Agent 遵守的约定, +而不是某个开关能提供的能力。 + +## 决策 + +1. **两条车道。** 判断改动是否站得住的廉价检查保持同步、阻塞。三件贵的——全量 + `mutation:teeth`、`test:product`、`evals/**` 套件——在被测代码定稿后异步起,提交前收: + + ```bash + (npm run mutation:teeth > .temp/teeth.log 2>&1; echo "exit=$?" >> .temp/teeth.log) & + ``` + + 启动必须立即返回,后面**不能跟等待**:不 `sleep`、不轮询、也不"先看一眼日志值不值得看"。 + 同一个会话里换个名字等,等于没省——异步的意义是这段时间去做改动剩下的活(写文档、跑廉价 + 车道、写下一次编辑),收集放在下一个自然检查点。**如果手头没有别的活,就同步跑**:一个没人 + 在旁边干活的异步跑,只是等得更慢一种。 + +2. **异步跑自己写退出码。** 不写 `echo "exit=$?"`,空日志就同时表示"还在跑"和"已经死了", + 而这两者读起来是一样的。 +3. **状态有三种,不是两种;中间那种靠存活判断。** 完成 = 有 `exit=`;还在跑 = 还没有 exit + 行且记录的 pid 还活着(在**另一个** shell 里 `kill -0 ` 成功);死了 = 没有 exit 行且 pid + 已消失——没有人写退出码,说明它是被杀死的,而不是跑完的。**流逝的时间故意不算一种状态**:慢而 + 健康的跑和刚启动的跑从外面看一模一样,所以绝不会因为"跑得久"就判它失败;要区分"还在干活"和 + "卡死了",看检查自己的进度(日志最后一行,或日志的推进:`node --test` 每个用例一行, + `mutation:teeth` 每个 mutant 一行),而不是看钟。本机实测(Windows 上的 Git Bash):20:55:25 + 启动、还在 sleep 的 pid 在 22 秒后由**另一个** shell 调用 `kill -0` 答"活着";被 `kill -9` 的 pid + 答"没了",从未存在的 pid 也一样——所以中间那种状态跨工具调用可判,不需要心跳。 +4. **启动必须自成一条语句。** `A && B &` 会把整个列表放到后台,这不是风格问题而是实测的坑:把写 + 头信息的重定向放在被后台化的列表里,它会在前台那行 `pid=` 写进状态文件**之后**把文件截断,pid + 就丢了——恰好留下那种读不出状态的状态。头信息在前台写,检查单独后台起,pid 最后追加。 +5. **异步跑记录它所测的树**(启动时 `git rev-parse HEAD` 与 `git diff --stat`),收集时与当前的 + 树比对:代码变了就重跑,只有文档变了结果仍然有效。靠这个比对来区分,而不是靠收集者的记忆。 +6. **mutation 正在改写某个目标时,不许有人编辑它。** `mutation:teeth` 会替换被测文件,且中断不 + 还原,半途被放弃的跑可能在树里留下 mutant;收集时先对该目标 `git diff`,把残留 mutant 当作 + "这次跑失败了"而不是"一个结果"。同一窗口内**暂存也不安全**:跑还在飞的时候 `git add` 可能暂存 + 到一个 mutant——这比检查失败更糟,因为 pre-commit 钩子会格式化它,提交会把它带走。 +7. **定向跑与全量跑回答不同的问题。** 改动期间用 `--targets=` 说明"这次改动自己的 mutant 都死了"; + 推送/合并前跑一次全量说明"没有别的目标的 mutant 活下来"。两者不能互相替代。 + +## 考虑过的替代方案 + +- **所有检查继续同步跑**(本记录之前的做法)。按上面的测量否掉:一次会话 40 分钟纯等待,且 + 对被测改动没有影响。 +- **不改习惯,改工具加护栏**:让 `mutation:teeth` 在目标有未提交改动时拒绝启动,或在中断时还原。 + 前半是**错的**而不是"以后再做"——目标带未提交改动正是常规情形(mutation 本来就发生在提交之前), + 这个护栏会恰好拒绝 Agent 真正需要的那次跑。后半(中断还原)是值得要的那一半,放在"推迟"一节, + 因为它会改动所有 Agent 共用的工具,而该风险在收集时用 `git diff` 就能发现。 +- **写一个封装工具**(`tools/verify-detached.ts` 的 `start`/`status`):不采纳。shell 已经支持这个 + 模式——就在这次改动里实测过:`test:product`(1433 个用例)20:52:49 启动,shell 调用立刻返回, + 20:54:05 读到日志 `exit=0`,而这段时间里 Skill、本记录的中英文两版、`docs:check`、 + `glossary:check`、`agent:verify` 都写完并跑完了——所以封装只会成为同一条规则的第二个家, + 而目前还没有证据表明这三行启动脚本写错比封装更常发生。规则的归属仍是 Skill。 +- **给 `evals/**` 单独路由,让 `agent:verify` 不再拖产品套件**:这是另一笔成本(≈ 70 s),属于 + `docs/design/ci-cd-and-quality.md` 的路由契约,不属于本决策。 + +## 后果 + +- 等待离开关键路径:同样的检查照跑,但跑的同时 Agent 在写台账、写设计记录、跑廉价车道。省下的 + 时间就**等于那一刻手上还有的别的活**;所以没有别的活可写的改动从异步得不到好处,应当同步跑。 +- 一个结果的好坏取决于它所对的树快照,所以收集时必须比对树,而不是相信日志。 +- 被放弃的跑是**工作树**风险,不只是丢了一次测量(决策 6);因此即便每个日志都写 `exit=0`, + pre-commit 格式化钩子和 `git diff` 复查仍然必要。2026-09-18 实测:**正在跑**的 `mutation:teeth` + 会把当前目标以活变异形式留在树里——`git status` 显示 `src/integration/ooo-board.ts` 被改、 + `digest` 被换成 `"%"`;在那个窗口里 stage 到的就是 mutant 而不是成果,所以 staging 要在收集结果之后。 +- 规则的家是 [`skills/repo-development/SKILL.md`](../../../skills/repo-development/SKILL.md), + 支撑它的实测数据在本记录里;两者可以各自修改。 + +## 推迟 + +- **`mutation:teeth` 中断还原。** 工具正常跑完每个目标会报告 "restored byte-identically",被杀死时 + 什么都不报。若将来出现"残留 mutant 被当作结果收集"的情形就补上;在那之前检查手段是对目标的 + `git diff`。 +- **启动/收集/比对三件套的封装。** 等同一段启动脚本被写到"写错比封装更常见"的程度再说;那时它必须 + 携带树快照与退出码。 +- **Skill 已经贴近体积上限。** `skills/repo-development/SKILL.md` 目前 14 931 B,而 `docs:check` 强制 + 的上限是 15 000 B,这条规则为了塞进去压缩了两次。下次再往那里面加东西,应该把一段解释移出去 + (移进记录或设计文档),而不是把别的规则的理由剪掉。 diff --git a/docs/decisions/implemented/2026-09-18-fusion-and-speculation-pilot.md b/docs/decisions/implemented/2026-09-18-fusion-and-speculation-pilot.md new file mode 100644 index 00000000..de44790d --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-fusion-and-speculation-pilot.md @@ -0,0 +1,148 @@ +# Fusion and speculation: offline first, then a budgeted pilot + +[中文](2026-09-18-fusion-and-speculation-pilot.zh-CN.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [task-unit-semantics design](../../design/task-unit-semantics.md) (§融合, §严格语义怎样重新打开推测), +[its obligations ledger](../../design/task-unit-semantics-obligations.md), +[the arms get their own driver](../implemented/2026-09-17-arms-get-their-own-driver.md), +[the declared slot budget](../implemented/2026-09-18-declared-slot-budget.md), +[the arms pilot](../../experiments/execution/ooo-arms-pilot-2026-09-18.md), +[the cost model record](../../experiments/execution/ooo-cost-model-2026-09-17.md), +[the older speculation proposal](2026-09-11-ooo-speculation.md) + +> **Updated 2026-09-18: F4's offline half has landed, and the live half is blocked by a measured harness fact.** +> The rules, the accounting and the driver policy are in +> [fusion legality is a pair predicate](../implemented/2026-09-18-fusion-legality-and-accounting.md) +> (12 product cases / 8 mutants, 12 cost-model cases / 4 mutants, 15 driver cases / 12 mutants). +> The paid D arm cannot run as specified: `executePiPatch` creates a session per call, so a live worker +> cannot continue one, and its only expressible form is the design's named fallback (a new session seeded +> with the accepted prefix) - sequential handoff, not fusion. F5 (the speculation lifecycle) stays unrun +> because the design orders E after D's evidence, and F6's D half has no mechanism to spend on until an +> extension can hold a session across calls. No paid call has been made. + +Implementation evidence, and the reason the recorded blocker is gone: the live worker can hold one session, so the paid arms ran. D measured session startup at ~1.9 s and the cache-aware re-measurement found cap 2 to be the knee (`docs/experiments/execution/archive/ooo-arms-2026-09-19/`); E found no realised gain. No paid call remains blocked. + +## Problem + +The design's arm programme has five arms and only three have run. A, B and C ran in +[the pilot](../../experiments/execution/ooo-arms-pilot-2026-09-18.md) (n = 8, one model, one held-out family, +cost-directional only: A 8.6 s / 11.2 k tokens per run, B 21.5 s / 31.5 k, C 16.2 s / 29.9 k). **D +(execution fusion) and E (budgeted speculation) have never run**, and the design orders them in that +order: "E 在确定并发与融合得到证据后独立评估". + +What each arm needs, measured rather than assumed: + +- **D** is "the same Agent session runs several logical units in a row, each still taking its own + ticket, delivering its own artifact and crossing the host boundary" - execution fusion, explicitly + _not_ transaction fusion. The mechanism does not exist: `piWorker` in + `evals/ooo-execution/plan-driver.ts` calls `executePiPatch(frozen, provider, model)` once per unit, so + every unit gets a fresh session. The adapter that _could_ continue a session exists in the research + tree (`live-continuation.ts` ran a real continuation), and the cost model already carries the + per-boundary term (`contextMsPerUnit`, `coarseContextSaving`) - but nothing reuses a session across + units, and nothing reports a shared startup/context cost separately. +- **E** is "prepare a candidate ahead of one declared, finite-valued fact": `assumptions=[{predicateId, +version, expected}]`, the host re-checks every assumption against authoritative evidence inside the + publishing transaction, a false assumption discards the candidate and closes the branch session while + the real path re-executes under a new ticket. The first experiment is bounded by the design: **one** + pending fact, **one** candidate, no speculative successors, no irreversible external operation, and + cost/waits capped. The seam for `assumptions` exists (the advisers port refuses an unmodelled `fuse` + or `prepare` action), but no candidate is ever prepared ahead of a fact. + +Two facts also bound what a paid pilot could show. First, the cost model has **no quality term by +construction**, and the pilot found no quality difference to measure on the existing families (every arm +accepted everything). Second, the design requires D's context-reuse effect to be _measured_ ("仍要测量 +上下文偏置对质量的影响") - and that effect is precisely the one a cost-only measurement cannot see. So the +paid stage is only meaningful beside (a) the offline legality/accounting work and (b) a family whose +checks can actually fail on the fused or speculated path. + +## Decision + +Four slices, in this order, with the spend gated behind an explicit operator approval. + +**F4 (offline, no spend): fusion legality, ordering and accounting.** + +1. A pure function for legal fusion candidates and their order in the shared layer, honouring the + design's five conditions - compatible capability/authority/visibility; a successor only starts after + its dependency is _accepted_ (an unverified answer from the same Agent is not an accepted + dependency); per-unit identity, deadline, cancellation, verification and cost stay separate; a host + yield boundary between units; no reuse across a pending speculative branch. Each condition is pinned + by a named mutant, in the same style as the slot-budget rules. +2. Session reuse across fused units on the research side, with the host yield boundary and the per-unit + ticket/record flow unchanged. Fused units are one _session_, never one _acceptance_. +3. A fusion mode in the advisory cost model: the shared startup/context saving is reported as its own + line and is never counted twice per unit, and the model refuses a net-gain verdict when that term is + assumed rather than measured - mirroring how `coarseContextSaving` is already bracketed. + +**F5 (offline, no spend): the speculation lifecycle.** + +On the driver, with canned workers: one pending fact, one candidate carrying `assumptions`, the host +re-checking them inside the publish transaction, a false assumption discarding the candidate and the real +path re-executing under a new ticket, the wasted cost accounted as its own line, no speculative +successor, and a refusal to start when the assumption could not complete a _data_ input or when it +touches a permission precondition (the design's gate table). Offline proof of the lifecycle, not of the +payoff. + +**F6 (paid, needs approval): a small two-arm pilot.** + +One family, four arms, three repetitions each: `{unfused, fused}` for D and `{no speculation, +speculation on one fact}` for E, the same model F3 used (`deepseek/deepseek-v4-flash`, for comparability), envelope limits fixed per arm as in F3. Reported +separately per the design: latency, extra cost (including the wasted candidate), tokens, and quality +against the fixed checks. + +**Budget request.** From F3's measured per-run costs, twelve runs of this shape are ≈ 350-400 k tokens +and ≈ 4-6 minutes of model time (F3: 8 runs / 23 calls / 188 k tokens). **Approved ceiling: 1 000 k +tokens** (operator, 2026-09-18, with "尽量别都花完" - do not spend it all), against a planned spend of +about 400 k: the pilot stops as soon as a measurement is decisive, and adds no repetition for marginal +precision. Refusals declared in advance: if the family's checks cannot fail on the fused/speculated +path, the run reports cost and latency only and the family is replaced before any further spend; if the +fused arm cannot even reach its checks (a mechanism failure rather than a cost result), the pilot stops +and the result is "the mechanism is not ready". + +## Alternatives considered + +- **Go straight to the paid arms.** Rejected: the design orders an advisory offline model before any paid + call, and today there is no fusion mechanism, no candidate-ahead-of-a-fact lifecycle and no accounting + line to attribute the spend to. +- **Stay offline and never measure the payoff.** Rejected: the arms exist to weigh a cost against a + quality effect, and the design's own sentence asks for latency, extra cost _and_ quality together. +- **Run E before D.** Rejected by the design's own order: speculation's payoff depends on a fusion + decision that has not been measured. +- **Reuse the F3 families as they are.** Rejected as the _only_ family: every arm accepted everything, so + a fused/speculated path could not be shown to be right or wrong; a family whose checks can fail (F2c's + held-out `pipeline` with a wrong-answer case) is used, and its coarse/fine pairing keeps the fused arm + comparable. +- **Implement fusion as one acceptance for several units (transaction fusion).** Rejected by the design + for this slice: it changes what an acceptance is, which is the thing the arms are measuring. + +## Consequences + +With the mechanism in place the paid arms ran, so the budgeted pilot this record asked for is no longer blocked. D priced session startup and the cache-aware re-measurement moved the primary quantity to sessions avoided; E found no realised gain. + +1. F4's legality function refuses each of the five conditions when it is violated, with a named mutant + per condition, and the driver's fused run keeps per-unit tickets, verdicts and cost records. +2. The cost model's fused mode never books the same startup/context cost twice, and refuses a net-gain + verdict while its saving term is assumed (a test fails when the refusal is removed). +3. F5's lifecycle case: a false assumption discards exactly the candidate, accounts its cost separately, + and leaves the real path to re-execute under a new ticket; a candidate on a data input or a + permission precondition is refused by name. +4. F6 runs only after the operator's ceiling is recorded here (1 000 k tokens, approved 2026-09-18), + reports latency, extra cost and quality separately, and states which of the two refusals above fired + if the family or the mechanism was not ready. +5. The ledger gains the arms as rows (F4, F5, F6) with the evidence each one has, and the hidden-feature + registry gains the new research entry points in the same change. + +## Risks + +- **Context bias is the arm's real risk.** Reusing a session carries an unverifiable influence on the + next unit's answers; the design's fifth condition bounds it, and the pilot measures it only as quality + against fixed checks, not as a causal explanation. +- **A family that can fail is also a family that can fail for the wrong reason.** A wrong answer in the + held-out pipeline says the check works, not that fusion caused it; the pilot therefore compares + verdict patterns across arms rather than aggregate pass counts. +- **Speculation can look free in simulation and lose in reality.** The wasted candidate is cheap in + tokens but can occupy the host's check queue, which F1 already identified as the binding term; the + mock reports both, and the paid stage is where they separate. +- **Spend.** Twelve runs is a small sample; it will not separate two arms whose effect is smaller than + its variance, and the design's expectation of a _net_ gain is explicitly not guaranteed. diff --git a/docs/decisions/implemented/2026-09-18-fusion-and-speculation-pilot.zh-CN.md b/docs/decisions/implemented/2026-09-18-fusion-and-speculation-pilot.zh-CN.md new file mode 100644 index 00000000..9afa2461 --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-fusion-and-speculation-pilot.zh-CN.md @@ -0,0 +1,115 @@ +# 融合与推测:先离线,再有预算的试点 + +[English](2026-09-18-fusion-and-speculation-pilot.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [任务单元语义设计](../../design/task-unit-semantics.md)(§融合、§严格语义怎样重新打开推测)、 +[其义务台账](../../design/task-unit-semantics-obligations.md)、 +[实验臂自建驱动器](../implemented/2026-09-17-arms-get-their-own-driver.md)、 +[声明的槽位预算](../implemented/2026-09-18-declared-slot-budget.md)、 +[实验臂试点](../../experiments/execution/ooo-arms-pilot-2026-09-18.md)、 +[成本模型记录](../../experiments/execution/ooo-cost-model-2026-09-17.md)、 +[更早的推测提案](2026-09-11-ooo-speculation.md) + +> **2026-09-18 更新:F4 的离线部分已落地,live 部分被一项实测的 harness 事实阻断。** +> 规则、记账与驱动器策略见 +> [融合合法性是一个成对谓词](../implemented/2026-09-18-fusion-legality-and-accounting.md) +> (产品 12 用例 / 8 mutant,成本模型 12 用例 / 4 mutant,驱动器 15 用例 / 12 mutant)。 +> 付费的 D 臂无法按规格运行:`executePiPatch` 每次调用都新建会话,因此 live worker 无法接续会话, +> 唯一能表达的形式是设计点名的兜底(用已接受前缀作种子的新会话)——那是顺序交接,不是融合。F5 +> (推测生命周期)仍未跑,因为设计规定 E 晚于 D 的证据;在扩展能跨调用持有会话之前,F6 的 D 半边没有 +> 可花钱的机制。至今未发生任何付费调用。 + +实现证据,也是记录里那个阻塞消失的原因:活体 worker 现在能持有一个会话,所以付费臂跑起来了。D 臂测得会话启动约 1.9 秒,计入缓存后的复测发现 cap 2 就是拐点(`docs/experiments/execution/archive/ooo-arms-2026-09-19/`);E 臂未发现实际收益。已无付费调用被阻塞。 + +## 问题 + +设计的实验臂有五条,只跑过三条。A、B、C 在[试点](../../experiments/execution/ooo-arms-pilot-2026-09-18.md)里跑过 +(n = 8、单模型、单留出家族,只有成本方向性:每轮 A 8.6 s / 11.2 k tokens,B 21.5 s / 31.5 k,C 16.2 s / +29.9 k)。**D(执行融合)与 E(有预算推测)从未跑过**,而设计规定了顺序:"E 在确定并发与融合得到证据后独立 +评估"。 + +两条臂各需要什么,是量出来的而不是假设的: + +- **D** 是"同一 Agent 会话连续执行多个逻辑单元,每个单元仍独立拿票据、交产物并过宿主边界"——执行融合, + 明确**不是**事务融合。机制今天不存在:`evals/ooo-execution/plan-driver.ts` 的 `piWorker` 对每个单元调用 + 一次 `executePiPatch(frozen, provider, model)`,所以每个单元都是新会话。能延续会话的适配器在研究树里是 + 有的(`live-continuation.ts` 跑过一次真实接续),成本模型也已经带着每个边界的项(`contextMsPerUnit`、 + `coarseContextSaving`)——但没有任何代码跨单元复用会话,也没有把共享的启动/上下文成本单列。 +- **E** 是"对一个已声明、有限取值的事实提前准备候选":`assumptions=[{predicateId, version, expected}]`, + 宿主在发布事务内对每项假设核对权威证据,假设为假则丢弃候选、关闭分支会话,真实路径在新票据下重新执行。 + 首个实验按设计受限:**一个**未决事实、**一个**候选、不运行推测后继、不产生不可逆外部操作,资源/费用/等待 + 都有上限。`assumptions` 的接缝已经存在(advisers 端口拒绝未建模的 `fuse`/`prepare` 动作),但没有任何候选 + 会被提前准备。 + +两个事实还限定了付费试点**能**说明什么:成本模型按构造**没有质量项**,而试点在既有家族上测不出质量差异 +(每条臂都接受了一切);同时设计要求 D 的上下文复用效果**必须被测量**("仍要测量上下文偏置对质量的影响") +——而这恰恰是纯成本测量看不见的那一项。所以付费阶段只有在(a)离线的合法性/记账工作与(b)一个**其检查真的 +会失败**的家族同时在位时才有意义。 + +## 决策 + +四片,按此顺序;花钱那一步以操作者明确批准为门。 + +**F4(离线,不花钱):融合的合法性、排序与记账。** + +1. 在共享层实现"合法融合候选 + 排序"的纯函数,满足设计的五个条件——执行能力/授权主体/数据可见性兼容; + 后继只在其依赖**被接受**后启动(同一 Agent 刚生成的未验收答案不算依赖已满足);每个单元的身份、截止时间、 + 取消、验证与费用仍单独记录;单元间设宿主 yield 边界;不跨未决推测分支复用。每个条件由具名 mutant 钉住, + 风格与槽位预算规则一致。 +2. 研究侧实现跨单元的会话复用,宿主 yield 边界与逐单元票据/记录流保持不变。被融合的单元是**一个会话**, + 永远不是**一份接受结果**。 +3. 成本模型增加融合模式:共享的启动/上下文节省单列一行,且绝不按单元重复记账;当该项是假设而非实测时, + 模型拒绝给出净收益结论——与 `coarseContextSaving` 现在被标注为 assumed 的做法一致。 + +**F5(离线,不花钱):推测的生命周期。** + +在驱动器上用 canned worker:一个未决事实、一个带 `assumptions` 的候选、宿主在发布事务内重核、假即丢弃候选并让 +真实路径在新票据下重跑、白跑的代价单列、不运行推测后继,以及在假设无法补全**数据**输入、或触及**权限**前提时 +拒绝启动(设计的门表)。证明的是生命周期,不是收益。 + +**F6(付费,需批准):四臂小试点。** + +一个家族、四条臂、各三次重复:D 用 `{不融合, 融合}`,E 用 `{不推测, 一个事实上的推测}`;与 F3 同一模型(`deepseek/deepseek-v4-flash`,以保可比); +各臂的 envelope limits 与 F3 一致。按设计分开报告:延迟、额外费用(含白跑的候选)、tokens,以及相对固定检查的 +质量。 + +**预算请求。** 按 F3 实测的每轮成本,这种形状的十二轮约 **350-400 k tokens、约 4-6 分钟模型时间** +(F3:8 轮 / 23 次模型调用 / 188 k tokens)。**已批准上限:1 000 k tokens**(操作者,2026-09-18,附条件"尽量别都花完"),对应计划用量约 400 k:一旦某个测量已经能定论就停,不为边际精度加重复。预先声明的拒绝: +若家族的检查在融合/推测路径上无法失败,运行只报成本与延迟,并在任何进一步花钱前替换家族;若融合臂连检查都到不了 +(机制失败而非成本结果),试点停止,结论是"机制尚未就绪"。 + +## 考虑过的替代方案 + +- **直接跑付费臂。** 否决:设计规定任何付费调用前先有 advisory 离线模型,而今天既没有融合机制、也没有"候选先于 + 事实"的生命周期与可归属的记账项。 +- **只留在离线、永不测收益。** 否决:这两条臂存在的意义就是把成本与质量效果放在一起称,而设计自己的句子要求 + 延迟、额外费用与质量同时报告。 +- **先跑 E 再跑 D。** 设计自己的顺序否决:推测的收益取决于一个尚未被测量的融合决定。 +- **直接复用 F3 的家族。** 作为**唯一**家族否决:每条臂都接受一切,融合/推测路径正确与否都看不出来;改用 + F2c 的留出 `pipeline`(含错答案用例)家族,其粗/细配对让融合臂仍可比。 +- **把融合实现成"多个单元一份接受结果"(事务融合)。** 设计为本片否决:它会改变"接受"是什么,而接受正是这些臂 + 要测的东西。 + +## 后果 + +机制就位后付费臂得以跑完,记录要求的预算内试点不再被阻塞。D 臂给会话启动定了价,计入缓存后的复测把主要量改为“避免的会话数”;E 臂未发现实际收益。 + +1. F4 的合法性函数在五个条件各自被破坏时都拒绝,每个条件一个具名 mutant;驱动器的融合运行保持逐单元票据、 + 裁决与费用记录。 +2. 成本模型的融合模式不重复记同一笔启动/上下文成本,并在节省项仍是假设时拒绝给出净收益(去掉该拒绝时测试失败)。 +3. F5 的生命周期用例:假设为假时**只**丢弃该候选、单独记其代价,并让真实路径在新票据下重跑;落在数据输入或权限 + 前提上的候选按名字拒绝。 +4. F6 只在操作者上限被记录在本记录之后运行(1 000 k tokens,2026-09-18 批准);分开报告延迟、额外费用与质量,并写明上面两条拒绝中哪一条触发过。 +5. 台账新增这些臂的行(F4、F5、F6)及各自的证据,隐藏特性注册表在同一改动里新增研究入口。 + +## 风险 + +- **上下文偏置是这条臂真正的风险。** 复用会话会对下一单元的答案带来无法验证的影响;设计的第五条件界定了它, + 而试点只能把"质量"测成相对固定检查的通过情况,不能给出因果解释。 +- **会失败的家族也会为错误的原因失败。** 留出 pipeline 里的一个错答案说明检查有效,不说明是融合造成的; + 因此试点比较的是各臂的裁决模式,而不是聚合的通过数。 +- **推测在模拟里可能看起来免费,在现实里不是。** 白跑候选在 tokens 上便宜,但会占用宿主的检查队列——F1 已经 + 指出那才是约束项;模拟两侧都报,付费阶段才把它们分开。 +- **花费。** 十二轮是小样本;若两条臂的效应小于其方差,它分不出来,而设计对**净**收益的期望明确不作保证。 diff --git a/docs/decisions/implemented/2026-09-18-fusion-legality-and-accounting.md b/docs/decisions/implemented/2026-09-18-fusion-legality-and-accounting.md new file mode 100644 index 00000000..0a5894bb --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-fusion-legality-and-accounting.md @@ -0,0 +1,90 @@ +# Fusion legality is a pair predicate, and fusion's cost is two lines + +[中文](2026-09-18-fusion-legality-and-accounting.zh-CN.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [task-unit-semantics design](../../design/task-unit-semantics.md) (§融合), +[its obligations ledger](../../design/task-unit-semantics-obligations.md) (row F4), +[the fusion and speculation pilot proposal](../implemented/2026-09-18-fusion-and-speculation-pilot.md), +[the cost model record](../../experiments/execution/ooo-cost-model-2026-09-17.md), +[the arms pilot](../../experiments/execution/ooo-arms-pilot-2026-09-18.md) + +## Problem + +The design defines execution fusion - one Agent session running several logical units, each keeping its +own ticket, artifact and host boundary - and puts five conditions on a pair of units that may share one. +Nothing could answer that question: `selectableTasks` is a "now" predicate over one plan, fusion is a +"next" relation between two units, and the driver had no session concept at all (`piWorker` called +`executePiPatch` once per unit, so every unit got a fresh session). The cost model charged a session +boundary per unit and had no way to say what a _shared_ startup costs, so the design's trade - "saved +startup/context cost − added wait/verification/context burden" - could not be priced either. The pilot +added a third gap: the design's fifth condition needs a branch that is still pending, and no code knows +one. + +## Decision + +**1. Legality is one pair predicate in the shared dispatch rules, composed with the board's own +answer.** `sharedSessionLegal(before, after, plan)` in `src/integration/ooo-execution.ts` holds the +design's five conditions, one line each, so a violation has one name; `fusionSuccessors` and +`fusionCandidates` expose the legal set in plan order. The predicate takes both views it needs - the +unit's declaration (capability, authority, its own visibility) and the runtime facts - and it composes +with `candidates()`/`accepted()` rather than re-deriving them, because the board stays the authority on +staleness, cancellation, delivery, claims and a declared external wait. The **bound** ("short ready +chains, not a greedy swallow of the DAG") and the **ranking** are the runtime's policy: the shared layer +owns legality and nothing else. + +**2. Fusion's cost is two lines and a verdict that cannot come from a guess.** +`fusionAccounting` reports `boundarySavedMs` (what the removed boundaries are worth, on the term a pilot +has measured) and `sharedStartupMs` (booked once per session, never amortised into a unit) as separate +values, and `fusionVerdict` returns `unmeasured` until a run has priced the session startup - so the +design's "共享启动和上下文成本另列,不能重复记账" is a property of the accounting rather than a note in a +report. `assertModelProperties` refuses the two ways this can be silently wrong: booking the startup per +unit, and a bound that removes no boundary reporting a saving. The synthesis takes `sessionStartMs`, +`sessionStartMeasured` and `unitsPerSession` as declared parameters, and `--sweep` varies the startup on +both sides of its turning point instead of trusting one value. + +**3. The driver reports fusion only from the worker's own session report, and a worker that cannot hold a +session refuses by name.** `PlanDriverSpec.fusion` declares the bound and the per-unit declarations; +`runPlan` continues a session only from an **accepted** unit, ends it at a rejected verdict, a failed +worker, a successor that is not legal at the boundary or the declared bound, and records one entry per +session in `PlanRun.sessions`. Fusion's evidence is `WorkerMetrics.sessionId` - the session the worker +says it used - so a worker that quietly starts its own session is reported as boundaries, not as fusion. +`piWorker` **refuses** a continuation it cannot honour, naming the session, because `executePiPatch` +creates a session per call; a fused live run's mechanism is therefore not landed, and a run says so +instead of reporting a fresh session as reuse. + +## Alternatives considered + +- **A chain builder in the shared layer.** Rejected: a bound is a runtime policy, and the design's own + division puts only legality in the shared module. `fusionSuccessors` returns the legal set; the driver + applies the bound. +- **Recording fusion from the driver's request rather than the worker's report.** Rejected by + measurement: the new case "a worker that starts its own session is not reported as fusion" fails under + the mutant that counts the requested session, which is exactly the silent lie this avoids. +- **One net number per fused run.** Rejected: the design asks for the two lines precisely because a net + number hides which term answered, and this file's convention is that an assumed term is marked and + varied, never read as a measurement. +- **Having the live worker answer a continuation with a new session seeded by the accepted prefix.** + Rejected as _fusion_: that is the fallback the design names ("若 harness 没有这种能力,就新建会话"), and + reporting it as session reuse would turn the D arm's cost claim into a comparison of two identical + mechanisms. It stays available as what a live fused arm would actually run, and the ledger row records + it as not landed rather than as a result. +- **Putting the legality function beside the compiler (`task-semantics.ts`) instead of the dispatch + rules.** Rejected: the question it answers is "may the host hand this unit out inside this session", + which is the same family as `selectableTasks`, and its inputs are that module's runtime facts. + +## Consequences + +- The offline half of F4 is proven: 12 product cases with 8 named mutants for legality, 12 cost-model + cases with 4 named mutants for the accounting, and 15 driver cases with 12 named mutants for the + policy - each mutant caught by the case that names the rule it breaks. +- **The paid F6 arm's D half cannot run as specified in this harness yet.** A reusable session needs the + Pi extension to hold one session across calls; until then the only expressible form is the design's + named fallback (a new session seeded with the accepted prefix), which is sequential handoff, not + fusion. This is a measured fact about the harness, and it is the reason the pilot's D arm is not run. +- `src/integration/ooo-execution.ts` gains a rule with no product caller today: the arms are its + consumers, and the design assigns legal fusion candidates to the shared layer. If the D arm is never + promoted to the product, this rule retires with it - the same disposition the round instrument got. +- The refusal text in `piWorker` is the place a future extension change must delete: when the extension + can hold a session, that branch becomes the continuation it describes. diff --git a/docs/decisions/implemented/2026-09-18-fusion-legality-and-accounting.zh-CN.md b/docs/decisions/implemented/2026-09-18-fusion-legality-and-accounting.zh-CN.md new file mode 100644 index 00000000..dcfca659 --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-fusion-legality-and-accounting.zh-CN.md @@ -0,0 +1,72 @@ +# 融合合法性是一个成对谓词,融合的成本是两条线 + +[English](2026-09-18-fusion-legality-and-accounting.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [任务单元语义设计](../../design/task-unit-semantics.md)(§融合)、 +[其义务台账](../../design/task-unit-semantics-obligations.md)(行 F4)、 +[融合与推测试点提案](../implemented/2026-09-18-fusion-and-speculation-pilot.md)、 +[成本模型记录](../../experiments/execution/ooo-cost-model-2026-09-17.md)、 +[实验臂试点](../../experiments/execution/ooo-arms-pilot-2026-09-18.md) + +## 问题 + +设计定义了执行融合——同一 Agent 会话连续执行多个逻辑单元,每个单元仍保留自己的票据、产物与宿主边界 +——并给"可以共用会话的一对单元"加了五个条件。之前没有任何东西能回答这个问题:`selectableTasks` 是 +对单个计划的"现在"谓词,融合是两单元之间的"下一步"关系;而驱动器根本没有会话概念(`piWorker` 对每个 +单元调一次 `executePiPatch`,于是每个单元都拿到新会话)。成本模型按单元收取会话边界成本,也没有任何 +方式表达"共享启动"值多少,所以设计的那个权衡——"节省的启动/上下文成本 − 新增等待/验证/上下文负担" +——同样无法定价。试点还暴露出第三个缺口:设计的第五个条件需要一个"仍未决的分支",而没有任何代码知道 +它是什么。 + +## 决策 + +**1. 合法性是共享调度规则里的一个成对谓词,并与黑板自己的答案复合。** +`src/integration/ooo-execution.ts` 的 `sharedSessionLegal(before, after, plan)` 承载设计的五个条件, +每条一行,因此每个违规都有唯一的名字;`fusionSuccessors` 与 `fusionCandidates` 按计划顺序给出合法集合。 +该谓词接收它需要的两个视图——单元声明(执行能力、授权主体、它自己的可见性)与运行时事实——并与 +`candidates()`/`accepted()` **复合**而不是重新推导,因为陈旧、取消、交付、认领与声明的外部等待,权威 +始终在黑板。**上界**("就绪短链,不贪心吞并整个 DAG")与**排序**是运行时的策略:共享层只拥有合法性。 + +**2. 融合的成本是两条线,且结论不能来自猜测。** `fusionAccounting` 把 `boundarySavedMs`(被去掉的边界 +按实测项值多少)与 `sharedStartupMs`(每个会话只记一次,绝不摊到单元上)作为两个值分别报告; +`fusionVerdict` 在没有任何一轮为会话启动定价之前一律返回 `unmeasured`——于是设计那句"共享启动和上下文 +成本另列,不能重复记账"是记账本身的性质,而不是报告里的一句说明。`assertModelProperties` 拒绝这两种 +可能悄无声息出错的算法:把启动按单元记账,以及一个"没去掉任何边界"的上界报出节省。合成时 +`sessionStartMs`、`sessionStartMeasured`、`unitsPerSession` 都是被声明的参数,`--sweep` 在转折点两侧 +分别扫启动项,而不是只信一个取值。 + +**3. 驱动器只按 worker 自己的会话报告记录融合;无法持有会话的 worker 按名字拒绝。** +`PlanDriverSpec.fusion` 声明上界与逐单元声明;`runPlan` 只从**被接受**的单元继续会话,并在被拒裁决、 +worker 失败、边界处后继不合法或达到声明上界时结束会话,并在 `PlanRun.sessions` 里每个会话记一条。 +融合的证据是 `WorkerMetrics.sessionId`——worker 自己说它用了哪个会话;因此一个悄悄新建会话的 worker +会被记录成边界,而不是融合。`piWorker` 对它无法兑现的接续**拒绝**并报出会话名,因为 `executePiPatch` +每次调用都新建会话;也就是说融合的 live 机制尚未落地,而运行会如实这么说,而不是把新会话报成复用。 + +## 考虑过的替代方案 + +- **在共享层里做一个链构造器。** 否决:上界是运行时策略,而设计自己的分工只把合法性放进共享模块。 + `fusionSuccessors` 返回合法集合,上界由驱动器施加。 +- **按驱动器的"请求"而不是 worker 的"报告"记录融合。** 由测量否决:新用例"worker 自建会话时不算融合" + 在"按请求的会话计数"这个 mutant 下失败,而这正是要避免的静默谎言。 +- **每个融合运行只给一个净数。** 否决:设计要两条线正是因为净数会掩盖答案来自哪一项;而本文件的约定是 + 假设项必须被标注并被扫,绝不当作实测来读。 +- **让 live worker 用"以已接受前缀为种子的新会话"来回答接续。** 作为**融合**否决:那正是设计点名的兜底 + ("若 harness 没有这种能力,就新建会话");把它报成会话复用,会把 D 臂的成本论断变成两种相同机制的 + 比较。它仍然可以保留为"live 融合臂实际会跑的东西",而台账把这条记成"未落地"而不是一个结果。 +- **把合法性函数放在编译器旁边(`task-semantics.ts`)而不是调度规则里。** 否决:它回答的问题是"宿主 + 是否可以在该会话里把这个单元派出去",这与 `selectableTasks` 属于同一族,而它的输入正是那个模块的 + 运行时事实。 + +## 后果 + +- F4 的离线部分已有证据:合法性 12 个产品用例 + 8 个具名 mutant,记账 12 个成本模型用例 + 4 个具名 + mutant,策略 15 个驱动器用例 + 12 个具名 mutant——每个 mutant 都被"点名它破坏的那条规则"的用例抓住。 +- **付费 F6 的 D 臂在当前 harness 下无法按规格运行。** 可复用会话需要 Pi 扩展跨调用持有一个会话;在那 + 之前唯一能表达的形式是设计点名的兜底(用已接受前缀作种子的新会话),那是顺序交接,不是融合。这是关于 + harness 的实测事实,也正是试点 D 臂没有跑的原因。 +- `src/integration/ooo-execution.ts` 新增了一条目前没有产品调用方的规则:实验臂是它的消费者,而设计把 + "合法融合候选"指派给共享层。若 D 臂最终不进入产品,这条规则随它一起退休——与轮次仪器相同的处置。 +- `piWorker` 里那段拒绝文字,就是未来扩展改动必须删掉的位置:当扩展能持有会话时,那个分支就变成它所 + 描述的那个接续。 diff --git a/docs/decisions/implemented/2026-09-18-retire-the-round-instrument.md b/docs/decisions/implemented/2026-09-18-retire-the-round-instrument.md new file mode 100644 index 00000000..fb6163c9 --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-retire-the-round-instrument.md @@ -0,0 +1,139 @@ +# Retire the round instrument: distribute what has a home, delete the rest + +[中文](2026-09-18-retire-the-round-instrument.zh-CN.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [task-unit-semantics design](../../design/task-unit-semantics.md), +[its obligations ledger](../../design/task-unit-semantics-obligations.md), +[the arms get their own driver](../implemented/2026-09-17-arms-get-their-own-driver.md), +[the declared slot budget](../implemented/2026-09-18-declared-slot-budget.md) + +Governing meta-rule: [self-governance meta-rule](../implemented/2026-09-07-self-governance-meta-rule.md). +This record replaces the disposition draft committed as `7821c5ed` ("keep / merge / cut"), which asked +whether to split the round's role layer from its mechanism; measurement answered that question and the +operator then chose distribution plus deletion over relocating the module. + +## Problem + +`src/integration/ooo-cycle.ts` (1 082 lines, 13 exports) is the S4 out-of-order instrument. The +[2026-09-17 decision](2026-09-17-arms-get-their-own-driver.md) deferred its disposition on purpose, and +the ground has moved since: the arms have their own driver, the daemon has a run surface (D13), the +coordinator owns coordinated writes and the run-entry binding (D11/D12), and the ordinary collaboration +path no longer needs a round (G1). + +Measured over `src/`, `evals/`, `tests/`, `tools/` (556 files; the one-off script and its JSON are in +untracked `.temp/`, because they are not a maintained entry point): + +- **70** references to the round's fixed A/B/C names, over **17** of its 44 internal symbols; +- only **4** calls into the store/board port inside the module, so the "role layer versus mechanism + layer" framing had almost no mechanism to split — that alternative is rejected by measurement; +- `runCycle` and `openRoundStore` have **9 callers each, all under `evals/ooo-execution/`**; no `tests/` + caller; the only `src/` callers are type imports; +- it is **not** a registered mutation target, although its siblings are (17 targets); +- its candidate-verification orchestration is implemented a **second time** in + `evals/ooo-execution/plan-driver.ts` over the same imports. + +The decisive question was whether the round carries a capability the driver lacks. It does not, for the +property the design is about. A one-off probe (`.temp/ooo-interleave-probe.ts`) ran a two-unit plan at +two slots where unit A's check sleeps ~3 s: B's worker window **1505 ms sits entirely inside A's check +window 3923 ms**, with `slotsUsed=2` and order `A-start, A-end, B-start, B-end`. "A's check is +outstanding while B works" is therefore expressible in the driver's own vocabulary of plan, slots and +per-unit checks — no roles required. + +## Decision + +Distribute the pieces that have a home, then delete the module and everything that only served it. + +**Moved to its concept owner (the only two `src/` consumers the file had):** + +- `Requirement` → `src/integration/task-semantics.ts`, which already defines preconditions over a + dependency's accepted artifact and is the file that imported the type; +- `WorkerMetrics` → `evals/ooo-execution/plan-driver.ts`, its sole surviving consumer once the round and + `ooo-round-log.ts` are gone. The product's run surface records no worker metrics today (measured: no + `tokens`/`metrics`/`cost` in `src/integration/ooo-execution.ts`), so keeping a product home for it + would be unused product surface. + +**Evidence re-homed, because the implementer is product and only the harness was the round:** + +- D9 (a post-commit notification failure is recorded, not thrown) is implemented by + `src/integration/ooo-board.ts` (`lastNotificationFailure()`), not by the round. The evals suite said so + itself: it drove a real round only because the artifact envelope is the board's business. The property + is pinned on the board instead, in `tests/` where a product obligation belongs, and the hand mutation + (drop the guard around the post-commit notification) must still fail it. +- The S4 interleaving gets its own case in `evals/ooo-execution/plan-driver.test.ts`: with two slots, one + unit's worker runs while another unit's check is outstanding. This is the first end-to-end pin of that + property on the mechanism the arms actually use; it fails if the driver's batch loop stops overlapping. + +**Deleted (no home, nothing in `src/` calls it):** + +- `src/integration/ooo-cycle.ts` and `src/integration/ooo-round-log.ts`; +- in `evals/ooo-execution/`: `round-runner.ts`, `round-compare.ts`, `round-spec.ts`, `live-cycle.ts`, + `probe-check-duration.ts`, and the suites `cycle.test.ts`, `replay.test.ts`, `notification.test.ts`, + `round-plan.test.ts`, `round-cli.test.ts`, `cancellation.test.ts`, `early-cancel.test.ts`; +- `DEFAULT_ROUND_PLAN` and the types with no surviving consumer (`CaseRule`, `CycleWorker`, + `CheckRunner`, `CycleOptions`, `CycleResult`, `OooRoundOperations`, `WorkerResult`, + `CheckOutcomeSummary`) — the 2026-09-17 record's five shared types reduce to the two that moved. + +**Kept, and checked to be independent of the round:** `plan-driver.ts` and its suite, `round-client.ts` +and `round-host.ts` (D14's daemon evidence, which serve a store, not a round), `board-*.ts`, +`board-slots.test.ts`, `narrow-dispatch.test.ts`, `patch-cycle.test.ts`, `cost-model*`, +`families.test.ts`, `live-patch.ts`, `live-continuation.ts` (G7's executable check — measured to import +nothing from the round), `pilot.ts`, `rename-probe.ts`, `report-family.test.ts`. + +**Ledger, in the same change:** D10 is rewritten as "no counterpart can fail" (the row's own wording was +about `runCycle`'s early-cancel branch, and with no round nothing borrows a store to close — the B7 +precedent); D9's evidence names the board test; G4's cancel half points at the product test that already +covers it (`tests/integration/ooo-round-query.test.ts`, "the terminal decision outlives the host that +made it"); the ledger's stale closing paragraph is corrected. Nothing else cites the retired suites: of +the ~84 cases, only those three were obligation-bearing. + +## Alternatives considered + +- **Keep the module where it is.** Rejected: no product caller, no route, no mutant, no `tests/` caller, + a second implementation of the driver's orchestration, and an instrument's subject in `src/`. +- **Split the role layer from the mechanism layer.** Rejected by measurement: 4 store call sites against + 70 role references, and the interleaving the roles existed for is already carried by the driver. +- **Relocate the module and its suites to `evals/` (the draft's "cut").** Rejected by the operator: it + would create a 1 082-line module that nothing owns, and it preserves machinery whose only consumer was + the round. +- **Keep `round.jsonl` replay and compare as research method.** Rejected by the operator: `plan-driver` + replaces `round-runner`/`round-compare`, and the design already classes `round.jsonl` as an export ("可 + 导出作研究重放", a file that decides no state), so replay served the instrument, not the product. +- **Delete outright without re-homing.** Rejected: D9's evidence and the interleaving property would go + with it, and both can be pinned cheaply on the mechanisms that remain. + +## Consequences + +- `src/` no longer contains an experiment file; the round's 1 082 lines leave the product tree and the + driver is the one research-side runner. +- The retry machinery the round carried — pushback, reopened dependencies, unmet preconditions, + `handlePushback`/`reopenDependency`/`handleUnmetPrecondition` — is gone. No ledger row pinned it, and + no product path used it, but the design's E arm (budgeted speculation) may need "discard and redo"; if + it does, that is the E arm's own requirement, implemented on the driver, not a reason to keep this file. +- The ability to replay a real run from `round.jsonl` goes with it. Archived runs + (`docs/experiments/execution/*`) remain as evidence of the experiments that happened; re-running them + would mean rebuilding an export path for the driver. +- The `evals/` suite count drops by the twelve retired files, and the surviving driver suite carries the + property the round was the only end-to-end carrier of. + +## What the change was verified against + +1. `rg "ooo-cycle" src evals tests tools` returns nothing (the name survives only in experiment records + and decisions, where it describes history). +2. The board's post-commit notification test fails if the guard is removed, and D9's evidence names it. +3. The driver's interleaving case fails if the batch loop stops overlapping units. +4. `DEFAULT_ROUND_PLAN` and the eight unreferenced types have no remaining reference anywhere. +5. `npm run check`, `npm run docs:check`, `npm run test:product`, the surviving `evals/ooo-execution/` + suites, `npm run mutation:teeth` and `npm run agent:verify` over the touched paths all pass. + +## Risks + +- **The instrument's own contract net is gone** (about 62 cases). They pinned the round's behaviour, not + a design obligation, and `tests/` never covered the module; the replacement is the driver's own suite + plus the two re-homed pins. If a future arm needs the round's semantics, it is rebuilt deliberately. +- **Two kinds of history now lack a live counterpart**: `early-cancel`'s branch and the replay format. + Both are recorded as retired here rather than left as a stale row pointing at deleted files. +- **A separate pre-existing gap surfaced while measuring**: no route in `agent-context.yaml` matches + `src/integration/**`, `ooo-board.ts` and `task-coordinator.ts` included. That is its own decision; it + is named here only so this record's measurements are not read as "the round was special". diff --git a/docs/decisions/implemented/2026-09-18-retire-the-round-instrument.zh-CN.md b/docs/decisions/implemented/2026-09-18-retire-the-round-instrument.zh-CN.md new file mode 100644 index 00000000..00116fc6 --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-retire-the-round-instrument.zh-CN.md @@ -0,0 +1,125 @@ +# 退役轮次仪器:有家的分发到各家,其余删掉 + +[English](2026-09-18-retire-the-round-instrument.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [任务单元语义设计](../../design/task-unit-semantics.md)、 +[其义务台账](../../design/task-unit-semantics-obligations.md)、 +[实验臂自建驱动器](../implemented/2026-09-17-arms-get-their-own-driver.md)、 +[声明的槽位预算](../implemented/2026-09-18-declared-slot-budget.md) + +治理 meta-rule:[self-governance meta-rule](../implemented/2026-09-07-self-governance-meta-rule.md)。 +本记录取代以 `7821c5ed` 提交的处置草案("留/合/裁")——那份草案问的是"要不要把轮次的角色层与机制层拆开"。 +实测回答了这个问题,随后操作者选择**分发 + 删除**,而不是把模块整体挪走。 + +## 问题 + +`src/integration/ooo-cycle.ts`(1 082 行、13 个导出)是 S4 越序仪器。 +[2026-09-17 的决策](2026-09-17-arms-get-their-own-driver.md) 故意推迟了它的去向,而地基此后动了:实验臂有了 +自己的驱动器,daemon 有了运行面(D13),协调器拥有受协调写入与运行-entry 绑定(D11/D12),普通协作路径 +不再需要轮次(G1)。 + +在 `src/`、`evals/`、`tests/`、`tools/` 上实测(556 个文件;一次性脚本与其 JSON 在未跟踪的 `.temp/`, +因为它们不是维护中的入口): + +- 对轮次固定 A/B/C 名字的引用 **70 处**,分布在 44 个内部符号里的 **17** 个; +- 模块内对存储/看板端口的调用只有 **4 处**,因此"角色层 vs 机制层"这个框子几乎没有机制可拆——该替代 + 方案**依据实测被否**; +- `runCycle` 与 `openRoundStore` **各 9 个调用者,全部在 `evals/ooo-execution/` 下**;`tests/` 无调用者; + `src/` 下的唯一调用者是类型导入; +- 它**不是**已登记的变异目标,而它的兄弟模块都在 17 个目标里; +- 它的候选验证编排在 `evals/ooo-execution/plan-driver.ts` 里基于同一组导入**实现了第二遍**。 + +决定性的问题是:轮次是否承载着驱动器缺少的能力。对设计所关心的那条性质而言,答案是"没有"。一次性探针 +(`.temp/ooo-interleave-probe.ts`)跑了一个双单元计划、两个槽位,其中单元 A 的检查 sleep 约 3 秒:**B 的 +worker 窗口 1505 ms 完整地落在 A 的检查窗口 3923 ms 之内**,`slotsUsed=2`,顺序为 +`A-start, A-end, B-start, B-end`。因此"A 的检查在飞时 B 在工作"用驱动器的 `plan + 槽 + 每单元检查` +词汇就能表达——**不需要角色**。 + +## 决策 + +把有家的部分分发到各自的家,然后删除该模块及只服务于它的一切。 + +**搬到各自的概念所有者(它原本在 `src/` 里仅有的两个消费者):** + +- `Requirement` → `src/integration/task-semantics.ts`:那里本来就定义了"对依赖产物被接受"的前提, + 而且就是导入该类型的文件; +- `WorkerMetrics` → `evals/ooo-execution/plan-driver.ts`:一旦轮次与 `ooo-round-log.ts` 消失,它是唯一 + 剩下的消费者。产品的运行面今天不记录 worker 费用(实测:`src/integration/ooo-execution.ts` 里没有 + `tokens`/`metrics`/`cost`),因此为它保留一个产品之家只会成为无人使用的产品表面。 + +**证据改钉,因为实现者本是产品、只有夹具是轮次:** + +- D9(提交后的通知失败被记录、不被抛出)的实现在 `src/integration/ooo-board.ts` + (`lastNotificationFailure()`),不在轮次里——那个 evals 套件自己也这么写明:它用真轮次只是因为 + artifact 信封是 board 的事。该性质改钉在 board 上,落在 `tests/`(产品义务该在的地方),并且手动变异 + (去掉提交后通知外围的守卫)必须仍然让它失败。 +- S4 的越序性质在 `evals/ooo-execution/plan-driver.test.ts` 里获得自己的用例:两个槽位下,一个单元的 + worker 在另一个单元的检查尚未完成时运行。这是该性质**第一次**钉在实验臂真正使用的机制上;当驱动器的批 + 循环不再重叠时它会失败。 + +**删除(无家,`src/` 里没有调用者):** + +- `src/integration/ooo-cycle.ts` 与 `src/integration/ooo-round-log.ts`; +- `evals/ooo-execution/` 下的:`round-runner.ts`、`round-compare.ts`、`round-spec.ts`、`live-cycle.ts`、 + `probe-check-duration.ts`,以及套件 `cycle.test.ts`、`replay.test.ts`、`notification.test.ts`、 + `round-plan.test.ts`、`round-cli.test.ts`、`cancellation.test.ts`、`early-cancel.test.ts`; +- `DEFAULT_ROUND_PLAN` 与无剩余消费者的类型(`CaseRule`、`CycleWorker`、`CheckRunner`、`CycleOptions`、 + `CycleResult`、`OooRoundOperations`、`WorkerResult`、`CheckOutcomeSummary`)——2026-09-17 记录里说的 + 五个共享类型,收敛为搬走的两个。 + +**保留,且已核实与轮次无关:** `plan-driver.ts` 及其套件、`round-client.ts` 与 `round-host.ts` +(D14 的 daemon 证据,服务的是 store 而不是轮次)、`board-*.ts`、`board-slots.test.ts`、 +`narrow-dispatch.test.ts`、`patch-cycle.test.ts`、`cost-model*`、`families.test.ts`、`live-patch.ts`、 +`live-continuation.ts`(G7 的可执行检查——实测不导入轮次任何东西)、`pilot.ts`、`rename-probe.ts`、 +`report-family.test.ts`。 + +**台账,同一个改动内:** D10 改写为"无对手可失败"(该行的措辞本就是 `runCycle` 的提前取消分支,而没有轮次 +之后就没有谁借用 store 再关闭它——B7 先例);D9 的证据改指 board 那条测试;G4 的 cancel 半指向已经覆盖它的 +产品测试(`tests/integration/ooo-round-query.test.ts`,"the terminal decision outlives the host that made +it");台账末尾那段过时的收尾文字一并修正。除此之外没有行引用被退役的套件:约 84 个用例里只有那三处承载 +义务。 + +## 考虑过的替代方案 + +- **原地保留。** 否决:没有产品调用者、没有路由、没有 mutant、`tests/` 不碰、驱动器编排的第二份实现, + 而且把仪器的主题留在 `src/`。 +- **拆开角色层与机制层。** 依据实测否决:4 处存储调用点 vs 70 处角色引用,而角色存在的理由(越序)已经由 + 驱动器承载。 +- **把模块与套件整体挪到 `evals/`(草案的"裁")。** 操作者否决:那会造出一个没人拥有的 1 082 行模块, + 并保留下只为轮次服务的机械。 +- **保留 `round.jsonl` 的重放与比对作为研究方法。** 操作者否决:`plan-driver` 已取代 + `round-runner`/`round-compare`,而设计本就把 `round.jsonl` 归为导出("可导出作研究重放",文件不决定 + 任何状态),所以重放服务的是仪器,不是产品。 +- **不重新安置、直接删。** 否决:D9 的证据与越序性质会一起消失,而这两者都能被便宜地钉在留下来的机制上。 + +## 后果 + +- `src/` 不再包含实验文件;轮次的 1 082 行离开产品树,驱动器成为唯一的研究侧运行器。 +- 轮次携带的重试机械——pushback、被重开的依赖、未满足前提、`handlePushback`/`reopenDependency`/ + `handleUnmetPrecondition`——一并消失。没有任何台账行钉它,也没有产品路径使用它,但设计 E 臂(有预算 + 推测)将来可能需要"作废重做";若需要,那是 E 臂自己的需求、实现在驱动器上,而不是保留这个文件的理由。 +- 从 `round.jsonl` 重放真实运行的能力随之消失。归档的运行 + (`docs/experiments/execution/*`)仍作为"实验确实发生过"的证据存在;要重跑它们,得为驱动器重建一条导出 + 通路。 +- `evals/` 的套件数减少十二个文件,而幸存的驱动器套件承载了此前唯有轮次端到端承载的那条性质。 + +## 本次改动据以验证的内容 + +1. `rg "ooo-cycle" src evals tests tools` 返回空(该名字只留在实验记录与决策里描述历史)。 +2. board 的提交后通知测试在移除守卫后失败,且 D9 的证据指向它。 +3. 驱动器的越序用例在批循环不再重叠单元时失败。 +4. `DEFAULT_ROUND_PLAN` 与那八个无引用类型在任何地方都不再被引用。 +5. `npm run check`、`npm run docs:check`、`npm run test:product`、幸存的 `evals/ooo-execution/` 套件、 + `npm run mutation:teeth` 与覆盖改动路径的 `npm run agent:verify` 全部通过。 + +## 风险 + +- **仪器自有的契约网消失**(约 62 例)。它们钉的是轮次的行为,不是设计义务,而 `tests/` 从未覆盖过该模块; + 替代物是驱动器自己的套件加那两处改钉。若将来的实验臂需要轮次的语义,那就明确重建。 +- **两类历史此后缺少活的对手**:`early-cancel` 的分支与重放格式。两者都在此记录为"已退役",而不是留下一行 + 指向已删文件的过时引文。 +- **测量时浮出一个独立的预存缺口**:`agent-context.yaml` 里没有任何路由匹配 `src/integration/**`, + `ooo-board.ts` 与 `task-coordinator.ts` 也一样。那有它自己的决策;记在这里只是为了让本记录的测量不被读成 + "轮次很特别"。 diff --git a/docs/decisions/implemented/2026-09-18-skill-grows-by-routing.md b/docs/decisions/implemented/2026-09-18-skill-grows-by-routing.md new file mode 100644 index 00000000..03daa264 --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-skill-grows-by-routing.md @@ -0,0 +1,109 @@ +# A Skill that outgrows its byte budget routes detail into `references/` + +**Status:** implemented +**Approved:** explicit +**Relates to:** [repository development skill](../../../skills/repo-development/SKILL.md), +[docs index byte-budget row](../../README.md#ci-contract), +[NMG memory skill](../../../skills/nmg-memory/SKILL.md), +[long detached checks](2026-09-18-detached-long-checks.md) + +Governing meta-rule: [self-governance meta-rule](2026-09-07-self-governance-meta-rule.md) — +this change alters a Skill convention, so it carries its own decision and alternatives. + +中文版: [2026-09-18-skill-grows-by-routing.zh-CN.md](2026-09-18-skill-grows-by-routing.zh-CN.md) + +## Problem + +The byte budget that `docs/README.md#ci-contract` documents and `scripts/verify-docs.mts` +enforces is pinned to five `skills/*/SKILL.md` entries at 15,000 B each. The ceiling is not +incidental: those entries are the standing rules a generative Agent reads every session, so +the budget buys a bounded first read. + +On 2026-09-18 that ceiling did real work and then became an obstacle. `skills/repo-development/SKILL.md` +reached 15,685 B when the detached-long-checks rule was added, and the first fix was to +compress the rule — twice — to 14,931 B, leaving 69 B of headroom. That is the wrong +pressure: the rule's measured justification (the ~13 min `mutation:teeth`, the three-state +reading, the `A && B &` trap) then had to live somewhere other than the rule it explains, +and the next rule to arrive would have to fight for the same 69 bytes. A document that +cannot grow except by deleting its own reasons is not a maintained document. + +The repository already answers this. `skills/nmg-memory/` is a Skill whose entry is 10.3 KB +and whose detail sits in eight `references/*.md` files (two of them over 10 KB), routed by a +"When to read the manual" list that pairs each trigger with its reference path. Those references are +read on demand, and the budget does not apply to them — the policy row's scope is "each +high-read `skills/*/SKILL.md`", not every file under the Skill. So the pattern that keeps the +first read bounded exists; what was missing was saying that it is the intended way for a +Skill to grow. + +## Decision + +1. **A Skill grows by routing, not by raising its ceiling.** When the entry approaches its + budget, move whole sections whose _trigger_ is occasional into `references/.md`, + and add a routing list to the entry that names the trigger, not the topic: it reads "For " and gives the reference path. + Do not raise `BYTE_BUDGETS` and do not compress a rule into losing the facts it depends on. +2. **Split by how often the section is needed, not by size.** The entry keeps every section a + change normally walks through — governance, discovery, test classification, the + implement-and-verify spine. Sections needed only for a specific kind of change move out. +3. **A moved section keeps its meaning and its incoming pointers.** The whole section moves + with its conditions and exceptions (a rule may not lose a clause to a smaller file), its + relative links are re-rooted for the new depth, and every document that pointed at it by + name or anchor is updated in the same change. Note that the link checker strips `#fragment` + and therefore _cannot_ catch a stale anchor: this is a by-meaning check, not a mechanical one. +4. **A moved rule gains an owner row.** If the reference now owns a topic that `agent-context.yaml` + routes (`packaging`, `repository-control-plane`, adapters), that route names the reference + among its `owners`, exactly as the adapter routes name `skills/nmg-memory/references/harness-adapters.md`. +5. **The policy row keeps the rule.** `docs/README.md#ci-contract` states next to the byte + budget that the ceiling covers the always-read entry and that detail routes into + `references/`; the reasoning stays in this record. + +Applied to `skills/repo-development/` in the same change: `## Repository Control Plane beyond +agent:verify` (2,741 B) becomes `references/control-plane.md` and `## Builds and generated +artifacts` (1,702 B) becomes `references/builds.md`, so the entry is ~10.4 KB with ~4.5 KB of +headroom. Both sections stay available and keep their text; the two sections that carried +incoming anchors (`#before-editing`, `#implement-and-verify`) stay in the entry precisely so +no implemented record needs its link rewritten. + +## Alternatives considered + +- **Raise the 15,000 B ceiling for this entry.** Rejected. The ceiling's purpose is the + always-read first read; raising it for one entry makes the number arbitrary for every other + entry, and the pressure that produced the compression would return at 18 KB instead of 15 KB. +- **Keep one file and compress harder.** Rejected on evidence: the detached-checks rule was + already compressed twice, and the second compression removed the measured detail from the + rule while leaving it in the decision record — the rule and its justification had started to + live apart. Continuing that way trades meaning for bytes. +- **Move the entry's largest section, including `Implement and verify` (5,475 B).** Rejected. + It is the spine every change walks, so the split would cost a second file read on the common + path, and it holds one of the two incoming anchors plus the `ci-and-tests` route owner. +- **Route into `docs/design/` instead of `references/`.** Rejected: a Skill's operating + procedure is not a design document, `docs/design/` content is checked as design, and the + `references/` convention already exists in this repository with an established routing style. +- **Make the budget apply to every file under a Skill.** Rejected: it would defeat the purpose + by capping the on-demand detail that exists to keep the entry small, and it would break + `skills/nmg-memory/` immediately. + +## Consequences + +- A Skill can keep growing without its entry growing, and the always-read cost stays bounded + by a number the policy table states. +- The entry becomes a map as well as a rule set: a reader who does not need the occasional + sections never pays for them, and one who does has a named trigger to follow. +- Splitting is a decision about triggers, so it must be re-examined when a "rare" section + becomes common — the cost of a wrong split is one extra file read on the hot path. +- Stale anchors are invisible to `docs:check` (it strips `#fragment`). The review of incoming + pointers is therefore an obligation on the change, and this record states it rather than + relying on the checker. + +## Deferred + +- **Making the fragment check real.** `verify-docs.mts` could validate that a link's `#fragment` + matches a heading in the target, which would turn the pointer half of decision 3 into a + mechanical check. Not done here: it is a change to the documentation contract with its own + false-positive surface (GitHub's heading slugs vs. the repository's), and no broken anchor has + been collected yet. +- **A second split if the entry approaches its ceiling again.** The remaining large sections are + `Implement and verify` and `Repository governance`. If either has to move, it is a decision + about the hot path rather than a mechanical follow-up. +- **A pointer that lives in another worktree.** `AGENTS.md` in a sibling checkout (branch + `feat/ooo-s4-comparison`, uncommitted there) refers to the "Builds and generated artifacts" + section of this Skill by name. When such a pointer lands, it names `references/builds.md`. diff --git a/docs/decisions/implemented/2026-09-18-skill-grows-by-routing.zh-CN.md b/docs/decisions/implemented/2026-09-18-skill-grows-by-routing.zh-CN.md new file mode 100644 index 00000000..271252c3 --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-skill-grows-by-routing.zh-CN.md @@ -0,0 +1,83 @@ +# Skill 超出体积预算时,把细节路由进 `references/` + +**Status:** implemented +**Approved:** explicit +**Relates to:** [仓库开发 Skill](../../../skills/repo-development/SKILL.md)、 +[docs 索引的体积预算行](../../README.md#ci-contract)、 +[NMG 记忆 Skill](../../../skills/nmg-memory/SKILL.md)、 +[长检查异步跑](2026-09-18-detached-long-checks.md) + +治理 meta-rule:[self-governance meta-rule](2026-09-07-self-governance-meta-rule.md) —— +本变更改动了 Skill 的结构约定,因此自带决策与替代方案。 + +English version: [2026-09-18-skill-grows-by-routing.md](2026-09-18-skill-grows-by-routing.md) + +## 问题 + +`docs/README.md#ci-contract` 记录、`scripts/verify-docs.mts` 强制的那条体积预算,钉在五个 +`skills/*/SKILL.md` 入口上,每个 15 000 B。这个上限不是偶然的:这些入口是生成式 Agent 每个会话都会读的 +常驻规则,预算买的是"第一次读有界"。 + +2026-09-18 这条上限先起了作用,然后变成了障碍。加入"长检查异步跑"规则时 +`skills/repo-development/SKILL.md` 涨到 15 685 B,第一版处理是压缩这条规则——压了两次——降到 14 931 B, +只剩 69 B 余量。这是错误的压力方向:规则那份实测依据(约 13 分钟的 `mutation:teeth`、三种状态读法、 +`A && B &` 的坑)于是只能搬到别处,而下一个规则要为同样的 69 B 打架。一个只能靠删掉自己理由才能生长的 +文档,不是被维护的文档。 + +仓库里其实已经有答案。`skills/nmg-memory/` 就是"入口 10.3 KB + 八个 `references/*.md`(其中两个超过 +10 KB)"的 Skill,靠"## When to read the manual"清单路由:清单把每个触发条件与对应参考文件路径成对列出。那些 +参考文件按需读,预算也不管它们——政策行的范围写的是"each high-read `skills/*/SKILL.md`",不是 Skill 下 +的每个文件。也就是说,"让第一次读有界"的结构早就存在;缺的只是说明:这就是 Skill 生长的正规方式。 + +## 决策 + +1. **Skill 靠路由生长,不靠抬上限。** 入口接近预算时,把**触发条件是偶发**的整节移进 + `references/<主题>.md`,并在入口加一份路由清单——清单写的是**触发条件**,不是主题: + "For <任务>" 后跟该参考文件的路径。既不抬 `BYTE_BUDGETS`,也不为省字节把一条规则压到丢掉它依赖的事实。 +2. **按"多久用一次"切,不按大小切。** 入口保留一次改动正常都会走到的节——治理、发现、测试分类、 + implement-and-verify 主干。只在特定一类改动里才需要的节移出去。 +3. **搬走的节保持原意,也保持传入指针。** 整节连同它的条件与例外一起搬(规则不许为了小文件丢条款), + 相对链接按新深度重新定位,所有按名字或锚点指过来的文档在同一个改动里更新。注意:链接检查会剥掉 + `#fragment`,所以**它抓不到失效锚点**——这一条是按意义复核的义务,不是机械检查能覆盖的。 +4. **搬走的规则要拿到 owner 行。** 如果这个参考文件现在拥有了 `agent-context.yaml` 里某条路由的主题 + (`packaging`、`repository-control-plane`、各适配器),那条路由就把该参考文件列进 `owners`,正如适配器 + 路由已经列了 `skills/nmg-memory/references/harness-adapters.md`。 +5. **规则留在政策行里。** `docs/README.md#ci-contract` 在体积预算旁写明:上限覆盖的是常读入口,细节路由进 + `references/`;理由留在本记录里。 + +同一个改动里对 `skills/repo-development/` 的落实:`## Repository Control Plane beyond agent:verify` +(2 741 B)成为 `references/control-plane.md`,`## Builds and generated artifacts`(1 702 B)成为 +`references/builds.md`,入口约 10.4 KB、余量约 4.5 KB。两节都保留可读、文字不改;两个带传入锚点的节 +(`#before-editing`、`#implement-and-verify`)留在入口,正是为了让已实现的记录不必改链接。 + +## 考虑过的替代方案 + +- **给这个入口抬高 15 000 B 上限。** 否决。上限的意义是常读的第一次读;为一个入口抬就等于让这个数字对 + 其它入口都变成随意值,而产生压缩的那种压力只会在 18 KB 处重演。 +- **继续单文件硬压缩。** 依据否决:这条规则已经压过两次,第二次把实测细节从规则里压走、只留在决策记录 + 里——规则和它的依据已经开始分居。继续这么换,是拿含义换字节。 +- **搬入口里最大的那节,包括 `Implement and verify`(5 475 B)。** 否决。它是每次改动都要走的主干, + 切走等于让常见路径多读一个文件,而且它持有一个传入锚点和 `ci-and-tests` 路由的 owner。 +- **路由进 `docs/design/` 而不是 `references/`。** 否决:Skill 的操作规程不是设计文档,`docs/design/` + 的内容按设计文档受检,而 `references/` 约定在本仓库已经存在、路由写法也已定型。 +- **让预算覆盖 Skill 下的每个文件。** 否决:那会立刻破坏 `skills/nmg-memory/`,并且把"为了入口小才存在的 + 按需细节"反过来卡住,恰好取消它存在的目的。 + +## 后果 + +- Skill 可以继续长大而入口不长大,常读成本被政策表写明的那个数字钉住。 +- 入口同时成为地图:不需要偶发节的读者不必为它付费,需要的读者有名字明确的触发条件可跟。 +- 切分是关于触发条件的判断,所以某个"偶发"节变常用时必须重新审视——切错的代价是热路径上多一次读文件。 +- 失效锚点对 `docs:check` 不可见(它剥掉 `#fragment`)。因此传入指针的复核是改动的义务,本记录把它写下来, + 而不是指望检查器。 + +## 推迟 + +- **把锚点检查做实。** `verify-docs.mts` 可以校验链接的 `#fragment` 是否命中目标文件里的标题,这样决策 3 + 的指针那一半就变成机械检查。本次没做:这是对文档合同的改动,自带误报面(GitHub 的标题 slug 与本仓库的 + 规则),而且至今没有收集到失效锚点。 +- **入口再次接近上限时的第二次切分。** 剩下的大节是 `Implement and verify` 与 `Repository governance`。 + 其中任何一个要搬,都是关于热路径的决策,而不是机械跟进。 +- **住在别的 worktree 里的指针。** 兄弟 checkout(分支 `feat/ooo-s4-comparison`,那里尚未提交)里的 + `AGENTS.md` 按节名指过本 Skill 的 "Builds and generated artifacts"。这样的指针一旦落地,应指向 + `references/builds.md`。 diff --git a/docs/decisions/implemented/2026-09-18-verify-summary-restates-the-failure.md b/docs/decisions/implemented/2026-09-18-verify-summary-restates-the-failure.md new file mode 100644 index 00000000..8ea6091b --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-verify-summary-restates-the-failure.md @@ -0,0 +1,106 @@ +# `agent:verify` restates a failing check's output and names the evidence file + +**Status:** implemented +**Approved:** explicit +**Relates to:** [repository development skill](../../../skills/repo-development/SKILL.md), +[CI and quality design](../../design/ci-cd-and-quality.md), +[long detached checks](2026-09-18-detached-long-checks.md) + +Governing meta-rule: [self-governance meta-rule](2026-09-07-self-governance-meta-rule.md) — +this change alters verification tooling, so it carries its own decision and alternatives. + +中文版: [2026-09-18-verify-summary-restates-the-failure.zh-CN.md](2026-09-18-verify-summary-restates-the-failure.zh-CN.md) + +## Problem + +On 2026-09-18 a run of `npm run agent:verify -- ` reported + +```text +- [blocking] npm run test:product: failed (command exited with code 1) +``` + +and nothing else was visible where the reader looked. The same suite passed 1433/0 when run +standalone minutes later, so the run could not be classified as a regression, a flake, or a +real assertion failure — the evidence needed to tell them apart was not in front of the reader. + +The first diagnosis was a swallowed failure, and reading the code disproved it. There is no +swallowed exception and no silent path: + +- `runCommand` (`tools/agent-verify.ts`) captures stdout+stderr and keeps + `output: ok ? undefined : output.slice(-8000)` for exactly this purpose. +- `runNpmScriptCheck` (`src/rcp/verification.ts`) streams the captured output back when it is + not quiet, and both callers pass `!json` — so the route plan _and_ the narrow path print the + failing check's own words, in check order, _above_ the summary. +- What erased them here was `| tail -20`, the ordinary way to read a long run's result. + +Two real gaps remained after that: + +1. The summary line carries only `reason` — an exit code — and never the `output` the runner + went out of its way to keep. So the verdict and its cause are separated by however many + lines the streamed output occupied, which is exactly what a scrollback or a pipe eats. +2. Text mode printed the RCP receipt path but never the path to `.nmg/verification/latest.json`, + the file that holds the full structured evidence; only `--json` carried `evidencePath`. + +A third, incidental finding: a test runs the CLI with `--root --dry-run`, and +the tool persists evidence even for a dry run, so the product suite overwrites the repository's +own verification evidence. Measured: after a real run, `latest.json` held +`scopes: [".gitignore"]` with every check `skipped` / `dry run`. + +## Decision + +1. **Text mode restates the tail of a failed check.** Under each failed check's summary line, + print the last 10 non-empty lines of its captured output, indented with `| `, each line + capped at 300 characters, followed by a marker that names the truncation and points at the + evidence file. Passing and skipped checks print nothing extra. +2. **Text mode always names the evidence file** (`Evidence: `). `--json` keeps its + `evidencePath` field and its already-complete `output`. +3. **A test must not write the repository's evidence.** The dry-run case that roots the CLI at + the repository passes `--output /verification.json`, so running the product suite no + longer overwrites `.nmg/verification/latest.json`. + +What this deliberately is not: a change to streaming. The check's output still streams while it +runs; the summary restates a bounded tail next to the verdict, because the two facts a reader +needs first — what failed and why — should not require scrolling. + +## Alternatives considered + +- **Treat it as a swallowed failure and add logging around the check.** Rejected on evidence: + nothing is swallowed, `spawnSync` returns the output, the runner stores it and the streaming + path already prints it. More `try`/`catch` would have added code to a non-existent defect. +- **Stop streaming and show only the summary tail.** Rejected. A long check's progress is worth + watching while it runs, and a failure that only appears at the end is harder to attribute to + the step that produced it. +- **Print the whole captured output in the summary.** Rejected: it duplicates the stream and can + be up to 8 000 characters, which is the flooding this summary exists to avoid. +- **Print only the evidence path, with no tail.** The cheapest honest option, and the one closest + to "the tool already wrote it down". Rejected because it lands in the exact situation this + record comes from: the reader has the last screen, the failure is off it, and the path alone + costs another round trip to a file they must search by hand. +- **Change JSON mode too.** Rejected: machine consumers already receive the full `output` field; + only the human summary was missing anything. + +## Consequences + +- A failure is readable in one place: verdict, exit reason, the command's own last lines, and + where the whole text lives. +- The added output is bounded (10 lines, ≤300 characters each) per failed check, so it cannot + flood a terminal, and a run with several failing checks grows by roughly that much each. +- `.nmg/verification/latest.json` becomes load-bearing for humans, not only for machine + consumers — which is precisely why decision 3 was needed in the same change. +- The recorded diagnosis is part of the value: the next reader who sees a silent-looking + failure in a piped view has the measurement, not the guess, to tell "the output was cut by + the pipe" from "the tool dropped it". + +## Deferred + +- **Dry runs still persist evidence.** Decision 3 stops tests from overwriting the repository's + evidence, but `agent:verify --dry-run` still writes `.nmg/verification/latest.json`, so a human + dry run can still replace the last real result with a plan whose checks are all `skipped`. + Revisit if that ever misleads a decision. +- **A `--verbose` flag for the complete output.** The summary tail is deliberately bounded; if a + real failure needs more than 10 lines in place, the extension is a flag rather than a larger + default. +- **The known `test:product` parallel-load failure.** The run that started this record may have + been the flake recorded twice before. This change makes that failure diagnosable in place; it + does not identify the flake, and the difference between the two is now one screen instead of + one tool call. diff --git a/docs/decisions/implemented/2026-09-18-verify-summary-restates-the-failure.zh-CN.md b/docs/decisions/implemented/2026-09-18-verify-summary-restates-the-failure.zh-CN.md new file mode 100644 index 00000000..699795cf --- /dev/null +++ b/docs/decisions/implemented/2026-09-18-verify-summary-restates-the-failure.zh-CN.md @@ -0,0 +1,85 @@ +# `agent:verify` 复述失败检查自己的输出,并写出证据文件路径 + +**Status:** implemented +**Approved:** explicit +**Relates to:** [仓库开发 Skill](../../../skills/repo-development/SKILL.md)、 +[CI 与质量设计](../../design/ci-cd-and-quality.md)、 +[长检查异步跑](2026-09-18-detached-long-checks.md) + +治理 meta-rule:[self-governance meta-rule](2026-09-07-self-governance-meta-rule.md) —— +本变更改动了验证工具的行为,因此自带决策与替代方案。 + +English version: [2026-09-18-verify-summary-restates-the-failure.md](2026-09-18-verify-summary-restates-the-failure.md) + +## 问题 + +2026-09-18 一次 `npm run agent:verify -- ` 只报出 + +```text +- [blocking] npm run test:product: failed (command exited with code 1) +``` + +读者眼前没有别的东西。几分钟后同一套件单独跑是 1433/0,所以这次失败既不能算回归、也不能算 flake、 +也不能算真断言失败——分辨这三者所需的证据不在读者面前。 + +第一次诊断是"失败被吞掉了",读代码后否掉了。既没有异常被吞,也没有静默路径: + +- `runCommand`(`tools/agent-verify.ts`)抓到 stdout+stderr,并且正是为此保留 + `output: ok ? undefined : output.slice(-8000)`。 +- `runNpmScriptCheck`(`src/rcp/verification.ts`)在非 quiet 时把捕获的输出**回流**打印, + 而两个调用点传的都是 `!json`——所以路由计划路径**和**窄路都会把失败检查自己的话按执行顺序打印出来, + 位置在摘要**上方**。 +- 在这里抹掉它们的是 `| tail -20`,也就是读一次长跑结果最普通的做法。 + +那之后还剩两个真实缺口: + +1. 摘要行只带 `reason`(一个退出码),从不带 runner 特意留下的 `output`。于是"判决"和"原因"之间隔着 + 流式输出占掉的那几十行——正好是滚动或管道会吃掉的部分。 +2. 文本模式打印了 RCP receipt 路径,却从不打印持有完整结构化证据的 `.nmg/verification/latest.json` + 的位置;只有 `--json` 带 `evidencePath`。 + +顺带发现第三点:有个测试用 `--root <本仓库> --dry-run` 跑 CLI,而工具连 dry run 也会持久化证据, +于是产品套件会覆盖仓库自己的验证证据。实测:一次真实运行之后,`latest.json` 的内容是 +`scopes: [".gitignore"]`、每个检查都是 `skipped` / `dry run`。 + +## 决策 + +1. **文本模式复述失败检查的尾部。** 在每条失败检查的摘要行下面,缩进(`| ` 前缀)打印它捕获输出的最后 + 10 个非空行,每行截到 300 字符,随后给出一行标记,写明被截断并指向证据文件。通过和跳过的检查不额外打印。 +2. **文本模式总是写出证据文件路径**(`Evidence: `)。`--json` 保留它的 `evidencePath` 字段和 + 本就完整的 `output`。 +3. **测试不许写仓库的证据。** 那个把 CLI 根指向本仓库的 dry-run 用例改为 + `--output <临时目录>/verification.json`,于是跑产品套件不再覆盖 `.nmg/verification/latest.json`。 + +这**不是**对流式的改动:检查运行时输出照旧实时流式打印;摘要在判决旁边复述一段**有界**的尾部, +因为读者最需要的两个事实——什么失败了、为什么——不应该需要滚动才能同时看到。 + +## 考虑过的替代方案 + +- **当成"被吞掉的失败"处理,给检查加日志。** 依据否决:没有东西被吞,`spawnSync` 返回了输出,runner + 存下了它,流式路径也已经打出来了。再加 `try`/`catch` 只会在一个不存在的缺陷上加代码。 +- **取消流式,只留摘要尾部。** 否决。长检查运行时的进度值得看,而只在最后才出现的失败更难归因到 + 产生它的那一步。 +- **在摘要里打印全部捕获输出。** 否决:它与流式重复,且最长 8 000 字符,正是这份摘要要避免的刷屏。 +- **只打印证据路径,不复述尾部。** 这是最省的诚实选项,也最接近"工具已经把它写下来了"。否决理由: + 它恰好落回本记录的起因场景——读者手里只有最后一屏,失败已经在屏外,而一个路径本身要让他多跑一次 + 工具调用、再在文件里手工找。 +- **连 JSON 模式一起改。** 否决:机器消费者本来就拿到完整的 `output` 字段,缺东西的只有给人看的摘要。 + +## 后果 + +- 失败变得可在一处读完:判决、退出原因、命令自己的最后几行、完整文本在哪。 +- 新增输出有界(每条失败检查 10 行、每行 ≤300 字符),不会刷屏;有多个失败检查的跑相应各增长这么多。 +- `.nmg/verification/latest.json` 从此不仅服务机器消费者,也服务人——这正是决策 3 必须同批做的原因。 +- **诊断过程本身也是产出**:下一个在管道视图里看到"看起来静默"的失败的人,手里有实测而不是猜测, + 能分清"输出被管道截掉了"和"工具把它丢了"。 + +## 推迟 + +- **dry run 仍会持久化证据。** 决策 3 只是让测试不再覆盖仓库证据;`agent:verify --dry-run` 依旧写 + `.nmg/verification/latest.json`,所以人工 dry run 仍可能把最近一次真实结果换成一个所有检查都 `skipped` + 的计划。若真的误导了判断再处理。 +- **完整输出的 `--verbose` 旗标。** 摘要尾部是刻意有界的;若某次真实失败需要就地看超过 10 行, + 顺势的扩展是加旗标,而不是把默认值调大。 +- **已知的 `test:product` 并行加载失败。** 引起本记录的那次跑可能就是此前记过两次的 flake。本变更让这类 + 失败可以就地诊断;它并不识别那个 flake,而现在 flake 与真回归之间的差别从"一次工具调用"缩到"一屏"。 diff --git a/docs/decisions/implemented/2026-09-19-acknowledge-archived-run-directories.md b/docs/decisions/implemented/2026-09-19-acknowledge-archived-run-directories.md new file mode 100644 index 00000000..2a47eff8 --- /dev/null +++ b/docs/decisions/implemented/2026-09-19-acknowledge-archived-run-directories.md @@ -0,0 +1,39 @@ +# CI coverage acknowledges archived run directories + +[中文](2026-09-19-acknowledge-archived-run-directories.zh-CN.md) + +**Status:** implemented +**Approved:** explicit + +## Problem + +`npm run ci:uncovered-tests` fails the "Static and package contracts" job for every `.test.ts` file +that no CI job reaches and that sits outside an acknowledged root. An archived run directory is not a +suite: `docs/experiments/execution/archive/ooo-arms-2026-09-19/` keeps the candidate tree a finished +arm was judged on, `.test.ts` files included, because that tree is the evidence of the run. Five such +files made the whole job red, so a required check reported a false positive instead of drift. + +## Decision + +`ACKNOWLEDGED_ROOTS` in `tools/ci-uncovered-tests.ts` gains +`docs/experiments/execution/archive/`, with the reason recorded next to the list: an archived run +directory holds the candidate tree it was judged on as evidence, not as suites anyone maintains. +The acknowledgement is a prefix rule on purpose; a future archive inherits it only by living under +that root. + +## Alternatives considered + +- Run the archived files in a CI job: their fixture belongs to a rejected candidate of a finished + run, so the job would assert on a tree that no longer exists in the working set. +- Move the archive under `evals/`: that acknowledges by naming accident, which the tool's own + comment forbids. +- Delete the archived `.test.ts` files: they are the record of what the arm was judged on. +- Teach the scanner to skip archives separately: a second notion of "not a suite" for one outcome, + when the tool already has the acknowledged-root concept and asks for a reason. + +## Consequences + +The job fails again only on files that are genuinely outside every CI job, which is what makes the +acknowledgement worth having. An archive placed outside +`docs/experiments/execution/archive/` still needs its own row, and the reason for keeping it stays +visible at the list rather than in a commit message. diff --git a/docs/decisions/implemented/2026-09-19-acknowledge-archived-run-directories.zh-CN.md b/docs/decisions/implemented/2026-09-19-acknowledge-archived-run-directories.zh-CN.md new file mode 100644 index 00000000..0a1bdc88 --- /dev/null +++ b/docs/decisions/implemented/2026-09-19-acknowledge-archived-run-directories.zh-CN.md @@ -0,0 +1,36 @@ +# CI 覆盖率检查认领归档运行目录 + +[English](2026-09-19-acknowledge-archived-run-directories.md) + +**Status:** implemented +**Approved:** explicit + +## 问题 + +`npm run ci:uncovered-tests` 会对每个"没有 CI job 覆盖、且不在已认领根之下"的 `.test.ts` 让 +"Static and package contracts" 这个 job 变红。归档的运行目录不是套件: +`docs/experiments/execution/archive/ooo-arms-2026-09-19/` 保存的是那次实验被裁定所依据的候选树, +其中包含 `.test.ts`,因为这棵树就是那次运行的证据。5 个这样的文件让整个 job 变红,于是必需检查 +报出的是假阳性,而不是真正的漂移。 + +## 决策 + +`tools/ci-uncovered-tests.ts` 的 `ACKNOWLEDGED_ROOTS` 增加 +`docs/experiments/execution/archive/`,理由写在列表旁边:归档的运行目录保存的是它被裁定所依据的 +候选树,是证据,不是任何人维护的套件。认领刻意采用前缀规则;将来的归档只有落在该根之下才自动 +继承这条认领。 + +## 考虑过的替代方案 + +- 把归档文件放进某个 CI job:它们的 fixture 属于一次已结束实验中被否决的候选,job 会去断言一个 + 已不存在于工作集里的树。 +- 把归档挪到 `evals/` 之下:那是靠命名巧合获得认领,而工具自己的注释明确禁止这样做。 +- 删掉归档里的 `.test.ts`:它们正是那次实验被裁定所依据的记录。 +- 让扫描器单独跳过归档:同一个结果造出第二套"不算套件"的概念,而工具本来就有已认领根机制, + 并且要求写明理由。 + +## 后果 + +这个 job 之后只会因为真正落在所有 CI job 之外的文件而失败,这正是这条认领有价值的原因。放在 +`docs/experiments/execution/archive/` 之外的归档仍需自己的条目,而保留它的理由就留在列表处,不必 +去翻提交信息。 diff --git a/docs/decisions/implemented/2026-09-19-fusion-planning-repair-first.md b/docs/decisions/implemented/2026-09-19-fusion-planning-repair-first.md new file mode 100644 index 00000000..afacead1 --- /dev/null +++ b/docs/decisions/implemented/2026-09-19-fusion-planning-repair-first.md @@ -0,0 +1,67 @@ +# Fusion planning: repair-first online, ceiling offline + +[中文](2026-09-19-fusion-planning-repair-first.zh-CN.md) + +**Status:** implemented +**Approved:** explicit + +The design this implements is [docs/design/ooo-fusion-planning.md](../../design/ooo-fusion-planning.md). + +## Problem + +Fusion had one policy knob - how many units one session may carry - and a bound is not a decision: it +does not choose among legal successors, and it cannot say whether fusing is worth taking. The measured +shape is narrow (the D arm: about 1 900 ms of session startup saved per avoided session, tokens flat, +against a union tool surface that costs a chain's first unit about 0.7 k extra tokens), so what was +missing was an offline ceiling that prices fusion, and a statement of what the online move actually is. + +## Decision + +Split fusion planning into two clocks, and write down neither as a plan. + +- **Online** is one move about the current session: *admit* the next legal successor or *close* the + session, naming the condition that closed it. A fused session is irreversible, so no move may + rewrite it. +- **Repair-first**: the default is to continue; only a declared change (rejected verdict, cancellation, + an unmet dependency, a declared external wait that is not ready) may end a session early. Repairing + keeps every commitment intact and decides only about what has not run. +- **Baseline**: a move is not revisited while its facts hold, and the same plan plus the same facts + yield the same move, ties broken by plan order. Without determinism two runs of one plan are not + comparable, which is what the arms need. +- **The cost model is read-only online**: fitted offline, frozen for a run. A query optimizer re-plans + at a boundary against collected statistics and never re-collects them mid-query. +- **Offline** computes a ceiling from an optimistic projection of the plan, fed to the *same* + `sharedSessionLegal` so the five conditions keep one home: a floor from the minimum chain cover + (Dilworth: equal to the maximum antichain, computed as `units - maximumMatching` over the relation's + transitive closure) and a feasible bound from greedy list scheduling at a declared cap. + +## Alternatives considered + +- **A chain-cover compiler plus a plan cache.** Recomputing a move is a pure function over a few dozen + units, so the compile/interpret distinction a tensor graph needs - an expensive artifact used many + times - does not arise; deciding again is cheaper than invalidating correctly. Rejected. +- **Full re-planning every boundary.** Documented as the churn side of the repair-versus-replan + trade-off, and it can oscillate without new facts. Rejected in favour of repair-first. +- **Plan competition, eddies, speculative execution.** Borrowed from query optimization, they assume + re-running is nearly free and replayable; a unit is a paid, irreversible model call. Rejected. +- **Taking the ceiling as a runtime policy.** It is computed from facts a run does not have yet. + Rejected. +- **Fitting the cost model with the repository's autodiff from the paid runs.** Attempted; the fit had + leave-one-out residuals up to 12 k tokens against an 11 k mean, and the session-startup term was not + identifiable because the arm held unit count constant. The blocker is a designed matrix, not an + optimizer, so the ceiling reports the measured constant instead of a fitted coefficient. + +## Consequences + +- The question "how much is fusion worth" becomes offline and free: sessions required at caps 1 to 4, + the floor, and milliseconds saved against the measured startup. +- The union tool surface's extra turn and the context a longer chain resends are *not* modelled; they + are named in the design so a chain is never assumed free. +- The cap experiment was re-run with cache accounting recorded (same four-unit plan, `--slots 1`, 2 reps per bound, bounds 1, 2 and 4): medians were 26 620 / 17 867 / 16 645 ms and 45 685 / 39 750 / 55 286 tokens, with 80-84 % of every arm's tokens being **cache reads**. Fusion saves at least as much wall clock as predicted - cap 1 to cap 2 saves 8 753 ms against a predicted 3 800 ms, so the startup constant is plan-dependent and `sessions avoided` is the ceiling's honest primary quantity - and the extra tokens are mostly cache reads, leaving fresh input nearly flat at 7 925 / 7 942 / 9 142. **Cap 2 is the knee**: it takes 8 753 ms of the 9 975 ms available at the fewest tokens, while cap 4 buys the last 1 222 ms for 39 % more. The earlier 1.3-1.9x token multiplier came from unpaired medians and does not survive the cache-aware reading. +- The ceiling reproduces the one paid measurement: for the D arm's own plan it predicts 1 900 ms saved at cap 2, and the arm measured 12 948 ms against 11 048 ms. On the two multi-unit fixtures it reports a floor of 1 session and 3.8 s saved at cap 2, 5.7 s at cap 4 - and says cap 3 buys nothing over cap 2 on those shapes, so the money is in reaching four units per session. +- The relation is not a partial order on its own: two independent units with compatible declarations may each follow the other, so the offline graph is restricted to plan order before a chain cover can be computed. +- If the structural relation is not transitive on a real plan, the floor does not apply and the + measurement says so; the list-scheduling bound stands on its own. +- A move must be recorded with the facts it used, or "baseline" is unfalsifiable. +- Speculation and caching stay unbuilt, and nothing in this change widens `sharedSessionLegal`. + diff --git a/docs/decisions/implemented/2026-09-19-fusion-planning-repair-first.zh-CN.md b/docs/decisions/implemented/2026-09-19-fusion-planning-repair-first.zh-CN.md new file mode 100644 index 00000000..dc79b16b --- /dev/null +++ b/docs/decisions/implemented/2026-09-19-fusion-planning-repair-first.zh-CN.md @@ -0,0 +1,42 @@ +# 融合规划:在线修复优先,离线算上限 + +[English](2026-09-19-fusion-planning-repair-first.md) + +**Status:** implemented +**Approved:** explicit + +本文实现的设计:[docs/design/ooo-fusion-planning.md](../../design/ooo-fusion-planning.md)。 + +## 问题 + +融合此前只有"一个会话最多带几个单元"这一个策略旋钮,而**上限不等于决策**:它不在合法后继里做选择,也说不出融合到底值不值得。实测形状很窄(D 臂:每省一次会话启动约 1 900 ms,tokens 持平;代价是并集工具面让一条链的第一个单元多花约 0.7 k tokens),所以缺的是**一个离线的上限**来给融合定价,以及把"在线到底在决定什么"写清楚。 + +## 决策 + +把融合规划拆成两个时钟,并且**两边都不写成一个计划**。 + +- **在线**只做关于当前会话的**一个动作**:*接纳*下一个合法后继,或*关闭*会话并说出关闭它的那个条件。已融合的会话**不可撤销**,所以任何动作都不能改写它。 +- **修复优先**:默认是继续;只有**声明的变化**(裁决被拒、取消、依赖未达成、声明的外部等待未就绪)才能提前结束会话。修复保留全部已作出的承诺,只决定**尚未运行**的部分。 +- **基线**:事实不变时,动作不重议;同样的计划加同样的事实得到同样的动作,平局按计划顺序打破。没有确定性,同一计划的两次运行就不可比——而两臂正需要可比。 +- **成本模型在线只读**:离线拟合、按运行冻结。查询优化器也是在边界上拿已收集的统计**重新规划**,而**绝不在查询中途重新收集统计**。 +- **离线**从一个**乐观投影**计算上限,并把投影喂给**同一个** `sharedSessionLegal`,让五条条件只有一个归属:下界来自**最小链覆盖**(Dilworth:等于最大反链,用 `单元数 − 最大匹配`(关系传递闭包上的二分图匹配)算出),可行界来自**按声明上限的贪心 list scheduling**。 + +## 考虑过的替代方案 + +- **链覆盖编译器 + 计划缓存**:重算一个动作只是几十个单元上的纯函数;张量图需要的"编译/解释"区分——昂贵产物被反复使用——在这里不成立,重算比正确地失效更便宜。已否决。 +- **每个边界全量重规划**:属于"修复 vs 重规划"权衡里 churn 的一侧,且没有新事实也会来回改主意。已否决,改为修复优先。 +- **计划竞争 / eddies / 投机执行**:从查询优化借来,前提是重跑近乎免费且可重放;而单元是**付费、不可逆**的模型调用。已否决。 +- **把上限当运行期策略**:它由运行**尚未拥有**的事实算出。已否决。 +- **用仓库自带 autodiff 从已付费的运行里拟合成本模型**:已尝试;留一法残差最高达 12 k tokens(均值 11 k),且会话启动项**不可辨识**(该臂把单元数固定住了)。瓶颈是**设计矩阵**而不是优化器,所以上限报告**实测常数**而不是拟合系数。 + +## 后果 + +- "融合值多少"变成**离线、零花费**的问题:cap 1~4 各需几个会话、下界多少、按实测启动项折算省多少毫秒。 +- **未建模**:并集工具面多出的那一轮、以及更长链重发的上下文;设计文档里点名,避免把链当成免费。 +- **带上 cache 记录重测了 cap 实验**(同一份 4 单元计划、`--slots 1`、每臂 2 次、上限 1/2/4):中位墙钟 26 620 / 17 867 / 16 645 ms,tokens 45 685 / 39 750 / 55 286,且每一臂 80~84% 的 tokens 都是 **cache 读取**。融合省下的墙钟不少于预测(cap 1→cap 2 省 8 753 ms,预测 3 800 ms ⇒ 启动常数是**随计划而变**的,上限该以"避开几个会话"为主量),而多出来的 tokens 主要是 cache 读取,非缓存的"新输入"几乎持平(7 925 / 7 942 / 9 142)。**cap 2 才是拐点**:用最少的 tokens 拿到 9 975 ms 里的 8 753 ms,cap 4 只多买最后 1 222 ms 却贵 39%。此前"1.3~1.9 倍 tokens"来自未配对的中位数,在带 cache 的口径下不成立。 +- 上限**复现了唯一一次付费实测**:对 D 臂自己的计划,它预测 cap 2 省 1 900 ms,而实测是 12 948 ms 对 11 048 ms。对两个多单元 fixture,它报告下界为 1 个会话、cap 2 省 3.8 s、cap 4 省 5.7 s;并说明这些形状上 **cap 3 相对 cap 2 没有任何收益**,钱在"每会话达到 4 个单元"上。 +- 关系本身**不是偏序**:两个互相独立、声明兼容的单元可以互相跟随,所以离线图先按计划顺序限制成无环,才能算链覆盖。 +- 若在真实计划上**结构关系不传递**,下界不成立,且**测量会说出来**;list scheduling 的可行界独立成立。 +- 动作必须**连同它使用的事实一起记录**,否则"基线"不可证伪。 +- 推测与缓存继续不建;本变更**不放宽** `sharedSessionLegal`。 + diff --git a/docs/decisions/implemented/2026-09-19-fusion-session-mechanism.md b/docs/decisions/implemented/2026-09-19-fusion-session-mechanism.md new file mode 100644 index 00000000..2fc21b45 --- /dev/null +++ b/docs/decisions/implemented/2026-09-19-fusion-session-mechanism.md @@ -0,0 +1,94 @@ +# The fusion mechanism: one session per run + +[中文](2026-09-19-fusion-session-mechanism.zh-CN.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [the pilot and its ceiling](2026-09-18-fusion-and-speculation-pilot.md), [fusion legality and accounting](../implemented/2026-09-18-fusion-legality-and-accounting.md) + +Implementation evidence: `createPiSessionRunner` and `patchSessionInput` in `.pi/extensions/nmg/ooo-execution.ts` hold one session per run behind the `UnitState` box, `executePiInputWith` delegates to it so the extension keeps exactly one tool surface, and `piSessionWorker` in `evals/ooo-execution/plan-driver.ts` holds one runner per session id. The smoke reported both units in one session (`sessions: [["alpha","summary"]]`, 21k tokens). What remains is not mechanism: no product-side caller yet decides to reuse a session. + +## Problem + +F4 landed fusion's **policy** half (`PlanDriverSpec.fusion`, `PlanSession`/`PlanRun.sessions`, +`piWorker` refusing a continuation by name) and left the **mechanism** half open: the live worker +creates a session per call, so a fused live arm had nothing to reuse. The pilot proposal reported that +as a blocker, which framed a missing mechanism as a property of the design and left an approved budget +unspent. It is a missing mechanism, and the design says what it must be. + +## Decision + +`.pi/extensions/nmg/ooo-execution.ts` creates a `ModelRuntime`, one in-memory session, one tool set and +one prompt per call, then disposes the session. Three facts make reuse possible without new SDK +capability: + +- `session.prompt(...)` may be called again on a live session; the session's message list is the shared + context fusion exists to reuse. +- The tools already read host-owned **mutable** state (`{ artifact: string | null; abort }`, + `{ report, abort }`, `{ value: number }`), so a tool can hold a *box* and read the current unit's + values at call time instead of closing over them. +- A tool surface is fixed at session creation, so a chain registers the union of what its units may need + and each tool refuses by name when the current unit lacks that capability. + +So: replace the per-unit values the four tool factories capture with one `UnitState` box, and put the +session/prompt loop in `createPiSessionRunner({ provider, modelId, limits, surface })`, exposing +`runUnit(input): Promise` and `dispose()`. `executePiInput` becomes a runner with exactly one +unit, so there is one code path and no duplicated tool surface. The runner re-points the box per unit, +resets `reads`/`runs`/`turns`/`artifact`/`report`, keeps the session, and reports the **per-unit token +delta** (the session total minus what it was at unit start) beside the session total, because fusion's +claim is that a later unit's delta is smaller than a fresh session's. + +Then `piSessionWorker` in `evals/ooo-execution/plan-driver.ts` holds one runner per session id and +reports `metrics.sessionId`, which is what the driver's landed fusion path reads. + +Two consequences decided here rather than discovered later: + +- The artifact tool's `conclusion` schema is a literal union of the *frozen* admitted kinds, which is + per unit; a chain registers the surface once, so chains use `Type.String()` and rely on + `artifactEnvelope` (which already refuses an invented kind) plus `constrainedSampling: prefer`. Tight + literals stay on the single-unit path. +- Today one `ModelRuntime` signal covers a whole call. A chain needs a per-unit abort + (`session.abort()` on the unit's timer) and a chain budget, so the runtime is created without a + per-call timeout and the unit timer owns cancellation. + +## Alternatives considered + +- **Use the design's own fallback (a new session seeded with the accepted bytes) as the fused arm.** + Rejected: it saves nothing at startup, which is the cost fusion exists to remove. It stays the + fallback, and the design already says it is sequential handoff rather than fusion. +- **A separate chain runner with its own copy of the tools.** Rejected: two copies of a security-shaped + surface (one bounded read, a fixed check, one artifact channel) drift, and the copy is the one nobody + reviews. +- **Make adoption or fusion mandatory in the driver.** Rejected: the policy half is landed and honest; + what is missing is execution, not scheduling. + +## Consequences + +All three criteria were met once the mechanism landed: the extension keeps one tool surface and delegates to the runner, the smoke ran two units in one session, and the paid arms then ran as recorded. + +1. `executePiInput` delegates to the runner and the extension keeps exactly one tool surface; `npm run + lint` and `npm run check` pass, and every existing live path (the adapters, F3's pilot) still runs + through it. +2. In a live smoke of two units in one session on `deepseek/deepseek-v4-flash`: both units report the + same `sessionId`, and the second unit's token delta is below what a fresh session spent on the same + work - fusion's hypothesis in miniature. Spend stays within ~10-30 k tokens. +3. The paid pilot then runs as recorded in [the proposal](2026-09-18-fusion-and-speculation-pilot.md): + `{unfused, fused}` for D and, after F5, `{no speculation, speculation on one fact}` for E, three + repetitions each, reporting latency, extra cost, tokens and quality separately, inside the recorded + 1 000 k ceiling and stopping as soon as a measurement is decisive. +4. A fused arm that cannot reach its checks is recorded as a **mechanism** result ("not ready"), never + as a cost result. + +## Risks + +- The extension ships with the product, so a refactor there is the riskiest place to work. Mitigated by + making the single-unit path the same code (every live run exercises the runner) and by the lint, + check and smoke gates. +- Loosening the artifact schema on chains gives up sampling-time literals. Mitigated because + `artifactEnvelope` validates the kind and the host still validates the result; the loosening is + confined to chains. +- A long chain grows one context, which is both the saving and the risk; compaction stays disabled, and + the pilot's envelope limits per arm keep a chain bounded. +- Spend can drift if the pilot is not stopped on a decisive measurement. Mitigated by the recorded + ceiling and by the standing rule that an inconclusive cost result stops the arm rather than buying + more repetitions. diff --git a/docs/decisions/implemented/2026-09-19-fusion-session-mechanism.zh-CN.md b/docs/decisions/implemented/2026-09-19-fusion-session-mechanism.zh-CN.md new file mode 100644 index 00000000..e59a45ce --- /dev/null +++ b/docs/decisions/implemented/2026-09-19-fusion-session-mechanism.zh-CN.md @@ -0,0 +1,73 @@ +# 融合机制:一个 run 一个会话 + +[English](2026-09-19-fusion-session-mechanism.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [pilot 与其上限](2026-09-18-fusion-and-speculation-pilot.zh-CN.md)、[融合合法性与其记账](../implemented/2026-09-18-fusion-legality-and-accounting.zh-CN.md) + +实现证据:`.pi/extensions/nmg/ooo-execution.ts` 的 `createPiSessionRunner` 与 `patchSessionInput` 通过 `UnitState` 盒子让每次运行持有一个会话,`executePiInputWith` 委托给它,扩展因此只保留一套 tool surface,`evals/ooo-execution/plan-driver.ts` 的 `piSessionWorker` 每个 session id 持有一个 runner。冒烟结果显示两个单元同处一个会话(`sessions: [["alpha","summary"]]`,21k tokens)。仍未完成的不再是机制:产品侧还没有调用方去决定复用会话。 + +## Problem + +F4 落了融合的**策略半**(`PlanDriverSpec.fusion`、`PlanSession`/`PlanRun.sessions`、`piWorker` 按名拒绝续接), +**机制半**仍是空的:活体 worker 每次调用新建会话,于是融合的活体臂没有东西可复用。pilot 提案把这件事报成了 +**阻塞**——那等于把一个缺失的机制说成了设计的属性,并且让已批准的预算躺在那里没花。它是缺失的机制,而设计 +已经写明它该是什么。 + +## Decision + +`.pi/extensions/nmg/ooo-execution.ts` 每次调用创建 `ModelRuntime`、一个内存会话、一套工具、一个 prompt, +然后销毁会话。三个事实让它无需新的 SDK 能力就能复用: + +- `session.prompt(...)` 可以在活着的会话上再次调用;会话的消息列表**正是**融合要复用的共享上下文。 +- 四个工具本来读的就是宿主持有的**可变**状态(`{ artifact: string | null; abort }`、`{ report, abort }`、 + `{ value: number }`),所以工具可以持有一个**盒子**、在调用时读当前单元的取值,而不是闭包捕获。 +- 工具面在会话创建时固定,因此一条链注册"其单元可能用到的并集",每个工具在当前单元没有该能力时**按名拒绝**。 + +于是:把四个工具工厂捕获的逐单元取值换成一个 `UnitState` 盒子,把会话/prompt 循环放进 +`createPiSessionRunner({ provider, modelId, limits, surface })`,对外暴露 `runUnit(input): Promise` 与 +`dispose()`。`executePiInput` 变成"只有一个单元的 runner"——**一条代码路径,工具面不复制**。runner 逐单元重指 +盒子、重置 `reads`/`runs`/`turns`/`artifact`/`report`、保留会话,并报告**逐单元 token 增量**(会话总量减去单元 +开始时的量),与会话总量并列——因为融合的主张就是"后一个单元的增量小于新建会话"。 + +随后在 `evals/ooo-execution/plan-driver.ts` 里加 `piSessionWorker`:每个 session id 持有一个 runner,并报告 +`metrics.sessionId`——正是 driver 已落地的融合路径所读的东西。 + +两个后果在这里定下,而不是留到以后发现: + +- artifact 工具的 `conclusion` schema 是**冻结的**准入种类的字面量联合,逐单元不同;而链只在创建时注册一次 + 工具面,因此链上使用 `Type.String()`,依靠 `artifactEnvelope`(它本来就拒绝臆造的种类)与 + `constrainedSampling: prefer`。紧字面量只留在单单元路径上。 +- 今天一个 `ModelRuntime` 的 signal 覆盖整次调用。链需要**逐单元**中止(单元定时器里 `session.abort()`)与 + 链级预算,因此 runtime 创建时不带逐调用超时,由单元定时器负责取消。 + +## Alternatives considered + +- **把设计自己的降级方案(新会话 + 已接受字节种子)当作融合臂。** 拒绝:它在启动时什么都省不下来,而启动正 + 是融合要消除的成本。它继续作为降级方案存在,设计本身也已写明那是顺序交接而非融合。 +- **另写一个链式 runner,自带一套工具副本。** 拒绝:一个安全形状的工具面(一次有界读、一个固定检查、一条 + artifact 通道)出现两份必然漂移,而漂移的那一份是没人 review 的。 +- **在 driver 里把采纳或融合变成强制。** 拒绝:策略半已经落地且诚实;缺的是执行,不是调度。 + +## Consequences + +机制落地时三条验收标准均已满足:扩展只保留一套 tool surface 并委托给 runner;冒烟里两个单元同处一个会话;随后各付费臂按记录跑完。 + +1. `executePiInput` 委托给 runner,且 extension 里只有**一套**工具面;`npm run lint` 与 `npm run check` 通过, + 并且所有既有活体路径(适配器、F3 的 pilot)仍走同一条路。 +2. 用 `deepseek/deepseek-v4-flash` 做一次"同会话两单元"的活体冒烟:两个单元报同一个 `sessionId`,且第二个单元的 + token 增量低于新建会话做同样工作的量——融合假设的缩微版。花费控制在 ~10–30k token 内。 +3. 随后按[提案](2026-09-18-fusion-and-speculation-pilot.zh-CN.md)记录的付费 pilot 执行:D 臂 `{unfused, fused}`、 + 以及 F5 之后的 E 臂 `{无推测, 一个事实推测}`,各 3 次重复,分别报告时延、额外成本、token 与质量;在已记录的 + 1000k 上限内,**一旦测度可决断即停**。 +4. 融合臂若连自己的检查都达不到,记为**机制**结果("not ready"),绝不记为成本结果。 + +## Risks + +- extension 随产品发布,在这里重构是风险最高的地方。缓解:让单单元路径与链式路径**是同一段代码**(每次活体运行 + 都在跑这个 runner),加上 lint、check 与冒烟三道门。 +- 链上放宽 artifact schema 会失去采样期的字面量约束。缓解:`artifactEnvelope` 校验种类,宿主仍校验结果, + 且放宽只限链上。 +- 长链会把上下文并成一个,这既是节省也是风险;compaction 保持禁用,pilot 的逐臂 envelope 限制让链有界。 +- 若不在可决断时停下,花费会漂移。缓解:已记录的上限,以及"成本结果不决断就停臂、而不是买更多重复"的既有规则。 diff --git a/docs/decisions/implemented/2026-09-19-name-the-check-and-runner.md b/docs/decisions/implemented/2026-09-19-name-the-check-and-runner.md new file mode 100644 index 00000000..8cf3a116 --- /dev/null +++ b/docs/decisions/implemented/2026-09-19-name-the-check-and-runner.md @@ -0,0 +1,47 @@ +# Name the check identity and the check runner by what they are + +[中文](2026-09-19-name-the-check-and-runner.zh-CN.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [The integration layer gets routes](2026-09-19-route-the-integration-layer.md) + +## Problem + +Two files under `src/integration` carried an `ooo-` prefix that told a reader +nothing about which was which: `ooo-check.ts` is 38 lines holding one identity +type (`CheckTicket`, the host-issued identity of an external check), while +`ooo-verifier.ts` is the thing that actually runs a check and returns a result. +Neither name says identity or runner. + +## Decision + +Rename exactly those two: `ooo-check.ts` becomes `check-ticket.ts` and +`ooo-verifier.ts` becomes `check-runner.ts`. The other six `ooo-*.ts` files keep +their prefix and the layer stays flat. + +## Alternatives considered + +- **Rename the subsystem away from `ooo`.** Rejected: the term is the project's + own name for the scheduling model it implements, and it is load-bearing in more + than a hundred documents - `docs/design/ooo-execution-bootstrap.md`, the decision + records, and the frozen run archives under `docs/experiments/execution/archive/`. + Renaming the code would leave every historical record citing paths that no longer + exist, and those archives are evidence rather than drafts. +- **Group the subsystem into `src/integration/ooo/`, so the name is said once.** + Rejected for now: it is a structural move across roughly eighty import edges, and + the two routes now make the layer's parts visible without moving a file. +- **Rename nothing.** Rejected: those two names are the pair a reader has to decode + by opening both files. + +## Consequences + +Six files updated, no documentation reference to repair (there were none) and no +public surface changed - neither name appeared in a CLI command or a tool name. +`docs/experiments/ooo-admission-2026-09-08.md` still names the old path in prose; +it is a dated measurement record and is left as it was written. + +The layer's routes list files rather than patterns, so the renames had to reach +their declarations in `agent-context.yaml` in the same change; the test added with +those routes fails until every file under `src/integration` is either claimed or +named as a known gap. diff --git a/docs/decisions/implemented/2026-09-19-name-the-check-and-runner.zh-CN.md b/docs/decisions/implemented/2026-09-19-name-the-check-and-runner.zh-CN.md new file mode 100644 index 00000000..0d546c97 --- /dev/null +++ b/docs/decisions/implemented/2026-09-19-name-the-check-and-runner.zh-CN.md @@ -0,0 +1,27 @@ +# 让"检查身份"与"检查运行器"各自叫自己的名字 + +[English](2026-09-19-name-the-check-and-runner.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [给 integration 层配路由](2026-09-19-route-the-integration-layer.zh-CN.md) + +## 问题 + +`src/integration` 下有两个文件带着一个对读者毫无信息量的 `ooo-` 前缀:`ooo-check.ts` 只有 38 行、装着一个身份类型(`CheckTicket`,即宿主签发的某次外部检查的身份),而 `ooo-verifier.ts` 才是真正跑检查并给结果的那个。两个名字都没说明谁是身份、谁是运行器。 + +## 决策 + +只改这两个:`ooo-check.ts` → `check-ticket.ts`,`ooo-verifier.ts` → `check-runner.ts`。其余六个 `ooo-*.ts` 保持前缀,这一层保持扁平。 + +## 考虑过的替代方案 + +- **把整个子系统从 `ooo` 改名。** 否决:这是本项目给自己实现的调度模型起的名字,且它在**一百多个文档**里承重——`docs/design/ooo-execution-bootstrap.md`、各决策记录、以及 `docs/experiments/execution/archive/` 下冻结的运行归档。改代码会让每一份历史记录都引用不再存在的路径,而那些归档是证据,不是草稿。 +- **把子系统收进 `src/integration/ooo/`,让名字只说一次。** 暂缓:这是一次覆盖约八十条 import 边的结构移动,而两条新 route 已经让这一层的各部分可见,不必搬文件。 +- **什么都不改。** 否决:这两个名字正是读者必须打开两个文件才能解码的那一对。 + +## 后果 + +改了六个文件,没有文档引用需要修(本来就没有),对外接口没变——这两个名字从未出现在任何 CLI 命令或工具名里。`docs/experiments/ooo-admission-2026-09-08.md` 仍在正文里写着旧路径:那是带日期的测量记录,按原样保留。 + +这一层的 route 是**逐文件**声明而非通配,所以这两个改名必须在同一次改动里落到 `agent-context.yaml`;随那两条 route 一起加的测试,会在 `src/integration` 下出现任何未被认领、也未被点名为已知缺口的文件时失败。 diff --git a/docs/decisions/implemented/2026-09-19-route-the-integration-layer.md b/docs/decisions/implemented/2026-09-19-route-the-integration-layer.md new file mode 100644 index 00000000..b221e64f --- /dev/null +++ b/docs/decisions/implemented/2026-09-19-route-the-integration-layer.md @@ -0,0 +1,79 @@ +# The integration layer gets routes, so editing it runs checks + +[中文](2026-09-19-route-the-integration-layer.zh-CN.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [Decline the shared checks when a route owns its own](2026-09-16-route-declines-shared-checks.md) + +## Problem + +`src/integration` is declared in the `memory-runtime` capability, but no route in +`agent-context.yaml` claims it. Every edit in that layer therefore answered +`no verification route matched: src/integration/...` - no owner documents, no +tests, no checks. The layer is not small: it owns the host-neutral Agent Surface +(candidate DTO, evidence layout, board conventions, chain labels, disclosure +budgets) and the OoO/task execution orchestration, and both halves already have +owner documents and tests. + +## Decision + +Declare two routes over the layer, split by owner document rather than by +directory: + +- `agent-surface` - the presentation and contract boundary: `agent-surface.ts`, + `chain-projection.ts`, `search.ts`, `search-projection.ts`, `evidence.ts`, + `tool-contract.ts`, `config.ts`, `controller-channel.ts`, + `reasoning-workspaces.ts`, `lab-capabilities.ts`; owner + `docs/design/design.md`; tests the matching `tests/integration/*.test.ts`. +- `ooo-execution` - the execution orchestration: the six `ooo-*.ts` files, the + two renames `check-ticket.ts` and `check-runner.ts`, and the five `task-*.ts` + files; owners `docs/design/ooo-execution-bootstrap.md`, + `docs/design/task-unit-semantics.md`, `docs/design/ooo-fusion-planning.md`; + tests `tests/integration/ooo-*.test.ts` and + `tests/integration/task-semantics*.test.ts`. + +Both declare `verify: blocking: [check, test:product, build]` and `advisory: []`, +the same declaration `core-memory` and `pi-adapter` already use, and both keep the +default `sharedChecks`. + +The two routes list their files one by one, because that is what the mechanism +supports: `matches()` in `tools/repo-context.ts` treats the first `*` in a pattern +as the start of a directory prefix, so `dir/**` and exact paths match while a +mid-name pattern such as `src/integration/ooo-*.ts` matches nothing at all. A +pattern that silently matches nothing is worse than a list, so the list is the +declaration and a test keeps it complete. + +## Alternatives considered + +- **One route over `src/integration/**`.** Rejected: the two halves have + different owner documents, and one route would put two homes for one rule under + a single owner. +- **Name the files by glob (`src/integration/ooo-*.ts`).** Rejected: it matches + nothing under the current matcher, so the route would look declared and never + be selected - the exact failure this change exists to remove. +- **Fold the layer into `core-memory`.** Rejected: `core-memory` owns + `src/core/**`, and a route is a claim about who owns a path, not a catch-all. +- **Have the routes point at `evals/ooo-execution/**` as well.** Rejected: the + eval drivers are the measurement surface, and a narrow verification must not + spend paid model calls. + +## Consequences + +An edit under `src/integration` now routes: `agent:context` names the owning route +and its owner documents, and `agent:verify` runs that route's blocking checks +instead of reporting that nothing matched. The layer's `desiredRevision` changes, +so the next verification run is required before routing reads clean again. + +Because the routes list files, a new file in the layer would be claimed by nobody +and nothing would complain. `tests/tools/repo-context.test.ts` therefore asserts that +the union of the routes' paths, together with a list of knowingly unrouted files, is +exactly the directory listing - a new file fails that test until it is claimed, and +that list is empty today. + +The four retrieval-index enrichment files (`leaf-summarizer.ts`, +`node-summarizer.ts`, `summary-drain.ts`, `openai-completion.ts`) turned out to be a +third part of the layer rather than part of either half - an external LLM writes +index text that the store then persists - so they have their own `retrieval-enrichment` +route, owned by `docs/design/design.md` and +`docs/design/tiered-disclosure-design.md`. diff --git a/docs/decisions/implemented/2026-09-19-route-the-integration-layer.zh-CN.md b/docs/decisions/implemented/2026-09-19-route-the-integration-layer.zh-CN.md new file mode 100644 index 00000000..9a015f05 --- /dev/null +++ b/docs/decisions/implemented/2026-09-19-route-the-integration-layer.zh-CN.md @@ -0,0 +1,37 @@ +# 给 integration 层配路由,改它才会跑到检查 + +[English](2026-09-19-route-the-integration-layer.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [当路由自带测试时就拒绝共享检查](2026-09-16-route-declines-shared-checks.zh-CN.md) + +## 问题 + +`src/integration` 虽被 `memory-runtime` 这个 capability 声明,但 `agent-context.yaml` 里没有任何 route 认领它。于是这一层的每次改动都得到 `no verification route matched: src/integration/...`——没有拥有文档、没有测试、没有检查。这一层并不小:它同时拥有 host-neutral 的 Agent Surface(候选 DTO、证据布局、黑板约定、链标签、披露预算)与 OoO/任务执行编排,而这两半本来就各自有拥有文档和测试。 + +## 决策 + +在这一层上声明**两条** route,按**拥有文档**划分,而不是按目录划分: + +- `agent-surface`——呈现与契约边界:`agent-surface.ts`、`chain-projection.ts`、`search.ts`、`search-projection.ts`、`evidence.ts`、`tool-contract.ts`、`config.ts`、`controller-channel.ts`、`reasoning-workspaces.ts`、`lab-capabilities.ts`;拥有文档 `docs/design/design.md`;测试为对应的 `tests/integration/*.test.ts`。 +- `ooo-execution`——执行编排:六个 `ooo-*.ts`、改名后的 `check-ticket.ts` 与 `check-runner.ts`、五个 `task-*.ts`;拥有文档 `docs/design/ooo-execution-bootstrap.md`、`docs/design/task-unit-semantics.md`、`docs/design/ooo-fusion-planning.md`;测试为 `tests/integration/ooo-*.test.ts` 与 `tests/integration/task-semantics*.test.ts`。 + +两条都声明 `verify: blocking: [check, test:product, build]`、`advisory: []`,与 `core-memory`、`pi-adapter` 已有的声明一致,并保留默认的 `sharedChecks`。 + +两条 route 之所以逐文件列举,是因为机制只支持这样:`tools/repo-context.ts` 的 `matches()` 把 pattern 里的第一个 `*` 当作**目录前缀的起点**,所以 `dir/**` 与精确路径能命中,而 `src/integration/ooo-*.ts` 这种中间通配**什么都不命中**。一个静默不命中的 pattern 比一张清单更糟,所以清单就是声明,并由一个测试保证它完整。 + +## 考虑过的替代方案 + +- **一条 `src/integration/**` 的大 route。** 否决:两半的拥有文档不同,一条 route 会把"一规矩一个家"同时压到两个拥有者身上。 +- **用通配写文件名(`src/integration/ooo-*.ts`)。** 否决:在当前匹配器下它什么都不命中,route 看起来声明了却永远不会被选中——正是这次要消除的那种失败。 +- **把这一层并进 `core-memory`。** 否决:`core-memory` 拥有的是 `src/core/**`;route 是对"谁拥有这个路径"的声明,不是兜底。 +- **让这两条 route 也指向 `evals/ooo-execution/**`。** 否决:eval 驱动是测量面,窄范围验证不该花付费模型调用。 + +## 后果 + +`src/integration` 下的改动现在有路由了:`agent:context` 会说出拥有它的 route 与拥有文档,`agent:verify` 会跑该 route 的阻塞检查,而不再回答"什么都没匹配"。这一层的 `desiredRevision` 因此改变,路由读数要重新变干净,需要先跑一次验证。 + +因为 route 逐文件列举,这一层里新增的文件会**没有 route 认领、且没人抱怨**。所以 `tests/tools/repo-context.test.ts` 断言:各 route 的路径并集,加上一份“已知未认领文件”清单,恰好等于目录清单——新增文件会让该测试失败,直到它被认领;而那份清单今天是空的。 + +那四个检索索引富化文件(`leaf-summarizer.ts`、`node-summarizer.ts`、`summary-drain.ts`、`openai-completion.ts`)其实是这一层的**第三部分**,并不属于任何一半——外部 LLM 写索引文本、存储层再持久化——因此它们有自己的 `retrieval-enrichment` route,拥有文档是 `docs/design/design.md` 与 `docs/design/tiered-disclosure-design.md`。 diff --git a/docs/decisions/implemented/2026-09-19-sweep-and-evidence-rules.md b/docs/decisions/implemented/2026-09-19-sweep-and-evidence-rules.md new file mode 100644 index 00000000..c793b8aa --- /dev/null +++ b/docs/decisions/implemented/2026-09-19-sweep-and-evidence-rules.md @@ -0,0 +1,71 @@ +# A running sweep makes the tree unreadable, and evidence has no third state + +[中文](2026-09-19-sweep-and-evidence-rules.zh-CN.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [post-mortem 0003](../../postmortem/0003-checks-read-a-live-mutant.md), [post-mortem 0004](../../postmortem/0004-flaky-was-a-clock-boundary.md) + +## Problem + +Two failure classes from this branch could not be closed by the mechanisms that already existed. + +1. **A check can read a mutant.** A mutation sweep substitutes a named wrong version into a target file + and restores it afterwards; between those writes the file on disk *is* the mutant. `lint` and + `complexity:gate` run in that window reported on the mutant, and a check that *passes* there is + evidence about code that never existed. The rule that covered this named only writes ("do not edit or + stage a file a sweep is rewriting"), which is the side the sweep's own process controls - a check only + reads, so the rule never fired. The same class then escaped a second time when a killed sweep left its + mutant in the tree and the next sweep inherited it as its baseline. +2. **A label can close a question.** `test:product` failed once under load, was recorded in the ledger as + "flaky, not fixed", and stayed unexamined for a session. It was a real product defect (a 1-2 ms + disagreement between JavaScript's `Date` and SQLite's `now` at the current-value boundary), found only + when the label was refused. + +The mechanism of (1) has a mechanical guardrail now (the lock, the refusal). What remains for both is +session behaviour that no check can decide: how a reader treats a tree while a sweep holds it, and what a +ledger row is allowed to say about a failure that could not be reproduced. + +## Decision + +Three rules land in `skills/repo-development/SKILL.md`, which already owns this workflow: + +- **A running sweep makes the tree unreadable, not only unwritable.** The lock is the signal + (`tools/mutation-lock.ts`), `npm run agent:verify` refuses while it is held, and `npm run agent:context` + prints it. A reader never re-derives this from memory. +- **Evidence has no third state.** An intermittent failure is recorded with its reproduction attempt and + rate, or left open - never as "flaky", which is a label that closes a question nobody answered. +- **Progress signals differ per lane, and are not interchangeable.** A test lane streams TAP; a sweep + writes its summary only at the end, so its live signal is the lock's `target` (which moves as it takes + each target) plus the target file's mtime. + +## Alternatives considered + +- **Leave all three as prose inside the two post-mortems.** Rejected: the promotion rule in + `docs/postmortem/README.md` says a class whose guardrail is the record's own prose is not caught, and + class (1) had already escaped twice - the trigger for promotion. +- **Write a check for the label ban** (fail a ledger row that says "flaky" without a reproduction). + Rejected as over-fitted: the word is legitimate in the sentence that records why it was wrong, and the + rule it would enforce is a judgement about what a failure *meant*, not about a string. +- **Make the harness print a line per target or per mutant** so the old Skill claim ("one log line per + mutant") became true. Not rejected but deferred, and the decision does not depend on it: the lock + already answers "is it alive and where" without a new output contract, and a harness change is its own + slice. +- **Put the rules in `AGENTS.md`** (the other home the promotion table allows). Rejected: the owning + document for this workflow is the repo-development Skill, and the rules are about how to run its lanes. + +## Consequences + +- The mechanical half is carried by code and is pinned: `tools/mutation-lock.ts`, the refusal in + `tools/agent-verify.ts`, the warning lines in `tools/repo-context.ts`, and + `tests/tools/mutation-lock.test.ts` (including a case that runs a real sweep and asserts a substituted + mutant is reported `live: true`). +- The judgement half is carried by the Skill, so a session that reads it cannot conclude "the sweep + prints nothing, so it is wedged" - and cannot close a failure by naming it. +- A ledger row can no longer end a question with a label: the row that said "flaky, not fixed" now + carries the reproduction rate and the fixed defect, and its earlier wording was wrong in a way a + reader can see. +- These rules were written into the Skill before this record, which the approval tier + ([2026-09-09](2026-09-09-approval-tiers.md)) requires to be explicit. The operator approved them on + 2026-09-19 in the same session that produced them; this record is that approval, and the ordering + mistake is stated rather than hidden. diff --git a/docs/decisions/implemented/2026-09-19-sweep-and-evidence-rules.zh-CN.md b/docs/decisions/implemented/2026-09-19-sweep-and-evidence-rules.zh-CN.md new file mode 100644 index 00000000..3b4c508a --- /dev/null +++ b/docs/decisions/implemented/2026-09-19-sweep-and-evidence-rules.zh-CN.md @@ -0,0 +1,56 @@ +# 扫描运行中的树不可读,且证据没有第三态 + +[English](2026-09-19-sweep-and-evidence-rules.md) + +**Status:** implemented +**Approved:** explicit +**Relates to:** [事故记录 0003](../../postmortem/0003-checks-read-a-live-mutant.zh-CN.md)、[事故记录 0004](../../postmortem/0004-flaky-was-a-clock-boundary.zh-CN.md) + +## 问题 + +本分支上的两个失败类别,无法用已有机制关闭。 + +1. **检查会读到 mutant。** mutation 扫描把一个具名的错误版本替换进目标文件、之后再还原;在两次写之间, + 磁盘上的文件**就是** mutant。在这个窗口里跑的 `lint` 与 `complexity:gate` 报的是 mutant,而在那里**通过**的 + 检查,是关于从未存在过的代码的证据。覆盖这一点的规则只写了"写"("不要编辑、不要暂存正在被扫描改写的 + 文件"),而那正是扫描进程自己控制的一侧——检查只是**读**,所以规则从未触发。随后同一个类别第二次逃脱: + 被杀掉的扫描把 mutant 留在树里,下一条扫描把它继承为基线。 +2. **标签会关闭问题。** `test:product` 在负载下失败过一次,被 ledger 记成 "flaky, not fixed",然后一整个会话 + 无人再看。它是真实产品缺陷(current-value 边界上 JavaScript `Date` 与 SQLite `now` 相差 1–2 ms),只在 + 这个标签被拒绝之后才被发现。 + +(1) 的机制部分现在有机械防护(锁与拒绝)。两者剩下的都是**检查无法裁定的会话行为**:读者在扫描持有树时如何 +对待它,以及一条 ledger 行可以对无法复现的失败说什么。 + +## 决策 + +三条规则落在 `skills/repo-development/SKILL.md`——这个工作流原本的归属地: + +- **运行中的扫描让树不可读,而不只是不可写。** 信号是那把锁(`tools/mutation-lock.ts`), + `npm run agent:verify` 在持有期间拒绝,`npm run agent:context` 打印它。读者不再靠记忆重建这件事。 +- **证据没有第三态。** 间歇失败要么连同复现尝试与比率一起记录,要么留在 open——绝不写成 "flaky", + 那是一个关闭了无人回答之问题的标签。 +- **各车道的进度信号不同,不可互换。** 测试车道流式输出 TAP;扫描只在结束时写总结,因此它的实时信号是 + 锁的 `target`(每换一个目标就前进)加上目标文件的 mtime。 + +## 考虑过的替代方案 + +- **让三条都只留在两篇事故记录的正文里。** 拒绝:`docs/postmortem/README.md` 的晋级规则明确说,防护只是记录 + 自身正文的类别**没有被抓住**;而类别 (1) 已经逃脱两次——正是晋级的触发条件。 +- **为"禁止贴 flaky 标签"写一条检查**(ledger 行出现 flaky 且无复现就失败)。拒绝:过度拟合——这个词在记录 + "它为什么错"的句子里是正当的,而这条规则要裁的是失败**意味着什么**,不是字符串。 +- **让 harness 每目标/每 mutant 打印一行**,好让 Skill 里旧的说法("每个 mutant 一行进度")变成真的。不是拒绝 + 而是**推迟**,且本决策不依赖它:锁已经回答了"还活着吗、在哪",无需新增输出契约;改 harness 是它自己的切片。 +- **把规则写进 `AGENTS.md`**(晋级表允许的另一个归属)。拒绝:这个工作流的归属文档是 repo-development Skill, + 而这些规则说的是如何跑它的车道。 + +## 后果 + +- 机械的一半由代码承担并被钉住:`tools/mutation-lock.ts`、`tools/agent-verify.ts` 的拒绝、`tools/repo-context.ts` + 的警告行,以及 `tests/tools/mutation-lock.test.ts`(其中一例真的跑一次扫描,断言被替换的 mutant 报 + `live: true`)。 +- 判断的一半由 Skill 承担,因此读到它的会话不会得出"扫描什么都不打印,所以它卡住了",也不能靠命名来关闭失败。 +- 一条 ledger 行不再能用标签结束问题:那句 "flaky, not fixed" 现在带着复现率与已修缺陷,而它原先的措辞错在 + 读者可以看见的地方。 +- 这些规则是在本记录之前写进 Skill 的,而审批层级([2026-09-09](2026-09-09-approval-tiers.zh-CN.md))要求它为 + explicit。操作者于 2026-09-19 在产生它们的同一个会话中批准;本记录即那次批准,顺序上的失误被写明而不是掩盖。 diff --git a/docs/decisions/rejected/2026-09-02-track-build-artifacts-in-git.md b/docs/decisions/rejected/2026-09-02-track-build-artifacts-in-git.md index ca1682b3..8b573f26 100644 --- a/docs/decisions/rejected/2026-09-02-track-build-artifacts-in-git.md +++ b/docs/decisions/rejected/2026-09-02-track-build-artifacts-in-git.md @@ -58,6 +58,6 @@ clone and the tree permanently clean. `.nmg-search-scope` are untracked and ignored. - Fresh consumers of `dsh/dsh-nmg` must run `pnpm install --frozen-lockfile && pnpm run build` before the package is usable (see - `skills/repo-development/SKILL.md`). + `skills/repo-development/references/builds.md`). - CI verifies subpackage buildability from a clean checkout via `verify:packages`, so artifact exclusion cannot silently rot the build. diff --git a/docs/decisions/rejected/2026-09-02-track-build-artifacts-in-git.zh-CN.md b/docs/decisions/rejected/2026-09-02-track-build-artifacts-in-git.zh-CN.md index 538ad2a2..a228ad7b 100644 --- a/docs/decisions/rejected/2026-09-02-track-build-artifacts-in-git.zh-CN.md +++ b/docs/decisions/rejected/2026-09-02-track-build-artifacts-in-git.zh-CN.md @@ -29,5 +29,5 @@ ## Consequences - `dsh/dsh-nmg/lib/`、`src/prompts/nmg-prompts.generated.ts`、`.nmg-search-scope` 不再跟踪并被忽略。 -- `dsh/dsh-nmg` 的新消费者必须先运行 `pnpm install --frozen-lockfile && pnpm run build` 才能使用该包(见 `skills/repo-development/SKILL.md`)。 +- `dsh/dsh-nmg` 的新消费者必须先运行 `pnpm install --frozen-lockfile && pnpm run build` 才能使用该包(见 `skills/repo-development/references/builds.md`)。 - CI 通过 `verify:packages` 在干净 checkout 上验证子包可构建性,因此产物排除不会悄然腐蚀构建。 diff --git a/docs/design/ci-cd-and-quality.md b/docs/design/ci-cd-and-quality.md index e55f6208..82243c67 100644 --- a/docs/design/ci-cd-and-quality.md +++ b/docs/design/ci-cd-and-quality.md @@ -209,7 +209,7 @@ Repository Observer 按 contract include 做保守目录剪枝:仅跳过可证 当前 `observedBytes` 累计成功读取的普通文件的 stat 大小,超过默认 256MiB 时在 观察完成后附加非阻塞诊断;它不是扫描前估计或大小硬门禁。文本 plan 展示观察完成后的 文件数、字节数和诊断;JSON plan 保留 observation 字段。两者均不是读取前预警。 -Agent 默认不扫描超大目录的操作规则由 `skills/repo-development/SKILL.md` 维护; +Agent 默认不扫描超大目录的操作规则由 `skills/repo-development/references/control-plane.md` 维护; 运行时仍须完整观察 scope 内文件,不可静默跳过。是否尊重 `.gitignore` 是独立的 摘要语义决策,不在本次剪枝范围内。 @@ -527,8 +527,9 @@ Agent 的误改、自报完成和检查削弱,不防同权限恶意进程、 `npm run agent:verify` 的默认 gate 由改动归属决定。当每个改动 scope 恰好被一个路由拥有, 且没有任何 scope 落在共享 / 横切根(`src/`、`tests/`、`scripts/`、`tools/`、`.github/`、 `package.json`、`package-lock.json`、`tsconfig.json`、`tsconfig.build.json`、 -`agent-context.yaml`、`AGENTS.md`)之下时走 narrow:运行常驻共享不变量 `check`、 -`docs:check`、`format:check`、`lint`、`package:check`,加上属主路由自己的测试文件 +`agent-context.yaml`、`AGENTS.md`)之下时走 narrow:运行常驻共享不变量 +(`NARROW_SHARED_CHECKS`:`check`、`docs:check`、`format:check`、`glossary:check`、`lint`、 +`package:check`、`rtm:check`),加上属主路由自己的测试文件 (由路由 `tests:` 模式解析为具体文件,排除被匹配到的目录;零匹配时 fail-closed), 合成为 `node-test:` 检查;该子进程不继承父进程的 `NODE_TEST_CONTEXT`, 否则 node 会跳过执行并把空过记为通过;路由测试采用与可信基线相同的 TAP 接受规则 @@ -543,6 +544,14 @@ contract 覆盖时合成一个内存 contract,不存在旁路执行路径。re 完整 gate”成为机器可读的可审计事实,而非推断。`validateReceipt` 拒绝 `fullGateRun` 与 `mode` 矛盾的 receipt;`nmg-rcp receipt-verify` / `agent:verify --receipt ` 可独立复核。 +路由可以声明**常驻共享检查不适用于它自己拥有的表面**:`verify.sharedChecks: none`(默认 +`always`)。它只在本路由独占一个改动时生效,生效时 narrow 计划只跑该路由自己的测试; +它**不能**让共享/横切根免于升级(那些 scope 仍然升级为路由声明的 blocking 集),而且 +路由未声明自己的测试时在配置加载期就被拒绝(否则计划会一个检查都不执行,而验证工具 +绝不能把“什么都没跑”报成通过)。receipt 的 `gate.reason` 会写明这条声明,使它不是靠数 +检查条数才能发现的事实。理由与替代方案见 +[decision](../decisions/implemented/2026-09-16-route-declines-shared-checks.md)。 + narrow 只减少运行的检查面,不削弱绑定:结果仍绑定候选 / 基线 / 验证器 / 策略 / 调用摘要, 不复用历史证据,缺检查、跳过、超时、空报告、快照变动或依赖变化一律 fail-closed。 narrow 不声称覆盖未运行的检查;`gate.fullGateRun=false` 就是这一事实的记录。选择规则是 diff --git a/docs/design/design.md b/docs/design/design.md index 11ebeb42..fd6d6750 100644 --- a/docs/design/design.md +++ b/docs/design/design.md @@ -296,7 +296,7 @@ work. Read replicas or a worker-thread database queue remain performance options only if measurements show the event-loop writer exceeds its latency budget. Protocol compatibility epoch `nmg.v9` exposes the typed lifecycle, memory, -retrieval, maintenance, STG-sync, memory-chain, Task Board, Lab, and +retrieval, maintenance, STG-sync, memory-chain, Task Board, run-record, Lab, and session-Active-Graph methods declared in `protocol.ts` over JSON-RPC 2.0. HTTP is the only resident protocol; NMG does not maintain a parallel NDJSON or platform-specific socket API. @@ -345,7 +345,13 @@ ids from persisted retrieval-trace ids to immutable runtime projection ids. A v8 daemon cannot preserve that ownership or feedback identity, so the epoch change is intentionally incompatible. Further additive features remain within v9 and negotiate by capability unless they alter established semantics, ownership, -security, or durable interpretation. +security, or durable interpretation. An additive method carrying an optional +field that an older same-epoch daemon would silently ignore is admissible only +when the field's availability is discoverable from the hello method set and +clients gate on it: the run surface (`taskRun`, plus the optional `adopt` a +board put may carry) is the case that rule covers - a v9 daemon rejects the +method by name and would otherwise create an entry the caller believes its run +manages. ### 4.2 Modular harness adapters @@ -699,18 +705,18 @@ Relation type alone does not grant write authority. The establishment policy is defined by whether the edge is mechanically entailed by the transaction that creates it: -| Relation family | Immediate source | Otherwise | -| --- | --- | --- | -| provenance (`derived_from`) | Core derivation transaction with exact source memory IDs | Reject; a model assertion is not provenance | -| lifecycle (`supersedes`) | Validated same-domain state transition with an acyclic predecessor chain | Keep both states and request explicit resolution | -| transform structure (`is_a`, redirects) | Reviewed merge/split transaction and rollback journal | Store a topology proposal, not a consolidated edge | -| explicit domain attachment (`applies_to`, `part_of`) | Harness/tool supplies stable IDs and attributable evidence for both endpoints | Proposal or STG observation | -| regulatory (`contradicts`, `exception_to`, `distinct_from`) | Explicit attributable judgement after scope/time compatibility checks | Proposal; never infer from retrieval co-occurrence alone | -| identity/refinement (`same_as`, `refines`) | Reviewed topology proposal; `same_as` still requires the separate reversible merge actuator to canonicalize identity | Keep nodes separate | -| causal/dependency/support (`causes`, `depends_on`, `supports`) | Explicit attributable evidence or reviewed proposal | STG observation until independently supported | -| associative (`related_to`) | Stability consolidation from independent verified outcomes, or an explicit administrative relation | Co-retrieval remains an observation only | - -“Immediate” therefore means *entailed by a governed operation*, not “a model +| Relation family | Immediate source | Otherwise | +| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------- | +| provenance (`derived_from`) | Core derivation transaction with exact source memory IDs | Reject; a model assertion is not provenance | +| lifecycle (`supersedes`) | Validated same-domain state transition with an acyclic predecessor chain | Keep both states and request explicit resolution | +| transform structure (`is_a`, redirects) | Reviewed merge/split transaction and rollback journal | Store a topology proposal, not a consolidated edge | +| explicit domain attachment (`applies_to`, `part_of`) | Harness/tool supplies stable IDs and attributable evidence for both endpoints | Proposal or STG observation | +| regulatory (`contradicts`, `exception_to`, `distinct_from`) | Explicit attributable judgement after scope/time compatibility checks | Proposal; never infer from retrieval co-occurrence alone | +| identity/refinement (`same_as`, `refines`) | Reviewed topology proposal; `same_as` still requires the separate reversible merge actuator to canonicalize identity | Keep nodes separate | +| causal/dependency/support (`causes`, `depends_on`, `supports`) | Explicit attributable evidence or reviewed proposal | STG observation until independently supported | +| associative (`related_to`) | Stability consolidation from independent verified outcomes, or an explicit administrative relation | Co-retrieval remains an observation only | + +“Immediate” therefore means _entailed by a governed operation_, not “a model chose a relation label with high confidence.” Core methods may materialize such edges inside the same transaction; model-facing `remember(action="relate")` creates a pending proposal. Administrative `linkNodes` remains an explicit trust @@ -1524,8 +1530,8 @@ silently acquiring user authority before Core admission and consolidation. `remember` is the deliberate LLM intervention point between semantic judgment and deterministic storage. The two sides have different responsibilities: -| LLM / Agent | NMG core | -| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| LLM / Agent | NMG core | +| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Extract a self-contained statement; assign type, actor, time, scope, and importance; optionally provide a few short phrases by which the same fact is likely to be recalled; decide whether a returned candidate is the same meaning, a genuinely replaced old value, a related concept, or distinct. | Enforce admission policy, stable IDs, exact deduplication, scope/state invariants, provenance, transactions, trigger limits, index deltas, version history, and reversible graph changes. | `recallTriggers` are optional retrieval metadata on the same governed write, not @@ -1800,7 +1806,7 @@ from both storage tiers and the STG/LTG lifecycle: prompt as a history query. 3. **Agent-directed recall layer:** compact headers/cues that let the model call `nmg_search`, inspect costs, and expand the AG with exact details through - `nmg_get`. + `nmg_get`. Automatic recall and Agent-directed recall share the same indexed records. Explicit recall triggers may improve candidate nomination in either layer, but @@ -2216,13 +2222,13 @@ from the traversal path. ### Relationship to HierarchicalActivation -| | HA | MGR | -| --------------- | ------------------------- | ------------------------------ | -| Node role | passive data, scored | active operator, transforms | -| Scoring | 7-way similarity blend | single gate + local relevance | +| | HA | MGR | +| --------------- | ------------------------- | --------------------------------------------- | +| Node role | passive data, scored | active operator, transforms | +| Scoring | 7-way similarity blend | single gate + local relevance | | Graph structure | fixed candidate hierarchy | global seed, then optional directed neighbors | -| Parameters | 9 global | 2/node + 1 global | -| Best for | batch ranking over pool | multi-step path reasoning | +| Parameters | 9 global | 2/node + 1 global | +| Best for | batch ranking over pool | multi-step path reasoning | They remain separately gated engines but share the proposed session AG runtime: HA selects and maintains the active working set; MGR optionally transforms a diff --git a/docs/design/hidden-features-registry.md b/docs/design/hidden-features-registry.md index b0cac9be..a95c6dc1 100644 --- a/docs/design/hidden-features-registry.md +++ b/docs/design/hidden-features-registry.md @@ -12,40 +12,42 @@ them). ## Feature register -| Feature | Gate(s) | Default | Values / meaning | Location | Owner | Status | -| --------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| Feature | Gate(s) | Default | Values / meaning | Location | Owner | Status | +| --------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | | Retrieval relevance gate | program half: none (always on); model half: `/relevance-model.json` | program **ON**, no env switch; model off and **not armed** (no artifact ships) | Program half is deterministic and only removes: bounded floor 0.05, max cv 0.002, min count 3 (a flat list abstains). Model half is opt-in, declares its feature blocks plus embedder identity, and **orders the disclosed budget instead of only trimming it** (`rerankByRelevance` runs before the limit; an explicit floor still applies afterwards). Offline training/preparation is opt-in through `--manifest`, `--prepare-only`, `--qrels-split`, and `--min-retention`; source isolation and candidate-only output are governed by [relevance training](relevance-training.md). `--certify` requires complete judgments and a manifest; its result does not grant runtime admission. **Operating knob:** `coverageThreshold(scores, targetCoverage)` in `src/core/sufficiency-features.ts` sets how often a recall may come back empty; it needs no labels, and **the target coverage itself is chosen by formally running NMG's own memory evaluation suites** (`evals/omnimemeval`, dsh profile) — those suites are evaluation surfaces only and are **never training data** (the same rule that forbids LoCoMo/LongMemEval/BEAM/PersonaMem/HaluMem as training input). Plan and measurements: `docs/experiments/retrieval-quality/sufficiency-shape-features-2026-09-12.md`. **Provenance guard (planned):** a head may load only if its artifact declares `domain`, `labelSource`, `unit`, `trainedAt` and `targetCoverage`, and the loader refuses a domain mismatch the way it already refuses an embedder mismatch | `src/core/relevance-gate.ts`, `src/core/learned-gate.ts`, `src/core/sufficiency-features.ts`, `src/core/store/retrieval.ts`, `src/lab/relevance-model.ts`, `tools/relevance-model-train.ts` | this feature | program default-on (2026-09-09); model opt-in and unarmed; shape-feature head + coverage knob + provenance guard in progress (2026-09-12) | -| Fixed trusted RCP baseline | `nmg-rcp trust-install `; installed `src/rcp/trusted.ts verify ` / `trusted-verify` | off; ordinary reconcile unchanged | Explicit new installation is approval; committed snapshots only; baseline policy/test/dependency changes block pending separate approval | `src/rcp/trusted.ts`, `.rcp/trusted-policy.json` | [CI/RCP §7.13](ci-cd-and-quality.md#713-fixed-trusted-baseline-verification) | opt-in; implementation subject to regression verification | -| Narrow-default verification gate | `agent:verify` default; `--full` / `--narrow` overrides | narrow **ON** | Narrow only when every changed scope is singly owned by one non-shared route; runs `check`, `docs:check`, `format:check`, `lint`, `package:check` + route `node --test` globs; otherwise the route blocking set. Receipt `gate.mode`/`fullGateRun` records which gate actually ran | `tools/agent-verify.ts`, `tools/narrow-verify.ts`, `src/rcp/providers.ts` | [CI/RCP §7.14](ci-cd-and-quality.md#714-轻量验证默认narrow-gate) | implemented (2026-09-07) | -| Online context-use learning (ContextRouter) | `NMG_CONTEXT_ONLINE_LEARNING` | **ON** | `"0"` = off (any other value incl. `"false"` = on). Daemon stages every search that surfaced a disclosure graph (auto recall and explicit search alike; `persistTrace:false` internal probes skip); `recordFeedback` trains only the graph it names (no session-latest fallback) → RSCB reward → one router update; state `/context-router-online.json` (132 params, no rows persisted) | `src/lab/context-router-online.ts`, `src/cli/service.ts` | this feature (local commits) | default-on / in-progress-local | -| Self-contained recall-instance corpus | `NMG_RECALL_INSTANCES` | **ON** | `"0"` = off. For every disclosure recall (auto + explicit) the daemon appends one unlabeled instance `{trigger, candidates(+text), activeGraphId}` to `/recall-instances.jsonl`; relevance labelled offline by `recall-instance-judge` (independent of any answer) | `src/lab/recall-instance.ts`, `src/cli/service.ts`, `tools/recall-instance-judge.ts` | this feature (local commits) | default-on / in-progress-local | -| Controller shadow evaluation | `NMG_CONTROLLER_SHADOW` | off | `"1"` = on. Writes shadow events jsonl + the old `shadow_feedback_nudge`; powers QPP/rerank datasets | `.pi/extensions/nmg/controller-shadow.ts` | earlier feature | opt-in / dormant (superseded by online learning for the router) | -| Shadow collection-origin tag | `NMG_SHADOW_COLLECTION_ORIGIN` | natural | `natural` / `controlled` label on shadow events | `controller-shadow.ts` | earlier feature | opt-in | -| Controller runtime-state file | `NMG_CONTROLLER_RUNTIME_STATE` | auto path | file path for learned controller state | `controller-shadow.ts` | earlier feature | opt-in | -| Controller activation-receipt file | `NMG_CONTROLLER_ACTIVATION_RECEIPT` | — | file path for activation receipt | `controller-shadow.ts` | earlier feature | opt-in | -| QPP1 learned fold | `NMG_QPP1_MODE` | off | `off` / `shadow` / `active` | `src/integration/config.ts` + pi ext | earlier feature | opt-in / dormant | -| QPP2 second-pass fold | `NMG_QPP2_MODE` | off | `off` / `shadow` / `active` | `src/integration/config.ts` + pi ext | earlier feature | opt-in / dormant | -| Controller (DC) runtime rerank | `NMG_CONTROLLER_RUNTIME_MODE` | off | `off` / `shadow` / `controlled` / `active` | `src/integration/config.ts` + pi ext | earlier feature | opt-in / dormant | -| Search-recommendation nudge | `NMG_SEARCH_RECOMMENDATION` | off | `off` / `advisory` / `guardrail` | `src/integration/config.ts` + pi ext | earlier feature | opt-in | -| Lab tools (reasoning workspace, memory-graph reasoner, controller shadow; also dsh `dsh-nmg`) | `NMG_ENABLE_LAB_TOOLS` | off | `"1"` = on | pi ext / dsh plugin / daemon `lab` RPC | earlier feature | opt-in | -| Autodiff graph compile | `NMG_AUTODIFF_COMPILE` | off | compile autodiff graphs to JS for speed | `src/lab/autodiff.ts` | core | opt-in / perf knob | -| Auto-recall tuning knobs | `NMG_AUTO_RECALL_TIER` (1), `NMG_AUTO_RECALL_LIMIT` (13), `NMG_AUTO_RECALL_INITIAL_TARGET` (13), `NMG_AUTO_RECALL_STRONG_HIT_TOP_GAP` (0.05), `NMG_AUTO_RECALL_STRONG_HIT_INITIAL_TARGET` (3) | as listed | tune the default-on auto-recall | pi ext + dsh plugin | core | knobs of a default-on feature | -| Board wake cadence | `NMG_BOARD_WAKE_INTERVAL_SEC` | — | wake interval for board coordination loop | dsh plugin | earlier feature | knob | -| Daemon idle timeout | `NMG_DAEMON_IDLE_TIMEOUT_MS` | — | daemon idle shutdown | `src/cli/http-server.ts` | core | knob | -| Agent identity / capabilities | `NMG_AGENT_ID`, `NMG_AGENT_CAPABILITIES`, `NMG_AGENT_ONLINE_MS`, `DSH_SESSION_ID` | auto | board/agent registration identity | pi ext / dsh / `src/core/store/base.ts` | core | config | -| Paths | `NMG_DATA_DIR`, `NMG_PROJECT_DIR`, `HOME`, `USERPROFILE` | — | data/project directories | pi ext / dsh | core | config | -| Model selection | `NMG_EMBED_MODEL`, `NMG_SUMMARY_MODEL`, `NMG_JUDGE_MODEL`, `EVAL_MODEL`, `ANSWER_MODEL` | provider default | choose embedding/summarizer/judge/eval/answer model | embedding providers + summarizers | core | config | -| Retrieval backend mode (RPC param, not env) | `retrievalMode` | legacy | `legacy` / `fts5` / `hashing` / `qwen3` / `hybrid` | `src/cli/service.ts` (RPC) | core | config/knob | -| **Online recall feedback affordance (all recalls)** | — (no new gate; follows online learning default) | ON | Every recall output carries a compact rating invite naming its activeGraphId (attached to automatic pre-turn injection and the model's own explicit `nmg_search` results; dsh attaches once per recall snapshot). Consumer calls `nmg_remember action=feedback` with that graph id; the daemon trains only the named graph (no latest-staged fallback). Prompt key `recall_feedback_affordance` in `src/prompts/nmg-prompts.yaml`. pi + dsh both wired. Replaces the earlier one-shot auto-only `online_feedback_nudge` (removed 2026-09-07). Optional fallback still open: mechanical end-of-turn labeler (option B) if affordance responses still under-produce | pi + dsh adapters | — | implemented (2026-09-07) / follow-up B optional | -| Benchmark venv directory | `NMG_BENCH_VENV`, `NMG_HALUMEM_PYTHON` | `bge-venv` | Directory under `.benchmarks/` holding the single benchmark venv (Python 3.13 + CUDA torch + the official runners' requirements). `benchmark:omni` prepends its `Scripts/` (or `bin/`) to `PATH`; `eval:halumem:score` uses its `python.exe`. A missing venv no longer fails silently: the runner warns instead of falling back to the python on `PATH`. | `evals/omnimemeval/run.ts`, `evals/halumem/score.ts` | benchmark harness | implemented (2026-09-11) | +| Fixed trusted RCP baseline | `nmg-rcp trust-install `; installed `src/rcp/trusted.ts verify ` / `trusted-verify` | off; ordinary reconcile unchanged | Explicit new installation is approval; committed snapshots only; baseline policy/test/dependency changes block pending separate approval | `src/rcp/trusted.ts`, `.rcp/trusted-policy.json` | [CI/RCP §7.13](ci-cd-and-quality.md#713-fixed-trusted-baseline-verification) | opt-in; implementation subject to regression verification | +| Narrow-default verification gate | `agent:verify` default; `--full` / `--narrow` overrides; a route's `verify.sharedChecks: none` declaration | narrow **ON** | Narrow only when every changed scope is singly owned by one non-shared route; runs the always-run shared checks (`check`, `docs:check`, `format:check`, `glossary:check`, `lint`, `package:check`, `rtm:check` - `NARROW_SHARED_CHECKS`) + route `node --test` globs; otherwise the route blocking set. A route may declare `verify.sharedChecks: none` for a surface the shared checks cannot fail on: a narrow plan for a change that route solely owns then runs that route's own tests only, the declaration is refused at config load unless the route declares tests (the plan would otherwise execute nothing), it never covers a shared/cross-cutting path, and the receipt's `gate.reason` records it. Receipt `gate.mode`/`fullGateRun` records which gate actually ran | `tools/agent-verify.ts`, `tools/narrow-verify.ts`, `src/rcp/providers.ts` | [CI/RCP §7.14](ci-cd-and-quality.md#714-轻量验证默认narrow-gate) | implemented (2026-09-07) | +| Online context-use learning (ContextRouter) | `NMG_CONTEXT_ONLINE_LEARNING` | **ON** | `"0"` = off (any other value incl. `"false"` = on). Daemon stages every search that surfaced a disclosure graph (auto recall and explicit search alike; `persistTrace:false` internal probes skip); `recordFeedback` trains only the graph it names (no session-latest fallback) → RSCB reward → one router update; state `/context-router-online.json` (132 params, no rows persisted) | `src/lab/context-router-online.ts`, `src/cli/service.ts` | this feature (local commits) | default-on / in-progress-local | +| Self-contained recall-instance corpus | `NMG_RECALL_INSTANCES` | **ON** | `"0"` = off. For every disclosure recall (auto + explicit) the daemon appends one unlabeled instance `{trigger, candidates(+text), activeGraphId}` to `/recall-instances.jsonl`; relevance labelled offline by `recall-instance-judge` (independent of any answer) | `src/lab/recall-instance.ts`, `src/cli/service.ts`, `tools/recall-instance-judge.ts` | this feature (local commits) | default-on / in-progress-local | +| Controller shadow evaluation | `NMG_CONTROLLER_SHADOW` | off | `"1"` = on. Writes shadow events jsonl + the old `shadow_feedback_nudge`; powers QPP/rerank datasets | `.pi/extensions/nmg/controller-shadow.ts` | earlier feature | opt-in / dormant (superseded by online learning for the router) | +| Shadow collection-origin tag | `NMG_SHADOW_COLLECTION_ORIGIN` | natural | `natural` / `controlled` label on shadow events | `controller-shadow.ts` | earlier feature | opt-in | +| Controller runtime-state file | `NMG_CONTROLLER_RUNTIME_STATE` | auto path | file path for learned controller state | `controller-shadow.ts` | earlier feature | opt-in | +| Controller activation-receipt file | `NMG_CONTROLLER_ACTIVATION_RECEIPT` | — | file path for activation receipt | `controller-shadow.ts` | earlier feature | opt-in | +| QPP1 learned fold | `NMG_QPP1_MODE` | off | `off` / `shadow` / `active` | `src/integration/config.ts` + pi ext | earlier feature | opt-in / dormant | +| QPP2 second-pass fold | `NMG_QPP2_MODE` | off | `off` / `shadow` / `active` | `src/integration/config.ts` + pi ext | earlier feature | opt-in / dormant | +| Controller (DC) runtime rerank | `NMG_CONTROLLER_RUNTIME_MODE` | off | `off` / `shadow` / `controlled` / `active` | `src/integration/config.ts` + pi ext | earlier feature | opt-in / dormant | +| Search-recommendation nudge | `NMG_SEARCH_RECOMMENDATION` | off | `off` / `advisory` / `guardrail` | `src/integration/config.ts` + pi ext | earlier feature | opt-in | +| Lab tools (reasoning workspace, memory-graph reasoner, controller shadow; also dsh `dsh-nmg`) | `NMG_ENABLE_LAB_TOOLS` | off | `"1"` = on | pi ext / dsh plugin / daemon `lab` RPC | earlier feature | opt-in | +| Autodiff graph compile | `NMG_AUTODIFF_COMPILE` | off | compile autodiff graphs to JS for speed | `src/lab/autodiff.ts` | core | opt-in / perf knob | +| Auto-recall tuning knobs | `NMG_AUTO_RECALL_TIER` (1), `NMG_AUTO_RECALL_LIMIT` (13), `NMG_AUTO_RECALL_INITIAL_TARGET` (13), `NMG_AUTO_RECALL_STRONG_HIT_TOP_GAP` (0.05), `NMG_AUTO_RECALL_STRONG_HIT_INITIAL_TARGET` (3) | as listed | tune the default-on auto-recall | pi ext + dsh plugin | core | knobs of a default-on feature | +| Board wake cadence | `NMG_BOARD_WAKE_INTERVAL_SEC` | — | wake interval for board coordination loop | dsh plugin | earlier feature | knob | +| Daemon idle timeout | `NMG_DAEMON_IDLE_TIMEOUT_MS` | — | daemon idle shutdown | `src/cli/http-server.ts` | core | knob | +| Agent identity / capabilities | `NMG_AGENT_ID`, `NMG_AGENT_CAPABILITIES`, `NMG_AGENT_ONLINE_MS`, `DSH_SESSION_ID` | auto | board/agent registration identity | pi ext / dsh / `src/core/store/base.ts` | core | config | +| Paths | `NMG_DATA_DIR`, `NMG_PROJECT_DIR`, `HOME`, `USERPROFILE` | — | data/project directories | pi ext / dsh | core | config | +| Model selection | `NMG_EMBED_MODEL`, `NMG_SUMMARY_MODEL`, `NMG_JUDGE_MODEL`, `EVAL_MODEL`, `ANSWER_MODEL` | provider default | choose embedding/summarizer/judge/eval/answer model | embedding providers + summarizers | core | config | +| Retrieval backend mode (RPC param, not env) | `retrievalMode` | legacy | `legacy` / `fts5` / `hashing` / `qwen3` / `hybrid` | `src/cli/service.ts` (RPC) | core | config/knob | +| **Online recall feedback affordance (all recalls)** | — (no new gate; follows online learning default) | ON | Every recall output carries a compact rating invite naming its activeGraphId (attached to automatic pre-turn injection and the model's own explicit `nmg_search` results; dsh attaches once per recall snapshot). Consumer calls `nmg_remember action=feedback` with that graph id; the daemon trains only the named graph (no latest-staged fallback). Prompt key `recall_feedback_affordance` in `src/prompts/nmg-prompts.yaml`. pi + dsh both wired. Replaces the earlier one-shot auto-only `online_feedback_nudge` (removed 2026-09-07). Optional fallback still open: mechanical end-of-turn labeler (option B) if affordance responses still under-produce | pi + dsh adapters | — | implemented (2026-09-07) / follow-up B optional | +| Benchmark venv directory | `NMG_BENCH_VENV`, `NMG_HALUMEM_PYTHON` | `bge-venv` | Directory under `.benchmarks/` holding the single benchmark venv (Python 3.13 + CUDA torch + the official runners' requirements). `benchmark:omni` prepends its `Scripts/` (or `bin/`) to `PATH`; `eval:halumem:score` uses its `python.exe`. A missing venv no longer fails silently: the runner warns instead of falling back to the python on `PATH`. | `evals/omnimemeval/run.ts`, `evals/halumem/score.ts` | benchmark harness | implemented (2026-09-11) | ### Explicit research probes -| Feature | Gate | Default | Location / owner | Status | -| --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Narrow OoO dispatch and admission | all 12 `evals/ooo-execution/*.test.ts` files run on demand via `.github/workflows/on-demand-suites.yml` (manual dispatch only, Linux + Windows, `--test-concurrency=1`); `npm run ci:uncovered-tests` requires every unreached suite to be acknowledged, so a new suite here cannot fall out of CI silently; fixture-only daemon/worker roles and IPC clock/commit/external-event/input-version controls | off; no production wiring | `src/integration/ooo-{board,cycle,candidate,mutation,verifier,round-log}.ts` (shared layer, 2026-09-12) + `evals/ooo-execution/`; [experiment](../experiments/ooo-admission-2026-09-08.md) | fixed-plan, one-runner, safe-effect dispatch and multi-process admission probes; not an end-to-end Agent scheduler | -| Restricted OoO round entry point | `npm run ooo:round -- submit --run-dir `; `status`, `cancel`; `--live` required for a model round | off; no production runtime wiring | `evals/ooo-execution/round-cli.ts` + `round-spec.ts` + `round-runner.ts` | submit/query/cancel against a durable run directory; not a product CLI | -| Real Pi snapshot/patch execution | `node --experimental-strip-types evals/ooo-execution/live-pi.ts --live`, `evals/ooo-execution/live-patch.ts --live`, `evals/ooo-execution/live-cycle.ts --live | --replay` (`--replay`re-checks a recorded round through the host with no model call); explicit`PI_PROVIDER`/`PI_MODEL`, user-authorized provider only | off; no default session takeover | `src/integration/ooo-execution.ts`, `src/integration/ooo-patch.ts`, `src/integration/ooo-check.ts`, `.pi/extensions/nmg/ooo-execution.ts` (present but not imported by the extension index, so no session tool is registered); [live experiment](../experiments/ooo-admission-2026-09-08.md#live-pi-sdk-run--2026-09-09) | bounded snapshot/patch Pi SDK executor; `live-cycle.ts` runs one A/B/C round against a frozen baseline with host worktree checks, and never applies a proposal to the shared source or accepts one without the host check; explicit user-approved live provider | +| Feature | Gate | Default | Location / owner | Status | +| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Narrow OoO dispatch and admission | every `evals/ooo-execution/*.test.ts` suite (11 at 2026-09-18) runs on demand via `.github/workflows/on-demand-suites.yml` (manual dispatch only, Linux + Windows, `--test-concurrency=1`; the set is derived by `npm run ci:uncovered-tests`, which requires every unreached suite to be acknowledged, so a new suite here cannot fall out of CI silently); fixture-only daemon/worker roles and IPC clock/commit/external-event/input-version controls. `BoardAdmission`'s declared `slots` (default 1) and its `handoffTarget` are the only way a run admits more than one claim at once, and a count above 1 without a target is refused. Since the driver pass, `evals/ooo-execution/round-host.ts` also serves a round store as a real second process for `tests/integration/ooo-evidence-drivers.test.ts` - still fixture-only, no production wiring, and the clients it serves bound their calls and refuse the endpoint their own process serves | off; no production wiring | `src/integration/ooo-{board,candidate,mutation,verifier}.ts` (shared layer, 2026-09-12) + `evals/ooo-execution/`; [experiment](../experiments/ooo-admission-2026-09-08.md) | fixed-plan, one-runner, safe-effect dispatch and multi-process admission probes; not an end-to-end Agent scheduler | +| Bounded speculation pilot (E arm) | `node --experimental-strip-types evals/ooo-execution/speculation-pilot.ts --live` (requires `PI_PROVIDER`/`PI_MODEL`; `E_REPS` sets repetitions) | off; no default caller | One declared fact decides whether the round needs the unit. The candidate is prepared ahead of the fact and the shared layer's own `speculationOutcome` decides what happens to it - publish when the fact holds (the host still verifies the candidate with the unit's own frozen check, which is the quality term), discard and close the branch session when it does not. Every attempt runs under a fresh ticket, and a failed check keeps its candidate tree as evidence | `evals/ooo-execution/speculation-pilot.ts`; [F5](../design/task-unit-semantics-obligations.md) | +| Arms' plan driver | `node --experimental-strip-types evals/ooo-execution/plan-driver.ts run\|compare --spec --out [--slots ] [--runs ] [--session-runner]`; `--live` is required before a spec naming `worker.kind: "pi"` will call a model, and the spec names the provider and model. A spec may also declare `fusion` (`unitsPerSession`, opt-in and absent by default): the run then holds one session per slot, continues a session only from a unit the store accepted, ends it at a rejected verdict, an illegal successor or the bound, and records one entry per session in `PlanRun.sessions`. Fusion is reported from the session the worker says it used, never from the one the driver asked for, and a live `pi` worker refuses a continuation it cannot hold and names the session unless `--session-runner` is given, which holds one Pi session per driver session and is what makes the fused live arm real (the extension creates a session per call otherwise); a spec file that declares `fusion` has it copied into the run, so a spec asking for fusion is never run as the control arm. `evals/ooo-execution/pilot.ts --live --out ` runs the arms' paid pilot (A/B/C reps, seeded arm order, `PI_PROVIDER`/`PI_MODEL` required, envelope limits fixed per arm); `pilot.ts --report ` re-aggregates recorded runs and refuses a merge of two instruments, and makes no model call | `worker.kind: "stub"` in a spec makes the run offline; without `--live` a `pi` worker is refused, not downgraded. The spec's slot count is declared to the admission layer (each handoff is directed at its claimant) and reported as `slotsUsed`; a run that reached fewer slots than it asked for still says so, and `comparePlanSlots` refuses a time verdict for it. `pilot.ts` without `--live` is refused, and a spec it is given may not name a worker of its own | `evals/ooo-execution/plan-driver.ts` + `evals/ooo-execution/pilot.ts`; [the decision](../decisions/implemented/2026-09-17-arms-get-their-own-driver.md), [the slot budget](../decisions/implemented/2026-09-18-declared-slot-budget.md), [the pilot](../experiments/execution/ooo-arms-pilot-2026-09-18.md), [fusion legality and accounting](../decisions/implemented/2026-09-18-fusion-legality-and-accounting.md) | the granularity arms' research-side driver: one legal plan, a chosen slot count, the store's own verdicts, plus a fixed parent check; the pilot executes it against a real model and writes one result file per run; no product runtime wiring, and no session tool registers either entry point | +| Advisory cost model (fusion accounting) | `node --experimental-strip-types evals/ooo-execution/cost-model.ts --units --density <0..1> --work-ms --rederive-ms --hit-rate <0..1> --verify-ms --context-ms --session-start-ms --units-per-session [--session-start-measured] --coarse-context-saving <0..1> --slots [--seed ] [--out ]`, or `--sweep` | the fusion block is reported as two lines (`fusionSavedMs`, `sharedStartupMs`) and never as one net number; `fusionVerdict` returns `unmeasured` until `--session-start-measured` says a run has priced the session startup, so no threshold is read out of an assumption; `assertModelProperties` throws on a bound of half a unit, on a startup booked per unit and on a bound that removes no boundary reporting a saving | `evals/ooo-execution/cost-model.ts`; [the decision](../decisions/implemented/2026-09-18-fusion-legality-and-accounting.md), [the cost model record](../experiments/execution/ooo-cost-model-2026-09-17.md) | an offline advisory instrument (`model: "advisory-cost-only"`): it simulates cost only, has no quality term by construction, and its terms are declared parameters rather than fitted constants | +| Real Pi snapshot/patch execution | `node --experimental-strip-types evals/ooo-execution/live-pi.ts --live`, `evals/ooo-execution/live-patch.ts --live` | off; no default session takeover | `src/integration/ooo-execution.ts`, `src/integration/ooo-patch.ts`, `src/integration/check-ticket.ts`, `.pi/extensions/nmg/ooo-execution.ts` (present but not imported by the extension index, so no session tool is registered); [live experiment](../experiments/ooo-admission-2026-09-08.md#live-pi-sdk-run--2026-09-09) | bounded snapshot/patch Pi SDK executor; explicit user-approved live provider | ## Conventions diff --git a/docs/design/ooo-execution-bootstrap.md b/docs/design/ooo-execution-bootstrap.md index 58133405..0848567b 100644 --- a/docs/design/ooo-execution-bootstrap.md +++ b/docs/design/ooo-execution-bootstrap.md @@ -1,7 +1,10 @@ # 受限 OoO 的自举开发 **Status:** draft -**Related:** [选择理由](../decisions/proposed/2026-09-09-ooo-bootstrap.zh-CN.md)、[实验依据](../experiments/ooo-admission-2026-09-08.md) +**Updated:** 2026-09-18 +**Related:** [选择理由](../decisions/implemented/2026-09-09-ooo-bootstrap.zh-CN.md)、[实验依据](../experiments/ooo-admission-2026-09-08.md) + +> **2026-09-18 更新:** 轮次仪器(`round-cli.ts`、`round-runner.ts`、`round-spec.ts`、`round-compare.ts`、`ooo-round-log.ts`、`src/integration/ooo-cycle.ts`)已退役,研究侧入口改为实验臂驱动器 `evals/ooo-execution/plan-driver.ts`([决策](../decisions/implemented/2026-09-18-retire-the-round-instrument.md))。下面 S3/S4 行与 S2 说明保留为当时的记录。 ## 目标与适用范围 @@ -65,13 +68,13 @@ worker 失败、超时或输出被截断都不是完成:它被记为一次失 ## 按真实阻塞推进 -| 阶段 | 交付切片 | 离开该阶段的证据 | -| ------------- | ------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| S0 引导 | 上述三个最小能力;记录允许执行的工具与检查 | 真实补丁与真实等待事件可通过受限边界;安全负例拒绝 | -| S1 第一轮自举 | 执行 A/B/C,收集阻塞和无需修改结论 | 时间线证明合法越过;有用开发产物经独立验收,下一轮采用 | -| S2 恢复自举 | 用 S1 基线开发本轮实际暴露的恢复、取消或预算缺口 | 故障注入后无重复完成、错误解锁或无限等待;重启可恢复或明确终止 | -| S3 可用入口 | 只为重复出现的手工动作添加提交、查询、取消入口;稳定共享持久化契约 | 不修改研究脚本即可重复完成受支持任务;作用域回归通过。**已做**(2026-09-12):`round-spec.ts`(数据描述一轮:基线、检查、两个任务的指令/可编辑/可见集、用例、声明故障、准入结论、前提、预算与限额、worker 种类)+ `round-runner.ts`(把 spec 冻结成轮次、把运行目录做成持久面)+ `round-cli.ts`(`submit` / `status` / `cancel`)。重复一轮不需要改研究脚本:同一个 spec 先跑一遍并记录日志,再把 `worker` 指向那份日志重放。`cancel` 写在轮次自己的 store 里,运行中的轮次轮询它并据此自我取消、杀掉在途检查进程树;`status` 由另一个进程读同一份运行目录。边界(2026-09-12 更新):**共享层已就位**——协调器、轮次编排、宿主验证与轮次记录都在 `src/integration/ooo-{board,cycle,candidate,mutation,verifier,round-log}.ts`;`evals/ooo-execution/` 只留研究启动器(`live-*`、`round-{spec,runner,cli,compare}`)与测试。它仍**没有接进产品运行时**:不接 `src/cli`,也不默认接管会话。Pi 侧新增了一个薄面:`ooo_round` 工具(默认注册,与其它 NMG 工具一致:需要打开的消费者不算消费者。成本边界按次——`live` 才花 token,默认重放记录答案、不花钱;`submit` 分离式启动、`status`/`cancel` 由另一进程读写同一运行目录),它绑定到已评审的入口而不在此重新实现协调器。spec 由操作者编写,因此它的路径不作为不可信输入处理。 | -| S4 对照评估 | 对同一冻结计划比较顺序执行和 OoO | 报告质量、耗时、tokens、失败与人工干预,不预设收益。**已做(工具与离线臂)**(2026-09-12):`cycle.ts` 增加 `mode: "ooo" | "sequential"`(同一计划、同一输入、同一检查与验收规则,只改派发顺序;模式进入轮次身份,重放不得跨模式);`round-compare.ts`+`round-cli.ts compare --runs N` 跑两个臂、各臂保留自己的 store/日志/记录,报告质量(verdicts/accepted)、墙钟、宿主检查次数与耗时、被覆盖的等待、tokens、cacheRead、worker 检查次数、killed/survived、失败与 reopen,并明确写出“两臂质量不同就不比较时间”。两臂质量不一致时退出码非零。**未做**:真实模型下的对照需要 2×N 轮付费运行,属于操作者授权范围;离线臂用记录的答案,token 列为 0、测的是调度与宿主开销。 | +| 阶段 | 交付切片 | 离开该阶段的证据 | +| ------------- | ------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| S0 引导 | 上述三个最小能力;记录允许执行的工具与检查 | 真实补丁与真实等待事件可通过受限边界;安全负例拒绝 | +| S1 第一轮自举 | 执行 A/B/C,收集阻塞和无需修改结论 | 时间线证明合法越过;有用开发产物经独立验收,下一轮采用 | +| S2 恢复自举 | 用 S1 基线开发本轮实际暴露的恢复、取消或预算缺口 | 故障注入后无重复完成、错误解锁或无限等待;重启可恢复或明确终止 | +| S3 可用入口 | 只为重复出现的手工动作添加提交、查询、取消入口;稳定共享持久化契约 | 不修改研究脚本即可重复完成受支持任务;作用域回归通过。**已做**(2026-09-12):`round-spec.ts`(数据描述一轮:基线、检查、两个任务的指令/可编辑/可见集、用例、声明故障、准入结论、前提、预算与限额、worker 种类)+ `round-runner.ts`(把 spec 冻结成轮次、把运行目录做成持久面)+ `round-cli.ts`(`submit` / `status` / `cancel`)。重复一轮不需要改研究脚本:同一个 spec 先跑一遍并记录日志,再把 `worker` 指向那份日志重放。`cancel` 写在轮次自己的 store 里,运行中的轮次轮询它并据此自我取消、杀掉在途检查进程树;`status` 由另一个进程读同一份运行目录。边界(2026-09-12 更新):**共享层已就位**——协调器、轮次编排、宿主验证与轮次记录都在 `src/integration/ooo-{board,cycle,candidate,mutation,verifier,round-log}.ts`;`evals/ooo-execution/` 只留研究启动器(`live-*`、`round-{spec,runner,cli,compare}`)与测试。它仍**没有接进产品运行时**:不接 `src/cli`,也不默认接管会话。Pi 侧新增了一个薄面:`ooo_round` 工具(默认注册,与其它 NMG 工具一致:需要打开的消费者不算消费者。成本边界按次——`live` 才花 token,默认重放记录答案、不花钱;`submit` 分离式启动、`status`/`cancel` 由另一进程读写同一运行目录),它绑定到已评审的入口而不在此重新实现协调器。spec 由操作者编写,因此它的路径不作为不可信输入处理。 | (`ooo_round` 已于 2026-09-14 离开产品工具目录,普通协作路径不需要专用工具;见 `docs/design/task-unit-semantics-obligations.md` G5) | +| S4 对照评估 | 对同一冻结计划比较顺序执行和 OoO | 报告质量、耗时、tokens、失败与人工干预,不预设收益。**已做(工具与离线臂)**(2026-09-12):`cycle.ts` 增加 `mode: "ooo" | "sequential"`(同一计划、同一输入、同一检查与验收规则,只改派发顺序;模式进入轮次身份,重放不得跨模式);`round-compare.ts`+`round-cli.ts compare --runs N` 跑两个臂、各臂保留自己的 store/日志/记录,报告质量(verdicts/accepted)、墙钟、宿主检查次数与耗时、被覆盖的等待、tokens、cacheRead、worker 检查次数、killed/survived、失败与 reopen,并明确写出“两臂质量不同就不比较时间”。两臂质量不一致时退出码非零。**未做**:真实模型下的对照需要 2×N 轮付费运行,属于操作者授权范围;离线臂用记录的答案,token 列为 0、测的是调度与宿主开销。 | 每轮开始先复核上一轮暴露的问题,再固定本轮计划。遇到缺口,记录任务、预期、实际现象、最小复现和阻塞能力;选择影响下一轮最小闭环的问题修复。调查细节和时间线放实验记录,不复制进本设计。没有证据的通用平台能力继续留在范围外。 @@ -147,7 +150,7 @@ S2 的退出条件“故障注入后无重复完成、错误解锁或无限等 ### 推测的落点(规则由决策记录拥有,本节只列可落地的位置) -是否启用、以及启用条件,由[推测盈亏平衡决策](../decisions/proposed/2026-09-11-ooo-speculation.md)拥有;本节只回答“哪些点适合”。该决策仍为 proposed,所以上面“不推测”的现行规则继续有效,直到它进入 implemented。按**猜错花掉什么**分三类,这一点决定了一切。 +是否启用、以及启用条件,由[推测盈亏平衡决策](../decisions/implemented/2026-09-11-ooo-speculation.md)拥有;本节只回答“哪些点适合”。该决策仍为 proposed,所以上面“不推测”的现行规则继续有效,直到它进入 implemented。按**猜错花掉什么**分三类,这一点决定了一切。 **已有前提**。安全推测需要的三件事已经就位:(a)错误的推测产物必须能丢弃且不得生效——补丁契约只允许产出提案、由宿主应用,worker 没有不可逆副作用,所以 squash 无需回滚任何状态;(b)提交点必须能廉价判定猜测对错——真实检查终态带检查身份与 digest,正好是那个判定点;(c)提交顺序不变。`inputDigest` 会把假设绑定进输入,因此**猜错的产物自动失效**(fenced)而不是被误接受:这正是 ROB 保证精确状态的那件事在我们这里的对应物。 diff --git a/docs/design/ooo-fusion-planning.md b/docs/design/ooo-fusion-planning.md new file mode 100644 index 00000000..c3b6969f --- /dev/null +++ b/docs/design/ooo-fusion-planning.md @@ -0,0 +1,159 @@ +# Fusion planning: repair-first online, ceiling offline + +How a plan's units are assigned to Agent sessions: what the host decides at a run boundary, and what +is measured offline instead. The legality rule itself is not here - it lives in +`src/integration/ooo-execution.ts` (`sharedSessionLegal`) and this document never restates it. + +## The problem this answers + +Fusion saves a session's startup by running several units in one session. The bound on how many +units one session may carry is the only fusion policy the repository had, and a bound is not a +decision: it does not say which successors to take, and it cannot say whether fusion is worth taking +at all. The measured shape is narrow - the D arm found ~1 900 ms of startup saved per avoided +session with tokens flat, against a union tool surface that cost the first unit about 0.7 k extra +tokens - so the useful question is *how many sessions a plan can be compressed into*, not whether +fusion is a good idea in general. + +## Two clocks + +| | Online (a run boundary) | Offline (analysis) | +|---|---|---| +| Decides | the next move of the current session | how many sessions the plan could need at best | +| Facts | the ones the run actually holds | the optimistic projection: every unit accepted, nothing cancelled, no external wait pending, no pending branch | +| Cost | none (a pure function, milliseconds) | none (no model calls) | +| Fails by | closing a session it should have continued | overstating what fusion can save | + +The offline half is a **ceiling**, not a policy. Nothing in a run may read it to decide a move, because +it is computed from facts the run does not have yet. + +## Online: ready set, repair-first, baseline + +A fused session is an irreversible commitment: two units that ran in one session cannot be un-fused. +So the online decision is not a plan, it is one move about the *current* session: + +- **admit** the next legal successor, or +- **close** the session, naming the condition that closed it. + +Three properties make that move safe to make repeatedly: + +- **Repair-first.** The default is to continue the current session; a re-decision may only *end* it, on + a declared change: a rejected verdict, a cancellation, a dependency that did not become accepted, or + a declared external wait that is not ready. Repairing instead of re-planning is the documented + trade: reusing a plan saves work but risks acting on a stale one, and re-planning from scratch churns + (`plan repair versus full replanning`). Repair-first is the middle: keep what is committed, decide + only about what has not run. +- **Baseline.** A move, once made, is not revisited while its facts hold. Facts arrive at boundaries; + between boundaries there is nothing to re-decide, so an unchanged fact set yields an unchanged set of + moves. This is Oracle SQL Plan Management's plan-baseline idea: constrain the plan to accepted + alternatives so an unrelated change cannot silently move it. +- **Determinism.** The same plan and the same facts yield the same move, with ties broken by plan + order. This is the workflow-engine discipline (Temporal replays workflow code against recorded + history, so the code must be a pure function of that history; non-determinism belongs in activities): + the model call is the activity, the move is the replayable code. Without it, two runs of one plan are + not comparable, which is what the arms need. + +The cost model is read-only online. It is fitted offline and frozen for a run, which is exactly how a +query optimizer treats statistics: it re-plans at a boundary against collected statistics, and it never +re-collects them mid-query. + +## Offline: the ceiling + +The legal graph is state-dependent, so the offline half prices an optimistic projection of it: the +projection is fed to the *same* `sharedSessionLegal`, which keeps one home for the five conditions, and +the only condition the projection cannot answer - a successor must not need a unit that has not run yet +- is added as "not reachable by the successor relation in reverse": a chain is a linear extension, so +`a` may not be followed by `b` when `a` transitively depends on `b`. + +Two numbers come out of that graph: + +- **A floor**, when the structural relation is transitive: the minimum chain cover of a partial order + equals its maximum antichain (Dilworth), and both are `units - maximumMatching` over the bipartite + graph of the relation's transitive closure. That is the least number of sessions the plan could + possibly need. +- **A feasible bound** from greedy list scheduling at a declared per-session cap: walk the plan in plan + order and append each unit to the open session that can legally take it, preferring the fullest, or + open a new one. + +The gap between the two is the honest answer to "how much is fusion worth": the floor is what fusion +could save at best, and the list-scheduling result is what the current rule actually gets. Reported +against the measured startup, the difference is milliseconds saved. + +Not modelled by the ceiling, and now measured rather than merely named: the union tool surface's extra +turn (about 0.7 k tokens on a chain's first unit), and the tokens a longer chain spends carrying its +context. The cap experiment above prices the second at about 15 % more *fresh* input per four units - +most of a chain's extra tokens are cache reads - so the ceiling stays a wall-clock ceiling, and the +cost of fusing is real but far smaller than a raw token count suggests. + +## What the ceiling says today + +Run against the fixtures and against the D arm's own spec +(`node --experimental-strip-types evals/ooo-execution/fusion-ceiling.ts [--spec ]`): + +| plan | units | floor | cap 1 | cap 2 | cap 3 | cap 4 | +|---|---|---|---|---|---|---| +| `fixtures/report/fine.spec.json` | 4 | 1 | 4 sessions | 2 (3.8 s) | 2 (3.8 s) | 1 (5.7 s) | +| `fixtures/pipeline/fine.spec.json` | 4 | 1 | 4 sessions | 2 (3.8 s) | 2 (3.8 s) | 1 (5.7 s) | +| the D arm's `spec-2.json` | 2 | 1 | 2 sessions | 1 (**1.9 s**) | | | + +The third row is the check that makes the method credible rather than decorative: the ceiling predicts +1 900 ms saved at cap 2 for the plan the D arm actually ran, and the arm measured 12 948 ms at bound 1 +against 11 048 ms at bound 2 - the same 1 900 ms, from a tool that calls no model and reads only the +plan. Two things the table also says: the floor is 1 for both multi-unit fixtures, so a plan is fully +fusible in principle; and cap 3 buys nothing over cap 2 on these shapes, because the fourth unit has +to wait for the first three - the money is in reaching 4 units per session, not in raising the bound +one notch. + +### The cap experiment, measured (2026-09-19) + +The ceiling's prediction for a four-unit plan was tested on `fixtures/pipeline/fine.spec.json` - three +independent units and one that joins them, the shape of the report fixture - live, `--slots 1`, with +the spec's canned answers stripped so the units really run. The second run below records cache +accounting beside tokens, because that is what turns a token count into a cost. + +| bound | sessions | wall (medians of 2) | tokens | cache read | tokens - cache read | +|---|---|---|---|---|---| +| 1 | 4 | 26 620 ms | 45 685 | 37 760 | 7 925 | +| 2 | 2 | 17 867 ms | 39 750 | 31 808 | 7 942 | +| 4 | 1 | 16 645 ms | 55 286 | 46 144 | 9 142 | + +Three things, and the second one corrects this document's first reading of the same experiment: + +- **Fusion saves wall clock, and more than the constant predicted.** Cap 1 to cap 2 saves 8 753 ms and + to cap 4 saves 9 975 ms, against 3 800 ms and 5 700 ms predicted from the D arm's 1 900 ms. So the + startup term is **plan-dependent** (about 2.9-3.3 s here), and the ceiling's primary quantity should + be *sessions avoided* - exact and model-free - with milliseconds as an estimate that names its + constant. +- **The token multiplier was a count multiplier.** Every arm spends 80-84 % of its tokens on **cache + reads**, and the tokens that are not cache reads - the part that is priced like fresh input - are + nearly flat: 7 925, 7 942, 9 142. Fusing four units into one session costs about **15 % more fresh + input**, not the 1.3-1.9x an unpaired token median suggested earlier. A chain carries its context + forward, and the provider serves most of that from cache. +- **Cap 2 is the knee.** It takes 8 753 ms of the 9 975 ms available while sending the *fewest* tokens + of the three (39 750), and cap 4 buys the last 1 222 ms for 39 % more tokens. The policy worth + declaring is therefore two units per session, not four. + +## Why this shape, and what it is not + +Borrowed, with sources: stage-barrier re-planning (Spark's adaptive execution re-optimizes the +remaining query at a stage boundary using runtime statistics), ready-set scheduling (every workflow +engine), plan baselines (Oracle), deterministic replay (Temporal), dwell time and hysteresis against +chatter (switched-system receding-horizon control), and Dilworth. + +Not borrowed, because the analogy breaks: plan competition, eddies and speculative execution. Those +assume re-running a plan is nearly free and replayable; a unit is a paid model call with side effects +that cannot be replayed, so a wrong execution is money spent, not CPU time recovered. This is the same +reason speculative fusion is not built: a wrong bet destroys the session it bets on. + +Also not built: any cache of a plan, a compiled plan artifact, or a chain-cover compiler. Recomputing a +move is a pure function over a few dozen units, so the compile/interpret distinction that a tensor +graph needs - where the compiled artifact is expensive to build and used many times - does not arise +here; it is cheaper to decide again than to invalidate correctly. + +## Obligations + +- If the structural relation turns out not to be transitive on a real plan, the Dilworth floor does not + apply; the list-scheduling bound still does, and the measurement says so rather than assuming. +- A move decided online must be recorded with the facts it used, or "baseline" is unfalsifiable. +- The ceiling is only as good as the measured startup term; if a later run prices session startup + differently, the constant is what changes, not the method. +- Nothing here may widen `sharedSessionLegal`. A ranked candidate set is a policy over a fixed rule. diff --git a/docs/design/task-unit-semantics-obligations.md b/docs/design/task-unit-semantics-obligations.md new file mode 100644 index 00000000..6a268e39 --- /dev/null +++ b/docs/design/task-unit-semantics-obligations.md @@ -0,0 +1,382 @@ +# The contract's obligations, one node per line + +**Authority:** living ledger for `docs/design/task-unit-semantics.md` — each row is one +obligation from that design; progress is counted in rows moved to `proven`, not in edits made. + +Counts at this revision, computed from the rows below rather than from memory: **A** 5 proven, 0 partly, 0 owed (5 rows); **B** 9 proven, 0 partly, 0 owed (9 rows, B7 nothing to fail); **C** 4 proven, 0 partly, 0 owed (4 rows); **D** 13 proven, 0 partly, 0 owed, 1 not applicable (14 rows; D10's subject, `runCycle`, was retired by [the retirement decision](../decisions/implemented/2026-09-18-retire-the-round-instrument.md)); **E** 9 proven as a seam with offline proofs and no source wired (9 rows); **G** 7 proven, 0 partly, 0 owed (7 rows). **F** complete except the fusion slice: its offline layer, the plan's single home, the arms' driver and its slot budget, the two task families and the real-model pilot have all landed and are recorded below, the last one in [the pilot record](../experiments/execution/ooo-arms-pilot-2026-09-18.md) with its own sample-size caveat; **F4** (execution fusion) has landed its offline half (legality, accounting, the driver's session policy) **and** the live mechanism it needed - `createPiSessionRunner` in the extension, `piSessionWorker` + `--session-runner` in the driver - and the paid D arm is measured: fused `unitsPerSession: 2` against the `1` control, 3 reps each, one session of two units against two sessions of one, quality parity, ~1.9 s faster per run and token-neutral, which also prices the session-startup term the cost model had left `unmeasured` ([the D arm](../experiments/execution/ooo-arms-pilot-2026-09-18.md#d-arm-fusion-added-2026-09-19)). **F5** (speculation lifecycle) landed offline in the same shared module: the design's `assumptions=[{predicateId, version, expected}]` as a declaration, the first experiment's bounds (exactly one pending fact, no speculative successor, no irreversible operation) refused by name, and the three outcomes read from authoritative evidence - true publishes, false discards the candidate and closes its branch session, unknown waits, because a missing reading is not permission to publish - with 9 cases and 5 named mutants. The only arm still unrun is the **E arm** itself (budgeted speculation against a real model), which needs an instrument that decides the guessed fact and re-runs the real path under a new ticket. The counts sentence above is what this line used to over-claim: it read "**F** complete" while the F4 row said otherwise. F1 built the advisory cost model the design orders before any paid call (`evals/ooo-execution/cost-model.ts`, derived plan graphs, self-checks, no quality term) and recorded its sweep in [the experiment record](../experiments/execution/ooo-cost-model-2026-09-17.md); the sweep says one slot buys nothing (so B is predicted worse than A on cost), a chain or a two-unit refinement loses at every granularity, and the host check queue is what caps fine granularity. F2a then made the plan a value with one home and made the round's log name the plan it was given, which the runner's byte-identical copy of the default made worth doing on its own; the decision that the arms get their own research-side driver, with the 42 couplings measured behind it, is [recorded here](../decisions/implemented/2026-09-17-arms-get-their-own-driver.md). F2b then built that driver (`evals/ooo-execution/plan-driver.ts`, one slot at a time, with the ordered legal set taken from `BoardAdmission.candidates()` rather than re-derived), and **building it measured that the C arm has no mechanism**: a run can hold exactly one claim, so `slots: 4` runs with `slotsUsed: 1` and names the refusal instead of reporting a time. The operator's decision on that finding is [recorded here](../decisions/implemented/2026-09-18-declared-slot-budget.md) — a run declares its **slot budget** (`slots`, default 1), the ordered legal set is cut to it after ordering, and the status query reports the same cut — and it has landed (row `F2b-slot` below, with the measurement kept in "What F2b measured"). Before it, E landed: `src/integration/task-advisers.ts` is the port a source may speak to and the rules it cannot break, `selectableTasks` is the legal set the shared rules already decided (with `nextTask` as its head), and `BoardAdmission` takes optional advisers whose absence is the rule policy - nine obligations proven offline with nine registered mutants, no gate changed and no HA or MGR implementation wired. Before it, D14 gave the drivers a daemon to be clients of and made that client boundary pessimistic (a bounded, named call, and no client calling the endpoint it serves). + +Verification commands, run in the worktree that holds this branch, with the values they returned at this +revision (re-run them rather than trusting the numbers; the harness writes no log file): + +- `node --experimental-strip-types --test evals/ooo-execution/cost-model.test.ts` -> 8 pass, 0 fail, exit 0 (F1: the model's properties, including that a quality term cannot be added) +- `evals/ooo-execution/round-plan.test.ts` was retired with the round on 2026-09-18 ([decision](../decisions/implemented/2026-09-18-retire-the-round-instrument.md)): its property (a round's log names the plan it was given) had no product counterpart, because the log itself was the instrument's +- `node --experimental-strip-types --test evals/ooo-execution/narrow-dispatch.test.ts` -> 6 pass, 0 fail, exit 0 (F2b-slot: a declared budget is spent by claims in flight, a claimed task is not on offer, the budget cuts the start and not the ordered set, a claimed dependency still blocks, and zero or half a slot is refused) +- `node --experimental-strip-types --test evals/ooo-execution/board-slots.test.ts` -> 3 pass, 0 fail, exit 0 (F2b-slot: two claims held at once with the store's own `serialState`/`to` as the reason, the default licence being the head, and an acceptance freeing a dependent while the other slot is held) +- `node --experimental-strip-types --test evals/ooo-execution/plan-driver.test.ts` -> 8 pass, 0 fail, exit 0 (F2b/F2b-slot: plan order, a declared slot count reached with the claims overlapping in time, dependencies, a failed worker, the parent check, the comparison refusing a time verdict when a slot count was not reached, and - added by the retirement pass - a unit's check outstanding while an independent unit's worker runs, caught by the registered mutant `the-driver-awaits-each-unit-instead-of-the-batch`) +- `node --experimental-strip-types --test evals/ooo-execution/families.test.ts` -> 8 pass, 0 fail, exit 0 (F2c/F3: both task families, each with the coarse plan and the fine plan at one and two slots accepted, a wrong answer rejected by its own check and taking the composition with it, and a unit with no checks refused by name). ~30 s: every acceptance is a real candidate verification in a git worktree, which is the honest price of showing the family works before paying a model for it +- `node --experimental-strip-types --test --test-concurrency=4 "evals/ooo-execution/"*.test.ts` -> 93 pass, 0 fail, exit 0 (11 suites after the round's eight retired with it; the count is on the retirement pass, with the surviving suites unchanged) +- `npm run test:product` -> 1457 pass, 0 fail, exit 0. **A row that used to sit here said "one full run first reported a single failure under parallel load, then passed 1433/1433 on re-run; recorded as flaky, not fixed" - that label was wrong, and it hid a product defect.** The failure was `demoteMemory: demotes LTG memory to STG`, and it was a clock boundary: a memory written with `valid_from` a moment *after* the reading connection's `strftime('now')` read as not current (measured 2 of 3000 write-then-read rounds, stamp `…38.468Z` against `now` `…38.467Z`). Fixed by a named grace in `src/core/store/clock.ts`, pinned by `tests/core/store/current-value-window.test.ts` (6 cases) and 4 named mutants, decided in [the clock-grace record](../decisions/implemented/2026-09-18-clock-grace-window.md), recorded as [post-mortem 0004](../postmortem/0004-flaky-was-a-clock-boundary.md). The count moved 1446 -> 1457 with the fixed window and the cases added since +- `npm run mutation:teeth` -> 136 of 136 caught by the named test, 20 of 20 targets restored byte-identically, exit 0. Run on 2026-09-19 as four lanes, one sweep per tree, using `git worktree add --detach` on the same commit for three of them: a sweep is sequential *within* a tree because its mutants substitute into the same file, and parallel across trees, where each lane also gets the isolation property that no lane's suites can read another lane's mutant. Lanes: 42 of 42 (`ooo-board`, `task-coordinator`), 41 of 41 (`base`, `ooo-execution`'s 17, `task-semantics-interleavings`), 42 of 42 (thirteen small targets) and 11 of 11 (`plan-driver`) - the last serialised into its own lane because its suite has a 25 s case and does real candidate verification (~92 s per run, against ~2 s for the cheap suites). Three things the run itself taught, all fixed and pinned afterwards: the lock's `live` flag was never written on substitution (a multi-hunk edit failed as a whole and only the restore half was reapplied), so the field lied about a running sweep; `NODE_TEST_CONTEXT` inherited when a sweep is started from inside a `node --test` process made the nested runner exit 0 having run no test at all, which the harness reported as "the suite passed" and which turned every mutant of that target into a false "not caught"; and the refusal in `agent:verify` fired on `--dry-run` too, which made two of the verifier's own tests fail while a sweep held the tree (a dry run reads the plan and the route config, not the mutated file, so it is exempt now). The lane that reported a clean-run failure (`tools/agent-verify.ts`) had found the last of those three. A sweep is also refused while any lock is present, including one whose owner died, because a killed sweep leaves its mutant in the target (post-mortem 0003). Interruption note: two lane processes were killed by the console that launched them and were relaunched; the JSON each run writes at its end survived even when the buffered stdout summary was lost, so the lane results above were read from those files rather than from stdout. (the retirement pass added the driver's interleaving mutant; was 111 of 111 before this pass: `src/integration/task-semantics-interleavings.ts` gained three budget mutants and `evals/ooo-execution/plan-driver.ts` three for the per-unit checks, the canned worker and the parent composition). How these runs are scheduled (scoped during a change, full before a push, detached with a collected result) is a standing rule of the repository now, in [`skills/repo-development/SKILL.md`](../../skills/repo-development/SKILL.md) with its measured costs in [the decision](../decisions/implemented/2026-09-18-detached-long-checks.md) +- `npm run complexity:gate` -> exit 0, 18 methods above 15 unchanged from baseline. It caught the E pass's first version (adding the advisers option pushed `BoardAdmission`'s constructor to 17, so the options check moved into `admissionAdvice()`) rather than the threshold being raised; the slot pass added its option check the same way (`admissionSlots`), and the ordered-set/`publishReady` reads stayed under it. +- `npm run lint` (now over `src/ .pi/extensions/ claude-plugins/ workbuddy-plugin/ tests/ evals/ scripts/ tools/`) -> 0 findings, exit 0; `npm run check` -> exit 0. `npm run agent:verify` on a path under `evals/` still fails on the evaluation route's TAP rule for the skipped LoCoMo bridge - by decision, the rule stays and the reason names the skip. **A caveat found with the LSP this pass and not fixed**: `evals/**` is in no `tsconfig`, so `npm run check`/`check:tests` never type-check it and only the LSP sees those files - it reports 3 diagnostics there, all in files this pass did not touch: one type error each in `board-deliver.ts` and `board-judge.ts`, plus `tests/integration/ooo-evidence-drivers.test.ts:115` (a `{ pid: 0 }` fallback passed where a `ServerState` is required), and the 15 duplicate-key ones it used to report in `evals/ooo-execution/cycle.test.ts` went with that file when the round was retired. Recorded rather than repaired: the two remaining are outside this slice +- `node --experimental-strip-types --test --test-concurrency=4 tests/integration/ooo-ordinary-failure.test.ts tests/integration/ooo-managed-fence.test.ts tests/integration/ooo-read-paths-agree.test.ts tests/integration/ooo-round-query.test.ts tests/integration/ooo-task-tables.test.ts` -> 5, 3, 1, 2 and 4 pass, 0 fail, exit 0 + +How a row earns `proven`: it names a test that fails when the code satisfying it is broken. Where +such a mutation is registered, the mutant's name is given, because a test that cannot fail is a +description rather than a pin. Every mutant name below was read from +`tools/mutation-teeth.ts`, not recalled. Rows follow the design's own order, which is the work order. + +## A. Offline semantics (the design's first slice, already landed) + +| node | obligation | state | evidence | +| ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| A1 | The compiler returns a legal plan or a refusal naming task, field and obligation | proven | `tests/integration/task-semantics.test.ts` | +| A2 | The offline model dispatches without publishing or storing run state | proven | `tests/integration/task-semantics-model.test.ts` | +| A3 | The design's six discriminating cases are executable | proven | `tests/integration/task-semantics-cases.test.ts` | +| A4 | Field mapping catches budget-unit confusion, requires/deps confusion and silently dropped keys | proven | mutants `budget-inner-alias-is-dropped`, `over-maximum-budget-is-accepted`, `assumption-may-carry-a-dependency` | +| A5 | Every legal event interleaving of at most four units is enumerated, and every publication at every prefix is checked for its obligations, its inputs and its source | proven | `tests/integration/ooo-publication-invariants.test.ts` (18 cases) over `src/integration/task-semantics-interleavings.ts`. The enumeration is a merge of per-unit scripts, capped at four units and six events, and the checks read the derived view rather than a second model: a dispatch must rest on closed, accepted inputs, a completion (the accepted set closed over the same predicate) must have its verdict bound to the bytes, its bytes present, no cancellation, and inputs that are present and accepted. Both halves are pinned: each checker condition is deleted by a mutant and caught by the case that names it - `the-completion-does-not-bind-the-verdict-to-the-bytes`, `the-input-is-not-required-to-be-accepted`, `the-input-may-be-cancelled`, `the-completion-ignores-a-cancelled-unit`, `the-dispatch-does-not-require-a-closed-input`, `the-merge-enumerates-one-order` - and the clean run over the design's script sets reports nothing. The design's caveat stands and is quoted in the module: passing says this finite model satisfies the listed properties, not that any Agent program is correct | + +## B. Persistence (design: "进入持久化接入时另须证明") + +| node | obligation | state | evidence | +| ---- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| B1 | Two rounds with the same taskId do not collide | proven | `tests/integration/ooo-run-namespace.test.ts`; mutant `claim-is-not-scoped-to-its-run` | +| B2 | A retry does not deliver twice | proven | `tests/core/task-board-deliverable.test.ts`; mutants `stale-claim-may-deliver-again`, `deliverer-may-judge-its-own-work` | +| B3 | A transaction failure cannot write a verdict without its association | proven | `tests/integration/ooo-transition-atomicity.test.ts`; mutants `a-method-opens-its-own-transaction`, `nested-write-transaction-is-allowed`, `swallowed-failure-still-commits` | +| B4 | A retained board entry is not cleared by TTL | proven | `tests/core/task-board-retention.test.ts`; mutants `prune-ignores-retention`, `bounded-pin-never-expires` | +| B5 | The status query and dependency release call one predicate | proven | `tests/integration/ooo-acceptance-one-predicate.test.ts`; mutants `the-board-read-path-stops-calling-the-predicate`, `the-board-decides-acceptance-on-its-own` | +| B6 | A generic board write cannot move a managed round's own state, cannot take the live claim it holds, and cannot move an entry a run has adopted outside that run's transition | proven | the state half is structural and asserted (`tests/integration/ooo-managed-fence.test.ts`: the round's owner, attempt, acceptance and terminal reason stay its own facts, and a generic claim on its entry changes none of them). The claim it holds is not structural and is tooth-backed: `a-live-claim-can-be-taken-by-another-agent` - the store's CAS stops requiring the holder to be the claimant, and the same test's direct second reader then succeeds. The design's owed sentence - a generic write on a managed entry is applied _inside_ the coordinating transaction, and `judge`/`resolve` cannot go around the run's fence - is implemented and pinned by `tests/integration/ooo-managed-write.test.ts` (6 cases): a direct verb on an adopted entry is refused and the entry does not move, while an entry no run adopts takes the path it always did; a coordinated write lands the board transition and the run's fact in one transaction and a failure after the board write leaves neither; a cancelled run takes no further lifecycle writes; a write for the wrong run, or for an entry no run adopted, is refused before anything is written; and the daemon's own `claim` verb routes an adopted entry through the transition, writing the fact with its own store. Mutants: `a-managed-entry-ignores-the-coordinated-scope` (the store's guard), `the-daemon-verb-skips-the-coordinated-path` (the routing), `a-coordinated-write-skips-its-run-fact`, `a-cancelled-run-still-accepts-writes`, `a-coordinated-write-skips-the-binding-recheck`. What is still the design's step 3 rather than this row: nothing adopts entries into a run yet, so the research drivers' direct writes are not refused today - they will be, and are meant to be, once the runner and thin adapters work through the coordinator | +| B7 | JSONL export failure does not change the terminal state | not applicable | no JSONL export exists on this branch to fail | +| B8 | The field-mapping checks catch the three confusions | proven | A4: the offline compiler owns this check | +| B9 | A cancelled task is neither dispatched nor read as a closed input | proven | found by A5's enumeration rather than by reading: acceptance already refused a cancelled task (the one predicate reads `cancellations`), while the eligibility rule could not see the cancellation at all - `DispatchTask` carried no such fact - so a cancelled task stayed selectable as the next dispatch. `DispatchTask.cancelled` now carries the same recorded fact the predicate reads, and `nextTask` refuses it in `ready()` and in `valid()`; pinned by the cases "a cancelled task is not dispatched, and nothing reads one as a closed input" and "nextTask refuses a task marked cancelled, whatever else the caller set", and by mutants `a-cancelled-task-is-still-dispatched` and `the-dispatch-does-not-require-a-cancelled-input-to-be-closed`. Scope: the round has no per-task cancellation source yet (its cancel is run-level, and `recordedFacts` fills no `cancellations`), so nothing changes for it today - this closes the shared derivation's half of the design's 取消后的有效性 | + +## C. Recomputation (design: "重算检查须证明") + +| node | obligation | state | evidence | +| ---- | ----------------------------------------------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| C1 | Deleting every derived cache yields the same view | proven | `tests/integration/ooo-task-tables.test.ts`; mutant `derived-rebuild-is-a-no-op` | +| C2 | A lease crossing its boundary invalidates the old view | proven | `evals/ooo-execution/recovery.test.ts`; mutant `stale-claim-may-deliver-again` | +| C3 | Concurrent readers of one ready task: one legal claim | proven | `tests/integration/ooo-managed-fence.test.ts`. The test now reaches the board directly as well, because the round's own `live()` check refuses first and a mutant that broke only the store would have survived: a second reader on its own connection calls `claimTaskBoardEntry` and is refused, which is what makes "one legal claim" a property of the store rather than of one caller's discipline. Mutant `a-live-claim-can-be-taken-by-another-agent` (caught) | +| C4 | An unknown external result in a crash window is not guessed | proven | `tests/integration/ooo-external-window.test.ts`. `externalReady` refuses to make an event ready while a check for that task is outstanding ("managed check requires bound terminal evidence"), so the window cannot be closed by announcing it; after a restart the stored fact is still `external_ready = 0` with no artifact, and an invented event name is refused | + +## D. Lifecycle and integration pre-conditions (design: 事务参与与连接生命周期, 当前实现与接入前置条件) + +| node | obligation | state | evidence | +| ---- | ------------------------------------------------------------------------------------------------------------------------------------- | -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| D1 | Only the owner opens, migrates, checkpoints and closes a Store | proven | the round borrows its store and never closes it (D7), the read-only open is its own capability, and the daemon's close now refuses to drop accepted calls and is idempotent (D8). The narrow port and the borrowed status view carry the rest | +| D2 | A port exposes no raw connection, SQL, transaction control or `close()` | proven | `RoundQueryPort` and `TransactionPort` in `src/integration/ooo-board.ts`; `tests/integration/ooo-round-query.test.ts` | +| D3 | A write inside an open transition without a port is refused | proven | `tests/core/store-transaction-port.test.ts`; mutant `nested-write-transaction-is-allowed` | +| D4 | status's borrowed view migrates nothing, publishes nothing, initialises nothing | proven | `tests/integration/ooo-round-query.test.ts`; mutant `the-status-read-path-opens-the-rounds-store` (the mutant opens the writer's path, and the suite catches it) | +| D5 | The two read paths agree on the same facts and the same evaluation time | proven | `tests/integration/ooo-read-paths-agree.test.ts`. The comparison is no longer vacuous: the case accepts a task through the round's own host-verified path first, then requires both paths to report the same acceptance _and the same artifact bytes_. Mutant `the-offline-reader-decides-acceptance-on-its-own` (the borrowed port answers `{}` from its own rule; caught) | +| D6 | A read-only open is protected by the handle, not by `query_only` | proven | `tests/core/store-readonly-open.test.ts`; mutant `the-read-only-factory-opens-a-writable-handle` | +| D7 | `round` consumes an operations port and never calls `close()`; the outer host owns the Store | not applicable | **Retired with its subject.** The rule was proven on the round's ports and is now the shape of every host instead: the daemon owns the Store, a driver borrows a served store and closes nothing of its own, and the coordinated-write fence is the store's own state rather than a caller's discipline (D14). The round-side carriers (`OooRoundOperations`, `openRoundStore()`, `CycleOptions.operations`, the constructor-installed specs) were deleted with the round on 2026-09-18 ([decision](../decisions/implemented/2026-09-18-retire-the-round-instrument.md)) | +| D8 | Shutdown stops new work, lets the work in flight finish, then closes once | proven | `src/cli/service.ts`: `close()` refuses while calls are in flight, `drain()` is the awaited middle step, and `inFlight` counts what the daemon accepted. `tests/cli/service-drain.test.ts` holds an accepted call in flight (search opens the store, so the counter sees work a synchronous close could not), requires `drain(0)` to fail rather than return quietly, then closes once; `tests/cli/service.test.ts` 51/0 and `archive-shutdown` 3/0 are the regression evidence | +| D9 | A post-commit notification failure is recorded, not thrown | proven | `tests/integration/ooo-post-commit-notification.test.ts`: a unit whose post-commit `afterCommit` throws still submits as `accepted`, its artifact is the board's, and `lastNotificationFailure()` reports the message, while the same unit with a reachable notification reports nothing. Verified by mutation (hand-run, 2026-09-18): deleting the `try`/`catch` in `submit()` makes the notification escape and the case fails. The implementer was always the board (`src/integration/ooo-board.ts`, `lastNotificationFailure()`); the evals suite that used to pin it drove a real round only because the artifact envelope is the board's business, and the round was retired ([decision](../decisions/implemented/2026-09-18-retire-the-round-instrument.md)). The seam took three tries to find - `publishReady` is private and is called without a port at the commit, and `afterCommit` also runs on non-commit paths, so the failure is gated on a commit having landed | +| D10 | `runCycle`'s early-cancel branch is reachable, or it is dead code | not applicable | **Not applicable: the branch went with the round.** It was reachable and pinned twice (`evals/ooo-execution/early-cancel.test.ts` failed with "database is not open" when the round closed the store it borrowed; verified by hand, restored byte-identically), and the retirement deleted both the branch and its subject ([decision](../decisions/implemented/2026-09-18-retire-the-round-instrument.md)). The property it stood behind - nobody who borrows a store closes it - now holds by construction, because no client opens one: the daemon is the only writer and drivers borrow a served store (D14) | +| D11 | The run records live in the store's schema, and their typed writes join the store's transaction | proven | `task_run_manifest`, `task_run_tasks` and `task_run_facts` are created by `migrate()` like every other table (`tests/core/store/schema.test.ts` names them), and `NmgStoreBase` writes them through `registerTaskRun`, `freezeTaskRunTask` and `appendTaskRunFact`, each taking the same optional port the board writes take: outside a transition it opens one, inside it joins the caller's. What the cases prove (`tests/core/store/task-runs.test.ts`, 8/8): a board write and a run fact written through one port land together, and when the fact write fails the board row does not survive it; a retried fact is recorded once and keeps its first sequence; two runs in one store keep separate task and fact namespaces; a run's plan cannot be replaced and a frozen task cannot be redefined; an entry bound by two runs is refused rather than answered with one of them; the reads register nothing. Mutants: `the-run-fact-opens-its-own-transaction`, `a-retried-run-fact-is-appended-twice`, `a-second-plan-overwrites-the-frozen-one`, `a-frozen-task-is-replaced-by-a-different-definition`, `a-second-run-adopts-a-bound-entry`. One layer deliberately has no mutant of its own: the `UNIQUE (run_id, kind, task_id, attempt)` constraint is a storage-level backstop behind the explicit identity check, so a mutant that dropped only the constraint would survive - the check is the deciding layer and is the one mutated above | +| D12 | The binding of a logical task to its board entry is a run fact, and one routing rule decides what a binding makes managed | proven | `bindRunEntry` in `src/integration/task-coordinator.ts` records the binding as the run fact `entry-bound` (the name the D11 store tests already used), and it is the binding that the store's fence reads, so a bound entry's lifecycle write is refused unless it runs inside the run's coordinated transition. The op joins a caller's transaction when given a port, which is what lets a host create the entry and bind it in one transition: `tests/integration/ooo-managed-adopt.test.ts` (5 cases) checks that the board never holds an entry the run does not (a failure after both writes leaves neither), that a retry on the same task and attempt is not a second binding while another entry for that task and attempt is refused rather than silently dropped, and that every refusal names a stored fact (unregistered run, cancelled run, a task the run never froze, an entry that is not on the named channel, an entry another run already carries). `coordinatedEntryWrite` moves to the same module as the single routing rule, so the daemon's board verbs and any in-process writer decide "managed" in one place instead of each caller's belief about the entry; `src/cli/service.ts` no longer keeps its own copy. Mutants: `an-adopted-entry-takes-the-direct-path`, `a-binding-ignores-whether-the-task-was-frozen`, `a-binding-does-not-check-the-entry-exists`, `one-entry-is-bound-to-two-tasks`, `a-second-entry-rebinds-the-task`, `the-daemon-verb-skips-the-coordinated-path` (re-anchored to the daemon's claim handler, since the routing it mutates moved out of that file) | +| D13 | The run surface another process reaches is the daemon's: register, freeze, bind, cancel and status, and nothing else writes run facts | proven | `taskRun` in `src/cli/protocol.ts` over `registerRun` / `freezeRunPlan` / `bindRunEntry` / `cancelRun` / `taskRunStatus` / `createBoundEntry` in `src/integration/task-coordinator.ts`; `tests/cli/task-run-surface.test.ts` (11 cases) drives all of it through `service.invoke`, so what it exercises is the wire a second process uses, and it asserts `hello.methods` carries `taskRun` - the advertisement a client gates the `adopt` field on. Registration is the only transition that does not need a run to exist already (`coordinateRunWrite` refuses an unregistered run, which is what every later transition and every managed write rests on). A plan freeze is one transition: the request's array order becomes the positions, a dangling dependency and a self-dependency are refused by name while the plan is still only a proposal, and a batch the store refuses leaves the earlier tasks unfrozen - `the-plan-freezes-one-task-per-transaction`, `every-task-is-frozen-at-position-zero`, `the-plan-may-freeze-a-dangling-dependency`, `a-task-may-depend-on-itself` - and a cancelled run takes no further plan (`a-cancelled-run-takes-a-new-plan`). Adoption rides the transition that creates the entry (`createBoundEntry` takes the port), so a refusal leaves no entry behind (`the-entry-is-created-before-its-binding-is-checked`) and the wire cannot drop the request silently (`the-wire-drops-an-adoption-request`), which is the one failure mode the epoch rule in `design.md` is about. The binding fact records the channel that names its entry - that is what lets `status` resolve a binding without searching the board (`a-binding-does-not-record-its-channel`). `cancel` is the only writer of `run-cancelled`: before this row `src/` had none, so the state the fence and the dispatch derivation both read was reachable only from a test (the gap G4 named); a run-level cancellation carries the schema's empty task id (`a-run-cancellation-names-a-task`), a task-level one requires the task to have been frozen (`a-cancellation-ignores-whether-the-task-was-frozen`, `an-unknown-run-can-be-cancelled`), and cancelling twice is one fact because the fact's own identity is the duplicate key. `status` is a read and derives nothing: a run the store does not know stays unknown (`status-registers-the-run-it-cannot-find`), and ready/blocked/accepted stay with the shared pure function rather than with this view. The CLI exposes the operator half (`nmg run status`, `nmg run cancel`), which is also what the "every RPC method is a CLI command or intentionally RPC-only" gate asks for. One thing this pass corrected about itself: the first version's parser also computed a `position` per task, which `freezeRunPlan` immediately overwrote - the mutant that changed it survived, which is how the dead field was found, and it was deleted rather than kept | +| D14 | The evidence drivers reach the board through the daemon and open no database of their own | proven | `evals/ooo-execution/round-client.ts` resolves the daemon from the store's own lease and refuses by name when nothing serves it; `round-host.ts` serves an existing round store the way the product daemon does (`NmgService` + `serveHttp` + the store's lease) and is that store's only writer while it runs; `board-worker.ts`, `board-deliver.ts` and `board-judge.ts` now use the protocol's board verbs (`read`, `claim`, `release`, `deliver`, `judge`) and no longer import the store. `tests/integration/ooo-evidence-drivers.test.ts`: both end-to-end cases host the store and spawn the drivers as separate processes against the endpoint (spawned, not `spawnSync`, because the test process is what answers them); the managed case registers a run and freezes its plan through `taskRun`, adopts the entry in the same `put` that creates it, then runs the worker and the judge as separate processes - the run's log afterwards is exactly `entry-bound`, `board-claim`, `board-deliver`, `board-judge`, with the binding resolved to the entry the worker claimed and its verdict accepted. Two cases hold the boundary itself: a driver with no daemon refuses by name instead of falling back to the file, and no `board-*.ts` driver may mention the store or omit the round client. Mutants: `a-driver-falls-back-to-opening-the-store`, `the-round-client-does-not-require-a-daemon`, plus the drivers' two earlier ones (`worker-reads-only-one-reporter-shape`, `judge-may-judge-its-own-delivery`), re-pointed at the renamed case. Two further rules came out of building this, and both are about the same defect the first attempt hit: a process that serves an endpoint cannot answer a call to it while it is blocked, so "the answer will come" is not an assumption a client may make. `round-client.ts` therefore bounds every call (`ROUND_CALL_TIMEOUT_MS`, an opt-in bound on the thin client's `httpCall`, which by default still leaves the platform's own) and reports a timeout by naming the bound, the endpoint and the pid that serves it rather than the transport's; and it refuses when the lease's pid is the caller's own, directing an in-process host to `host.call(...)` (`round-host.ts`), which is the design's shape for an offline host. The test is now a client too: it starts the host as its own process (`[host] serving `, idle timeout as the backstop for a host whose owner died) instead of hosting in-process, and asks it to shut down rather than killing it, so what runs on the way out is the release path - a lease held by a process that is gone is a store nothing can serve. Cases: "a client refuses to call the endpoint its own process serves", "a call to a host that never answers gives up in seconds and names the reason" (raced against its own 5s deadline, so the check of the bound is itself bounded), "a host releases its lease when it stops, so the next host can take the store". Mutants: `the-round-client-calls-the-endpoint-it-serves`, `the-round-client-has-no-limit-on-how-long-it-waits`, and - new target, `src/cli/http-server.ts` - `the-serving-process-never-releases-its-lease`. Not covered by this row: `live-continuation.ts` still creates its own store and constructs `BoardAdmission` - the runner half of the same design sentence and the next obligation. `round-runner.ts` did the same and was retired with the round on 2026-09-18 ([decision](../decisions/implemented/2026-09-18-retire-the-round-instrument.md)) | + +## E. Optional HA/MGR integration (design: 复用 autodiff、HA 与 MGR) + +**Excluded from the earlier slices, by the design's own sentence** - the section says the reuse engines +were not to be marked as wired to the task runtime, and no gate or switch was to change. The later E +pass kept that sentence and added the seam plus its offline proofs (the table below): a suggestion +source may rank inside the legal set, and nothing in the shipped configuration consults one. + +1. an illegal best-scoring action is still refused; 2. a soft premise or `MemoryNode.requires` + gating cannot unlock a real task dependency; 3. closing or failing falls back to the rule policy; +2. state does not leak across session or branch; 5. a parameter or projection version change does + not reuse an old suggestion. + +**Landed since, as a seam and its offline proofs (this pass).** What exists now is the place a source +may speak to and the rules it cannot break - not an integration. `src/integration/task-advisers.ts` +holds the port: a source sees a bounded `AdviceProjection` and answers with suggestions that may only +_rank_ the legal set it was given. `selectableTasks` in `src/integration/ooo-execution.ts` is now that +set (the rules were not duplicated to expose it: `nextTask` returns its head, and the two cannot +disagree about what may be started). `BoardAdmission` takes optional `advisers`; with none - which is +the default and the only shipped configuration - `next()` answers exactly as the rule policy does, and +the design's sentence still holds: no HA or MGR implementation is wired, no gate changed, no switch +added. The four steps the design fixes are now code rather than prose: legal candidates computed by +shared semantics, an optional source ranking inside them, the shared policy ordering, and a re-check +at the claim. Each refusal names its task and its field, and `assumptions` are carried as provenance +and never read for legality, so a soft premise cannot make anything legal. + +| node | obligation | state | evidence | +| ---- | ------------------------------------------------------------------------------------------------ | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| E1 | An out-of-set action is refused even when it scores highest | proven | `tests/integration/ooo-advisers.test.ts` "a suggestion outside the legal set is refused, however high it scores" (the 1e9 score buys order inside the set, never a member); mutants `a-suggestion-outside-the-legal-set-is-scored`, `the-ordering-adds-a-task-to-the-set` (the ordering returns `[...legal, ...best.keys()]`) | +| E2 | A soft premise or `requires`-style gate cannot unlock a real dependency | proven | the same file, "a soft premise cannot unlock a dependency the shared rules refused": the fixture plan's `A` depends on `B`, and the suggestion for `A` carries `assumptions: ["requires: B", "hypothesis: B is satisfied"]` with a 1e9 score while `selectableTasks` returns `["B"]`; it is refused as out-of-set, and the same suggestion is adopted in the control case where `B`'s artifact is really accepted, so the refusal is about legality and not about the fixture. No line of the seam reads `assumptions`; the out-of-set refusal is the layer that enforces this, which is why E1's mutant is what fails this case too | +| E3 | A disabled or failing source falls back to the rule policy, and the run is told why | proven | "a disabled or failing source falls back to the rule policy, and says why": a `enabled: false` source and one that throws leave the order equal to the rule order, with two named fallbacks (`disabled`, `failed: no trained state is available`); mutants `a-disabled-source-is-asked-anyway`, `a-failing-source-takes-the-decision-with-it` | +| E4 | A score does not cross a session or a branch | proven | "a score from another session or branch is not reused": provenance from another session, and from another branch, is refused by name - the projection carries both identities and the module holds no state to remember one anyway; mutant `a-score-from-another-scope-is-reused` | +| E5 | A changed parameter or projection version does not reuse an old suggestion | proven | "a changed parameter or projection version makes an old score a new one": both mismatches are reported as re-scored with the differing input named, and neither orders the set; mutant `a-version-mismatch-still-counts-as-the-same-reading` | +| E6 | A score whose history is missing is never reported as a reproduction | proven | "a score that cannot name its own history is re-scored, never reported as a reproduction": a provenance without `observationOrder`/`initialState`, and one with a _different_ observation order, are both re-scored with the missing inputs named; mutant `a-score-with-no-recorded-history-counts-as-a-reproduction` | +| E7 | A suggestion cannot start an unmodelled action | proven | "an unmodelled action is refused rather than scored": `fuse` and `prepare` are refused on the `action` field - speculative execution and merging units into one acceptance are not rankings; mutant `an-unmodelled-action-is-scored` | +| E8 | An adopted ranking is re-checked at the claim point | proven | "an adopted ranking is re-checked where the write happens" plus `BoardAdmission.refuseStaleRanking`, which refuses the adopted task when the set it was adopted in no longer holds it; mutant `a-ranking-survives-into-the-claim` | +| E9 | The rule policy's answer is the head of the legal set, and the head rule is a legality condition | proven | `selectableTasks` extends `nextTask`'s own rules without duplicating them: "the round's own answer is the shared rule's answer, not an ordering's" asserts `nextTask(dispatch) === selectableTasks(dispatch)[0] === admission.next()` and that a head blocked by a stale input yields no candidate at all (that block is not an external wait license); mutants `a-head-blocked-by-a-stale-input-is-skipped`, `next-task-is-not-the-head-of-the-legal-set` | + +Two fixture findings from building this, recorded because both read like wrong answers when they recur. +The patch probe (`evals/ooo-execution/patch-cycle.test.ts`) froze the _live_ `src/integration/ooo-execution.ts` +as its rename target, so its own limits moved with that file: the artifact is the whole file, and a +dependent probe task carries it inside a snapshot, where the shared work contract bounds a dependency's +serialized bytes at 8 KB. The file had come within about a kilobyte of that bound, and adding the +selection function's documentation crossed it - a correct rename was then reported as `rejected`. +The probe now freezes a small copy (`evals/ooo-execution/fixtures/rename-baseline.ts`) instead of a +shared limit being raised, and the copy's comment states the shape its oracle requires. The second +finding is the same class in miniature: the probe's own `verify` created `src/integration/` in its temp +directory before writing the candidate, which silently threw once the target moved to the fixture +directory - `verifyCandidate` catches a throwing check and reports a rejection, so the failure looked +like a wrong patch. + +What the design does fix, and what this slice must not contradict: the reuse direction per owner +(autodiff's Tensor/UOp for cost or action scoring, HA for activation and context-retention scores, +MGR's traversal and what-if for sourced context or bounded hypotheses, the shared semantics layer +for the legal action set and acceptance), the boundary each one keeps, and the wiring order — legal +candidates from the shared semantics, then optional HA/MGR context or suggestions, then the shared +policy ranking _inside_ the legal set, then re-validation at claim and commit. A suggestion outside +the set is refused, not scored higher; a new task or dependency needs an explicit plan revision; and +association uses the existing run/task/attempt and AG projection identities, not a second task id or +board kind. + +## F. Later phase: the experiment arms + +The design's A–E arms and its stopping conditions are **landed, the paid pilot included**. The design +orders the offline layer first ("离线模型先覆盖不同粒度、依赖密度、共享上下文、事实命中率及验证成本;它只能 +发现逻辑错误和成本转折点,不能预测真实模型质量"), and the repository had none: this pass built it, +and the paid stage then ran on a family held out of it. + +| Step | State | Evidence | +| -------------------------------------------------------------------------- | -------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| F1: advisory offline cost model | landed | `evals/ooo-execution/cost-model.ts` (`--sweep`, derived graphs, self-checks) + `cost-model.test.ts` (12 cases) + [the sweep record](../experiments/execution/ooo-cost-model-2026-09-17.md) | +| F2a: the plan has one home, and the round's log names it | landed | the plan is one value with one home: the spec the driver runs (`PlanDriverSpec.plan`) and the run manifest the store freezes (D12). F2a's round-side carriers (`DEFAULT_ROUND_PLAN`, `CycleOptions.plan`, `openRoundStore(path, plan)`, `round-plan.test.ts` with its 6 cases) were retired with the round on 2026-09-18 ([decision](../decisions/implemented/2026-09-18-retire-the-round-instrument.md)); [the arms' driver decision](../decisions/implemented/2026-09-17-arms-get-their-own-driver.md) is what made the driver the home in the first place | +| F2b: a research-side driver for arbitrary legal plans | landed, one arm | `evals/ooo-execution/plan-driver.ts` (`runPlan`, `comparePlanSlots`, `verifyParent` path, refusal-naming CLI) + `plan-driver.test.ts` (15 cases) + 12 named mutants (`tools/mutation-teeth.ts`, target `evals/ooo-execution/plan-driver.ts`) + `BoardAdmission.candidates()` (the ordered legal set; `next()` is its head, with its own mutant) + [the decision](../decisions/implemented/2026-09-17-arms-get-their-own-driver.md) | +| F2c: pick the parent task from the sweep's turning point | landed | Two families of the shape the design's parent family needs - a frozen interface, three independent builders, one summary that depends on all three - as a spec pair each: `evals/ooo-execution/fixtures/report/` (the instrument's own) and `fixtures/pipeline/` (**held out**: written after the driver, and not the family anything was tuned against). The coarse spec is one unit over the whole task checked by every frozen test; the fine spec is four units with per-unit checks and no `join`, so the parent check composes all four. Sibling units are code-independent by construction (the summary takes the derived values as parameters), which is what lets a unit be verified before its siblings exist. Offline, with the instrument's own answers and a wrong one: `evals/ooo-execution/families.test.ts` (8 cases over both families) - both plans accept, a wrong answer is rejected by its own check and takes the composition with it, and a unit that declares no checks and has no file-wide list is refused by name. Driver support this needed: per-unit `checks` with the file-wide list as a fallback, a `canned` worker (the instrument's answer, so the family's acceptance is shown before any model is paid), and the parent composition fix recorded in the pilot's experiment record | +| F2b-slot: the C arm's mechanism (a run declares its slot budget) | landed | Rules: `selectableTasks(plan, slots)` / `startableTasks(plan, slots)` / `nextTask(plan, slots)` / `remainingSlots(plan, slots)` / `deriveStatus(units, facts, slots)` in `src/integration/ooo-execution.ts` + `src/integration/task-semantics.ts`; the ordered legal set is cut to `slots - claimed` **after** ordering, and the cut is what a claim licence may name. Admission: `BoardAdmissionOptions.slots` (default 1) + `handoffTarget` (required above 1, because the store queues a second un-directed actionable), `publishReady` offers every startable task one directed handoff and keeps it across a republish, and `claimableRow` checks `startable()`. Driver: `plan-driver.ts` declares the spec's count and names each claimant with one function. Cases: `board-slots.test.ts` (3), `narrow-dispatch.test.ts` (6, two new), `plan-driver.test.ts` (8, one replaced by the overlap case and one added by the retirement pass), `tests/integration/task-semantics.test.ts`. Mutants: `a-live-claim-does-not-block-selection` (re-anchored), `a-claimed-task-stays-on-offer`, `the-budget-is-not-cut-from-the-startable-set`, `half-a-slot-is-a-smaller-budget`, `the-status-query-ignores-the-declared-budget`, `the-licence-is-the-head-whatever-the-budget`, `a-second-slot-is-declared-without-a-target`, `only-the-heads-handoff-is-published`, `a-startable-handoff-is-retired-as-unselected`, `a-multi-slot-handoff-is-published-un-directed`, `the-driver-declares-one-slot-whatever-the-spec-says`, `the-driver-awaits-each-unit-instead-of-the-batch`. [Decision](../decisions/implemented/2026-09-18-declared-slot-budget.md) | +| F3: real-model pilot (A 3 / B 3 / C 2, current pi model, directional only) | landed | `evals/ooo-execution/pilot.ts` (`--live` required, `--report` to re-aggregate recorded runs with no model call, refuses a merge of two instruments, seeded arm order) + [the pilot record](../experiments/execution/ooo-arms-pilot-2026-09-18.md). Fixed: `deepseek/deepseek-v4-flash`, envelope limits `turns 6 / reads 3 / 120 s` for every arm, A 3 / B 3 / C 2, arms drawn from a seeded shuffle, the held-out `pipeline` family. Measured (8 runs, 23 model calls, every run complete, all 8 parent checks accepted): per run A 8.6 s / 11.2 k tokens, B 21.5 s / 31.5 k tokens, C 16.2 s / 29.9 k tokens, host 1.6 s / 3.4 s / 3.6 s. All three of F1's expectations held: B is 2.5× A in wall time and 2.8× in tokens (one slot buys nothing), C recovers part of it (0.76× B) and not the 2× a pure model-call overlap would give, and the host cost grows with candidates rather than slots. Wasted cost 0, human intervention 0. **Directional only**: n = 8, one model, one held-out family, and no quality difference was available to measure - every arm accepted everything | +| F4: execution fusion - legality, accounting and the driver policy | landed, live arm measured | Rules: `sharedSessionLegal`/`fusionSuccessors`/`fusionCandidates` in `src/integration/ooo-execution.ts` (the design's five conditions, one line each, composed with the board's candidate answer) + `tests/integration/ooo-fusion.test.ts` (12 cases) + 8 named mutants (target `src/integration/ooo-execution.ts`). Accounting: `fusionAccounting`/`fusionVerdict` + a fusion block in `cost-model.ts --sweep` - two lines kept apart, `unmeasured` until a run prices the session startup + `cost-model.test.ts` (12 cases) + 4 named mutants. Policy: `PlanDriverSpec.fusion` + `PlanRun.sessions` in `evals/ooo-execution/plan-driver.ts`, with the board's candidate set as the authority on staleness/cancellation/delivery/waits + `plan-driver.test.ts` (15 cases) + 11 named mutants. Scoped sweeps on this revision: `src/integration/ooo-execution.ts` 17 of 17 caught, `evals/ooo-execution/cost-model.ts` 4 of 4, `src/core/store/clock.ts` 4 of 4, `evals/ooo-execution/plan-driver.ts` 11 of 11, each restored byte-identically. A driver mutant that only restated `sharedSessionLegal`'s own rule was deleted rather than kept: the suite could not distinguish it from the shared predicate, which is what one home for that rule means. Both functions this slice pushed above the complexity limit (`sharedSessionLegal` 18, `runOneUnit` 16) were brought under it by extracting helpers, not by raising the threshold; `npm run lint` and `npm run check` are clean on this revision. **The live half landed.** `createPiSessionRunner` holds one Pi session and its tool surface across units (each unit re-points one mutable `UnitState` box; `patchSessionInput` is the one place a unit's prompt, snapshot and bounds are built, and `executePiPatch`/`executePiSnapshot` are thin callers of it), `PiRun` separates a unit's own `tokens`/`cacheRead`/`cacheWrite` from the session's `sessionTokens`, and `piSessionWorker` + `--session-runner` hold one runner per driver session. First paid D arm (2026-09-19, `deepseek-v4-flash`, 2 units, `--slots 1`, `turns: 6`, 3 reps per bound, only `fusion.unitsPerSession` differing): fused ran one session of two units and the control two sessions of one, quality parity in all six runs (every unit accepted), median 22 533 against 22 498 tokens and 11 048 against 12 948 ms - so no token saving yet (~1.9 s per run, ~15 % of the unfused wall, which prices the session-startup term at ~1.9 s instead of leaving it assumed) and per unit the second one cost ~8 % less while the first cost more: a chain's tool surface is the union of its units' capabilities because a session's surface is fixed at creation, so a unit can spend a turn on a tool that refuses by name. Also fixed here: `specFrom` had silently dropped a spec file's `fusion` block, so a spec asking for fusion ran as the control arm. | +| F5: speculation lifecycle - one declared fact, three outcomes | landed (offline); the paid E arm is unrun | `SpeculationAssumption` / `ResolvedPredicate` / `SpeculationCandidate` / `isBoundedSpeculation` / `speculationOutcome` in `src/integration/ooo-execution.ts`, beside the fusion conditions: the assumption is a declaration the summary binds to (it never discovers for itself that the guess was false), the first experiment's bounds are a predicate (exactly one pending fact, nothing prepared from the guess - a speculative successor or an irreversible operation each refuse it by name), and the outcome has three states rather than two - **true** publishes, **false** discards the candidate and returns `sessionReusable: false`, which is what makes "失效会话不能复用到真实路径" a rule the caller must honour instead of a note, and **unknown** (no reading, an unattested reading, or evidence about another version) waits without publishing. Asking for the outcome of a candidate that is not the bounded shape throws rather than folding a fourth state into the three. Cases: `tests/integration/ooo-speculation.test.ts` (9), the last of which joins this half to fusion's condition 5 - an invalidated branch is not a legal predecessor for the real path. Mutants: 5 (`speculation-guesses-several-facts-at-once`, `a-guess-with-no-evidence-publishes`, `an-unattested-reading-counts-as-evidence`, `evidence-about-another-version-is-the-same-fact`, `a-contradicted-guess-keeps-its-session`); the target's sweep is 22 of 22 caught. The E arm's instrument now exists (`evals/ooo-execution/speculation-pilot.ts`, registered) and ran once (2026-09-19, 6 paid units, ~43 k tokens): it decides the guessed fact, prepares the candidate, applies `speculationOutcome`, and verifies a published candidate with the unit's own frozen check. **No result is claimed**: the quality term was false in all four verified candidates, so by the design's own rule the latency and cost shape may not be reported as a gain. The search behind those failures is now closed and its first reading was wrong: `artifactEnvelope` builds two legitimate shapes (a patch, and a conclusion with `kind, conclusion, summary, evidence, citations`), and the instrument had fed every artifact to the patch reader. Eight of nine attempts answered with a conclusion, which this unit's check cannot pass and the board would refuse; the one patch attempt failed on a real mistake (`rows` for `lines`). The instrument now reads by kind, keeps every artifact, candidate tree and check output, and the run is archived. Measured outcome of the arm at this shape: the post-fact cost drops from ~6.2 s of work to 175 ms of verification when the fact holds, the false-fact case wastes 20 332 tokens, and the prepared candidate was publishable in 0 of 3 holding reps - so the cost is real, the gain is not, and the binding constraint is the candidate's admissibility | +### What F2b measured: a run can hold exactly one claim + + +F2b's declared job included running the fine plan in **N slots**. That cannot be done today, and the +measurement is the finding rather than an obstacle to it. `plan-driver.ts` asks the shared layer for +the ordered legal set and starts the head of it; with `slots: 4` on a plan whose first three units are +independent, the run reports `slotsUsed: 1` and `slotRefusal: "wanted 4 slots, the board allowed one: +no published handoff for this task"`. + +Three rules produce that, each in its own home: + +1. `selectableTasks` (`src/integration/ooo-execution.ts`) returns `[]` while **any** unaccepted task is + claimed — a rule that is already pinned by the named mutant `a-live-claim-does-not-block-selection`, + whose expected failing case is "with no fusion point the plan falls back to its declared order". +2. `publishReady` (`src/integration/ooo-board.ts`) publishes a handoff for the **selected** task only, + and `claimableRow` refuses anything that is not `next()` ("not selected by narrow dispatch") or has + no published handoff ("no published handoff for this task"). +3. The board serializes actionable entries per channel, which is the D14 pessimistic boundary + (`docs/decisions/implemented/2026-09-16-route-declines-shared-checks.md` and the D14 rows). + +Measured directly (probe, not inference): `candidates()` returns `["first","second"]` with nothing +claimed; after claiming `first` it returns `[]` and `claim("second")` raises "no published handoff for +this task"; after `first` is accepted, `candidates()` returns `["second"]` and the claim succeeds. +Sequential execution is unaffected: 4 units, 4 host checks, parent accepted. + +Two alternatives were measured and neither is the C arm the design asks for ("C 同一细计划、多槽 | 仅改变 +执行槽数/合法顺序", `docs/design/task-unit-semantics.md`): + +- **N concurrent runs on one store**: works (two `BoardAdmission` instances on one database each held a + claim at once), but it changes the channel, the plan and the serialization, so it is not "only the + slot count", and each run keeps its own board boundary. +- **Generation without a claim**: the design's own rule says the atomic claim is what grants execution + authority ("只有 daemon 的原子 claim/围栏校验能授予执行权"), so a candidate generated outside a fence + could not be submitted under one. + +So the C arm needed a governed decision: relax the per-run serialization for a declared slot count, or accept that F3 compares A and B only and record that concurrency is unreachable through this seam. **Decided 2026-09-18 and landed: the first, as a declared slot budget** ([the decision](../decisions/implemented/2026-09-18-declared-slot-budget.md)), with the store's channel serialization left alone because a directed entry is exempt from it. The commented rule above `selectableTasks` used to describe a **neighbour** rule ("a task whose earlier neighbour is still claimed blocks selection") that was narrower than the code's "any claim blocks selection"; the code and the comment now state the same budget rule. `comparePlanSlots` still refuses a time verdict whenever a requested slot count was not reached (`slotShortfalls` + `comparable: false`), so a run that reached fewer slots than it asked for cannot be reported as the C arm. + +The offline layer already fixed three expectations for F3: one slot buys nothing, so **B is predicted +worse than A on cost**; a chain or a two-unit refinement is predicted to lose at every granularity; +and a saving in the B arm would contradict the sweep, so the instrument would be checked before the +finding is believed. It fixes no model, budget or repetition, and it spends nothing: the record's +caveat is that the model has no quality term at all, and its test fails if one is ever added. + +Nothing in this ledger may be reported as a result from a paid arm **beyond what the pilot recorded**. The +design's rule stands: a speedup may not be claimed without equal parent quality. The pilot's numbers are +in [its own record](../experiments/execution/ooo-arms-pilot-2026-09-18.md), including what it cannot say +(n = 8, one model, one family, no quality difference available to measure), and they are quoted here +with that caveat rather than as a finding. + +## What the D7 attempt found + +Both findings were fixed in the D7 commit; the record stays because the first one was a real defect in +the round's ownership rather than an accident of editing. + +1. The round handed the store an **object it kept filling in**: `specs` was passed to the + constructor and then assigned into task by task, so the store's behaviour depended on sharing + that mutable object with its caller. A host-opened store breaks that path, which is why the + first conversion produced 33 failures with `patch task has no host spec`. The port needs an + explicit install point - an `installPatchTask(id, spec)` on the owner's object - and the round + must call it instead of writing into a record it also handed over. +2. `evals/ooo-execution/round-runner.ts` (retired with the round on 2026-09-18, [decision](../decisions/implemented/2026-09-18-retire-the-round-instrument.md)) had its own `openRoundStore` that **validates** a store + rather than creating one. With the round no longer creating the store, the host has to create + it, so the two same-named functions have to be told apart at the call site (the src factory + aliased, the local validator left alone). + +What the pass did verify before reverting: with both fixes the evals suite went from 33 failures to +4, and the remaining 4 were the `round-runner` name collision. The revert is not a verdict on the +refactor; it is my own rule for this node, declared before starting it: an unconverged conversion +of `evals/**` would leave the round's only real regression suite broken with no gate to notice. +Next attempt should carry the install point in the same change, and run the evals suite as the first +thing after the conversion, on a branch that can be thrown away. + +## What the B6 attempt found + +The design asks that a generic write request on a managed entry go through the same coordinating +transaction ("受管理条目的通用写请求也必须经过同一协调事务,禁止直接 judge/resolve 绕开运行围栏"). +Reading every caller before writing the guard is what showed that the literal version of it is not a +guard but a slice of the integration still outstanding: + +- The board's own verbs **are** how an ordinary handoff is worked, which is the design's own product + entry. The generic lifecycle writes on a run's published handoffs are made by + `evals/ooo-execution/board-worker.ts:74` and `:138`, `board-deliver.ts:67`, `board-judge.ts:78`, + `live-continuation.ts:527`/`:603`/`:748`, and by the daemon's `claim`, `deliver`, `judge` and + `resolve` verbs in `src/cli/service.ts:2854`/`:2894`/`:2898`/`:2952`. Refusing those writes on a + managed entry would refuse the product path the same design mandates, and the Pi tool that used to + bypass them is gone (G5). +- What makes the write "the same transaction" today is the store's transaction port: object identity + checked by `withPort()`, so a foreign port, a store with no open transition, and a caller that + opens its own boundary are all distinguishable. Wiring the run's coordinator _into_ the daemon's + board verbs is the shared-store integration the design's 当前实现 section already names as the + target shape that is not in place, not a check that can be added to one method. +- The harmful direction is closed without it, which is why B6 can still carry teeth: a generic claim + cannot take a live claim (`a-live-claim-can-be-taken-by-another-agent`), acceptance is the board's + verdict bound to _this_ attempt's artifact digest (`verdict-lookup-not-bound-to-the-artifact`), a + withdrawn acceptance withdraws the release of the dependent + (`round-releases-dependents-on-delivered-bytes`), and an entry retired while verification was + awaiting leaves the attempt stale rather than committed + (`the-commit-trusts-a-claim-the-board-retired`, pinned by the window case in + `tests/integration/ooo-managed-fence.test.ts`). +- Two guards, one tooth: an assertion satisfied by _either_ of two independent refusals cannot be + pinned by breaking one of them. The C3 case had that shape - the round's `live()` check refuses + before the store's CAS is reached - so the test was extended to reach the board directly. The + commit-window case had the mirror-image problem: `submit()` checks liveness before verification and + `commitArtifact()` re-checks it after, so the retirement has to happen _inside_ the verification + window for the second check to be the one that decides. + +The harness note from that pass is closed: a failed clean run now reports the observed failure lines +(`observedFailures` in `tools/mutation-teeth.ts`, main's #63 - this branch takes it by merging main, +not by a change of its own), so the verdict is diagnosable instead of repeated. The one-in-four flake +the note also recorded was never diagnosed here; the Windows temp-tree removal and shadow-lock flakes +main repaired in #63/#64 are the nearest known causes, and they arrive in the same merge. + +**Closed since (same branch, later pass).** Everything the bullet list above left open is now built +rather than described, and two of its statements were corrected by doing it: the store gained +`coordinateRunWrite` and a guard that refuses a lifecycle verb on a managed entry reached outside +that scope (so "a check that cannot be added to one method" was wrong - it is one helper called from +the lifecycle entry points, and the store can decide it because D11 put the binding in the same +schema); and the daemon's verbs in `src/cli/service.ts` route a managed entry through +`coordinatedBoardWrite` (`src/integration/task-coordinator.ts`), one indexed lookup per verb, with +unadopted entries paying nothing more. The driver callers named above still write directly - nothing +adopts entries into a run yet, so nothing about them changes until the runner and thin adapters work +through the coordinator. **Part of it landed since**: the surface another process reaches exists (D13 - `taskRun` and the `adopt` field on a board put), so a runner can register, freeze, bind, cancel and read a run without a database of its own, and the drivers are clients of it (D14): `board-worker.ts`, `board-deliver.ts` and `board-judge.ts` reach the board through the serving daemon, refuse by name when nothing serves the store, and no longer import it, which the driver smoke test holds both behaviourally and structurally. The client boundary is pessimistic in two ways that matter to the runner as much as to a driver: a call is bounded and names what it waited for, and a process that serves a store calls the round entry in-process instead of over HTTP, because a blocked host cannot answer itself (a `round-host.ts` started as its own process is what a client that wants the wire needs). What remains of step 3 is the runner: `live-continuation.ts` still creates its own store and builds `BoardAdmission`, so the research round's own plan, candidates and probe state still live in that private schema rather than in the run namespace the design chose - "runner ... 不打开可写数据库,也不各自构造 `BoardAdmission`" is the obligation that row will carry. (`round-runner.ts` did the same and was retired with the round on 2026-09-18, [decision](../decisions/implemented/2026-09-18-retire-the-round-instrument.md).) + +## G. The product entry: collaboration absorbs OoO (design line 216, added after this ledger) + +This section answers the question the earlier sessions kept circling, and it rules out the cheap +answer I would otherwise have reached for. OoO is an execution capability _inside_ collaboration, +absorbed by the board, the shared task semantics, the execution lifecycle and the acceptance +facilities; an Agent uses it through an ordinary task handoff and never chooses or calls an +dedicated OoO tool. Explicitly forbidden: a new `ooo` tool, a dedicated channel, a second task +body, or a `board.runRound` wrapper - each of which would re-create a separate plan, state and +lifecycle. Ordinary board text does not become an executable task by itself: only a handoff that +enters through an existing operation, with a stated contract and execution authority, participates. + +| node | obligation | state | evidence | +| ---- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| G1 | An ordinary board handoff reaches semantics, execution and acceptance with no `ooo_round` | proven | `BoardAdmission.next()` now answers with `nextTask(dispatchTasks(compileTaskUnits({plan, specs}).units, facts))`, so the coordination path consults the compiler it used to ignore (`src/integration/ooo-board.ts:812` and `:822`, and the hand-rolled candidate mapping is gone from `src/`: `rg -c schedulable src/` is 0 (the config's own anchor for that rule is what used to match, and it moved with the rule)). `tests/integration/ooo-ordinary-handoff.test.ts` walks claim -> deliver -> independent verdict -> accepted with the board's own verbs; the only mention of `ooo_round` in it is the comment saying it is untouched. Verified by mutation by hand: dropping `task.accepted` from `nextTask`'s validity check fails it | +| G2 | A real artifact is accepted on that path, and accepting it releases the dependent | proven | the same test: B's artifact is accepted by the board's own verdict, and `deriveStatus(units, facts).ready` goes from `["B"]` to `["A"]` over the recorded facts while `next()` gives the same two answers, so the two views are held together rather than assumed to agree. The same mutation (validity no longer requiring acceptance) fails the test at its first assertion: `assert.equal(gate.next(), "B")` at tests/integration/ooo-ordinary-handoff.test.ts:68 returns null (AssertionError: null !== 'B'), so the `ready` assertion later in the test is never evaluated. Measured, not recalled: the mutation was run and restored byte-identically while correcting this sentence. The rule this row's release depends on - an artifact delivered without an accepted verdict is not selectable - moved with the wiring, and its tooth moved with it: `selection-ignores-a-withdrawn-acceptance` is now anchored on `nextTask`'s `selectable` line in `src/integration/ooo-execution.ts` and is a configured target (full harness: 7 of 7 targets, 31 of 31 caught, 7 of 7 restored byte-identically, exit 0). | +| G3 | Failure, cancellation, the ordered fallback, and the accepted prefix | proven | `tests/integration/ooo-ordinary-failure.test.ts` (5/5): a refused deliverable neither accepts nor releases and is not re-selected on its own until the coordinator reopens it; a later refusal does not withdraw the prefix that was already accepted; a cancellation names the lease it revoked and every claim behind it is refused; with no fusion point the plan falls back to its declared order; a split that maps no parent obligation is refused by name with its location. Selection is tooth-backed (`selection-ignores-a-withdrawn-acceptance`, `a-live-claim-does-not-block-selection`, `cancellation-does-not-stop-a-claim`) and so is the prefix: mutant `reopening-one-task-clears-every-acceptance` (reopen fences every accepted task instead of the dependents of the one reopened; caught by the fifth case, which then finds only A accepted where it requires A and B). The design's fusion clause ("融合中途失败只保留已接受前缀") is **not applicable on this path**: fusion is declared unmodelled and refused by name (`UNMODELLED_ACTIONS`), so nothing on the ordinary path fuses and nothing can fail midway; the prefix property itself is what the fifth case asserts. The offline model covers fused-midway failure for its own enumerated plans under row A3 | +| G4 | The old entry's query and cancel are reachable from existing facilities | proven | query: `tests/integration/ooo-round-query.test.ts` "the query port reads a round without migrating, publishing or exposing a write", tooth-backed by D4's `the-status-read-path-opens-the-rounds-store`. cancel: the product's own lifecycle operation, tooth-backed at integration level by the durability case in the same file ("the terminal decision outlives the host that made it, and still refuses new work": a second host that opens the store reads the same terminal reason, a new claim is refused, the revoked claim is gone and the attempt is fenced). Mutant `cancelling-a-round-forgets-its-reason` (cancel stops persisting the reason; caught by that case - the cross-process suite that used to name it too (`evals/ooo-execution/cancellation.test.ts`) was retired with the round on 2026-09-18 ([decision](../decisions/implemented/2026-09-18-retire-the-round-instrument.md)), which leaves the integration case and the mutant as this row's evidence). Neither file mentions `ooo_round` or the extension: 0 hits | +| G5 | `ooo_round` leaves the product tool directory, with the tool directory, adapter, docs and hidden-features registry updated together | proven | Removed: `.pi/extensions/nmg/ooo-round.ts`, its import and its `registerTool` block in `.pi/extensions/nmg/index.ts` (0 remaining mentions of `ooo_round` in the tool directory), and `tests/extensions/nmg/ooo-round.test.ts`; the three tool lists in `tests/extensions/nmg/index.test.ts` no longer name it. What remains is the ordinary path: `nmg_board` for handoffs, the shared layer for selection. `ooo-execution.ts` stays in the extension because it is not the tool - `evals/ooo-execution/live-{cycle,patch,pi}.ts` import it as the eval-side Pi execution adapter, and G7 runs through them. `npm run ooo:round` also stays: it is the research CLI under `evals/`, not a product surface. Docs record the removal in the bootstrap design's S3 row and in the design's own `ooo_round` sentence. The hidden-features registry is deliberately unchanged: its OoO rows describe the research CLI and the live eval entries, whose gates did not change, and the Pi tool was registered by default, so it never had a hidden-feature row to remove. | +| G6 | The task view is a projection of existing facts: no second editable task truth, and a model cannot confirm `accepted`, change a lease or overwrite a cancellation by patching state | proven | the state side is covered (B5, B6, D3, D4). The consequences the design's information table has are now asserted rather than described: the frozen plan has one owner and a second, different plan is refused (`tests/integration/ooo-task-tables.test.ts`, mutant `a-second-plan-silently-adopts-the-run`), a terminal decision cannot be overwritten by a later patch (mutant `a-second-cancellation-overwrites-the-first-decision`), and acceptance is only ever the host-verified artifact plus the board's verdict about that digest (`verdict-lookup-not-bound-to-the-artifact`, `round-releases-dependents-on-delivered-bytes`). What stays prose is the table's owner assignment itself - which layer owns which information class is a design statement, not something a test can read off one column | +| G7 | One real handoff at a legal boundary, with another Agent session continuing the same parent task from the view plus retrievable evidence, judged by the fixed parent check | proven | `docs/experiments/execution/ooo-real-continuation-2026-09-14.md`, with the run's raw records beside it (`ooo-real-continuation-2026-09-14-g7-run.jsonl`, `-merge-retries.jsonl`) and the executable check `evals/ooo-execution/live-continuation.ts`. Five structurally identical, content-distinct tasks; every role a separate process; the parent is the channel and the continuation a second handoff in it; part 2 retrieves the part-1 artifact from the view and verifies it against the digest the store recorded before it may continue; the fixed parent check (6/8/7/6/7 frozen cases) decides the boundary and the parent. Model `deepseek/deepseek-v4-flash`; one clean pass over the five tasks = 9 model calls, 80 081 tokens, 84 395 ms, plus a four-attempt sample of `merge`. Result: four of five tasks accepted end to end; `merge`'s continuation failed 2 of 4 attempts with an unparseable patch artifact, and its parent stage then recorded **no verdict** rather than judging the part-1 artifact. The document states the cost and the limits, including that failed attempts' token counts are not recorded A follow-up comparison (`docs/experiments/execution/ooo-real-continuation-comparison-2026-09-14.md`) re-reads every stage record, classifies each failure by cause, and finds that 13 of the 20 continuation failures were the harness's own defects rather than the model's, while the stable first stage failed once in 35 samples. | + +Two consequences for the nodes above it: + +- D7 (the round borrowing its store) stays worth doing and is now also the node that would let a + non-round host run the same operations, which is what G1 needs. It does not, by itself, move any G + node: G1 is about which entry point exists, not about who closes a connection. +- The design's order is now: the B/C/D proofs, then the ordinary collaboration path (G1-G4), then the + one real continuation (G7), and only then the A-E arms. The A-E arms are unchanged; the + continuation check moved in front of them. + +## Blocked, or not applicable, and why + +- B7 has nothing to fail: no JSONL export exists on this branch. +- D7 is neither blocked nor deferred: the port is in place and its row records it (13 operations and + no `close`, `CycleOptions.operations` required, the specs carried by `installPatchTask`). What + survives the first attempt is why it took two: the conversion touches `evals/**`, a tree neither + `tsc` nor the product suite covers, so a silent breakage there would fail no gate. A later pass + over the same tree runs the evals suites as its first act, not its last - that rule is what turned + the first attempt into a revert instead of a branch with a broken regression suite. +- The design document is tracked and current here: this branch carries the 369-line revision, + `sha256 a7cebe00f503495d…` - the shared checkout's newer revision landed verbatim (`a061b2b5`, + squashed into `0d3dc09d`) - and `main` still carries the 272-line one (`1f2f22f2`, PR #54). Every + sentence these rows quote from the design was located by its text in that revision, not from a + remembered line number, and the re-check this pass caught one: the B6 obligation carried the wrong + character (`栅` for `围`) and could not be found in the file at all. Corrected. The rows therefore + do not depend on which revision lands, as long as the quoted sentences stay word-for-word. + +## What is left, and what it needs decided + +Everything in A-G is proven, excluded by the design's own sentence (E) or has nothing to fail (B7). +The last outstanding row was **B6's owed half** - the design's migration step 3, "最后让研究 runner 和 +薄适配经 daemon 调用", read with the truth table's managed-entry row - and it is built: + +- the groundwork (D11): the run's tables are the store schema's business, the typed writes are the + Store layer's, and the coordinator composes them through the port instead of inheriting a + connection - the design's "共享协调器不继承 Store 来获得另一条连接". The derived view stays derived, so + no table here competes with the board as a second truth; +- the fence: `NmgStore.coordinateRunWrite` opens the one transaction that authorizes writes for a + run, and the lifecycle entry points (`claim`, `release`, `deliver`, `judge`, `resolve`, `veto`, + `acknowledge`, both retention calls) call `requireManagedWriteScope` first, so a managed entry's + verb reached outside that scope is refused by name instead of applied beside the run. The scope is + the store's own state, never the caller's claim, in the same spirit as the transaction port; +- the coordinator's write path (`src/integration/task-coordinator.ts`): re-read the run's own state + (registered, not cancelled) and the binding _inside_ the transition, apply the board verb, append + the run's transition fact, all on one boundary - a failure after the board write leaves neither; +- the daemon routes it: the mutating `taskBoard` verbs in `src/cli/service.ts` send a managed entry + through the coordinator and leave an unadopted entry on the path it always took; +- the binding (D12): a logical task is bound to the board entry that carries it by a run fact, so + "managed" is decided by what the store holds and never by a caller's belief about an entry, and + the binding can be written in the same transition that creates the entry. + +With that, the design's _offline_ slice is complete as the design words it: the semantic judgement +(the compiler, the field-mapping checks, the six discriminating cases) plus the finite-model +acceptance - enumerate the legal event interleavings of at most four units and check every +publication's obligations, inputs and source (A5). That enumeration earned its place on its first +run: it turned up a task-level cancellation the derived view could not see (B9), which acceptance +had always refused but eligibility would still hand out. What it does not do is prove any Agent +program correct, which is the design's own caveat and is quoted in the module. + +What that does **not** yet do: nothing adopts an entry into a run outside the tests, so an entry +nobody adopts keeps taking the path it always took, and a driver that never adopts is not refused - +by design, because the fence refuses a *managed* entry's verb reached outside its run's scope rather +than making adoption mandatory. The wiring this paragraph used to call missing has landed since, in +**D13/D14**: the run surface a different process reaches is the daemon's (`taskRun` over register, +freeze, bind, cancel and status, with `tests/cli/task-run-surface.test.ts` driving it through +`service.invoke`, which is the wire a second process uses), and the evidence drivers reach the board +through the daemon's protocol verbs and open no database of their own, with `round-client` refusing +by name when nothing serves the store. The public surface still has no coordinated-write verb, and +that is the fence rather than a gap: the daemon's board protocol is the coordination surface, and a +second door for the same transition would be the bypass this fence exists to remove. What remains, +then, is not a missing surface but a caller - a runner that decides to adopt. + +Two debts that pass would carry, so that they are not discovered late: **Retired with the round.** `BoardAdmission` kept its own copy of the run tables for the round's private store (the design's "旧私有轮次只读保留作研究证据, +不与新运行双写" makes them coexisting, not duplicated), and the archived evidence readers have to move +with the retargeting or refuse an old schema by name instead of reading it as empty. The first of the two is discharged: the round and its private store were retired on 2026-09-18 ([decision](../decisions/implemented/2026-09-18-retire-the-round-instrument.md)). `evals/**` is the +tree with the weakest gates, which is why a pass touching it runs the evals suites first - weaker, not +absent, since 2026-09-16 (#68): the widened `lint` scans `evals/`, `tests/`, `scripts/` and `tools/` +and is blocking, and the tests surface is type-checked by the advisory `check:tests`. What no gate +covers is the `ooo-execution` drivers' behaviour - they need model calls - which is what running the +suites first is for. + +The design's **D arm (fusion) and E arm (budgeted speculation)** are the part of the arm programme that is still unrun, and that is not a code gap: they need real model calls and a budget decision, so no offline change can move them. **F** is not outstanding - the cost model, the plan's single home, the arms' driver, its slot budget, the two families and the paid pilot all landed and are recorded above, and the driver now also carries the interleaving the retired round was the only end-to-end carrier of. diff --git a/docs/design/task-unit-semantics.md b/docs/design/task-unit-semantics.md index 2cf162a4..f9f83e48 100644 --- a/docs/design/task-unit-semantics.md +++ b/docs/design/task-unit-semantics.md @@ -1,7 +1,7 @@ # 可拆分、可融合的 Agent 任务语义 **Status:** draft -**Related:** [受限 OoO](ooo-execution-bootstrap.md)、[选择与验收条件](../decisions/proposed/2026-09-13-task-unit-semantics.zh-CN.md) +**Related:** [受限 OoO](ooo-execution-bootstrap.md)、[选择与验收条件](../decisions/implemented/2026-09-13-task-unit-semantics.zh-CN.md) ## 问题与目标 @@ -11,17 +11,25 @@ 用户已确定的架构约束是:协作能力和工具语义位于共享层,harness 经薄适配接入,黑板是协作载体。本文的拆分规则、融合机制和推测方案仍是待检验设计;它不启用功能、不放宽现行受限 OoO,也不宣称已有通用多执行者运行时。 +### 主验证问题与范围 + +主问题是:同一个父任务采用比 Agent 更细的合法单元后,能否暴露粗粒度任务遮住的并发机会,并通过融合抵消拆分带来的交接、上下文与验证成本。确定并发与融合获得证据后,再研究有限推测是否带来额外收益。拆分不以执行者失败为唯一触发条件;一个 Agent 能完成的任务,也可能因可并发而值得拆分。 + +优先复用黑板的交接、认领、交付、判定、租约与 attempt,以及既有任务契约中的输入、访问范围和验证配置。对宿主已明确依赖、隔离和组合检查的固定工作流,这些语义可以足够;只有具体场景无法可靠判断依赖、效果或父义务覆盖时,才扩展对应 owner。本文的概念清单不要求先实现通用任务语言或调度平台。 + +跨会话接续用于检验交接边界;识别已满足义务用于消除冗余;事务与恢复用于支撑实际接入。这些工作保留各自价值,但不能代替粒度、并发与融合的主对照,也不要求先完成所有辅助实验或全量迁移。被执行路径仍须满足权限、隔离、验收与生命周期约束。 + ## CPU OoO 与 ILP 的出发点 用户明确指出,这些约束来自沿 CPU 乱序执行与 ILP(指令级并行)的思路继续推演:先有合法指令的含义,再讨论如何调度执行资源;执行安排可以改变,提交边界仍需保留。任务粒度小于 Agent 粒度及运行时融合,是这条思路在协作系统中的延伸。 -| CPU 中的角色 | 本设计中的对应问题 | -| --- | --- | -| 指令的可观察语义 | Task IR 的输入、效果与接受义务 | +| CPU 中的角色 | 本设计中的对应问题 | +| ------------------------- | ------------------------------------------- | +| 指令的可观察语义 | Task IR 的输入、效果与接受义务 | | 指令/微操作与执行单元分离 | 逻辑任务与 Agent 会话分离;不是逐项硬件同构 | -| 数据相关与就绪选择 | 版本化产物、事实依赖及受约束的资源效果 | -| 执行完成与 retire 分离 | 生成、验证、发布分别处理 | -| 分支恢复 | 候选作废、尝试围栏与分支上下文重建 | +| 数据相关与就绪选择 | 版本化产物、事实依赖及受约束的资源效果 | +| 执行完成与 retire 分离 | 生成、验证、发布分别处理 | +| 分支恢复 | 候选作废、尝试围栏与分支上下文重建 | [BOOM 的 ROB 文档](https://docs.boom-core.org/en/latest/sections/reorder-buffer.html)具体展示了执行完成后等待顺序提交,以及异常时恢复已提交的重命名状态。这里借鉴的是保留可观察状态的边界,不把 blackboard 本身等同于 ROB,也不把会话复用等同于某种 CPU 指令融合实现。 @@ -37,18 +45,40 @@ CPU 类比还暴露出两个研究缺口:自然语言任务没有现成 ISA - [Dask 图优化](https://docs.dask.org/en/latest/optimize.html)通过融合减少任务间通信。这里把融合视为成本优化,保留每个 Agent 任务的验收边界。 - [乐观并发检查](https://learn.microsoft.com/en-us/dotnet/framework/data/adonet/optimistic-concurrency)在更新时核对原始状态。这里借鉴提交点验证;版本匹配只证明输入未漂移,不证明模型结论正确。 +## 长任务的连续性与任务视图 + +[SKILL.state v3](https://arxiv.org/html/2608.26263v3)以固定技能说明、结构化执行状态和最新观察构造每步输入,由运行时验证模型提出的状态更新,跨步不携带中间推理。其局限包括状态未保留后来才显现价值的信息,以及尚未验证多 Agent 并发更新。本文借鉴显式状态承载任务连续性的方向,不把单 Agent 结果当作共享协作或 OoO 的收益证据。 + +[The Horizon Gap v1](https://arxiv.org/html/2608.06663v1)区分任务跨度、上下文容量与跨任务记忆,并将规划、记忆、执行恢复和评估放在同一生命周期内。本文据此把父任务完成与恢复能力作为检验目标,局部步骤通过或过程评分只作诊断;增加拆分、记忆或调度设施本身不代表长任务能力提升。 + +任务连续性的目标是:在显式交接边界,具备所需能力与权限的另一个 Agent,能够凭任务契约、当前事实和可访问证据继续工作,不依赖原 Agent 未交付的对话或内部推理。它是输入闭合与产物可交接的端到端检验,不要求任意推理步骤都可中断恢复,也不要求每个单元强制换 Agent。 + +| 信息 | 所属职责与进入下一步的方式 | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| 目标、允许动作与完成条件 | Skill 与既有任务契约提供程序说明和约束,不另建技能状态语言 | +| 认领、交付、判定、取消等执行事实 | 黑板与共享执行设施提供权威观察,按本文运行记录规则派生当前状态 | +| 当前相关知识、证据及待确认假设 | 各 Agent 的私有 [session AG](session-active-graph-runtime-design.md)组织获准可见的上下文;AG 不成为共享任务状态库 | +| 跨任务可复用经验与知识 | NMG 长期记忆按来源与有效性检索,不自动继承为当前任务的已验收事实 | +| 模型下一步输入 | 共享层组合契约、执行事实视图、相关证据及最新观察,薄适配层交给模型 | + +任务视图是这些既有信息的按需投影,不另存一份可编辑的任务真相,也不等同于 Task IR 的依赖分析视图。模型可以提出产物、观察或假设,不能通过状态补丁自行确认 accepted、改变租约或覆盖取消事实。结构校验与协议校验各自约束形状和状态转移,领域验收仍检查产物;开发任务沿用其 RCP/检查设施,不新增独立 OoO 验证器。 + +不携带完整历史,不等于删除恢复和审计所需的证据。持久事实及保留期限由下文运行记录规则拥有;AG 投影可重建,相关原始观察可按授权检索。检索或重建缺少必要依据时明确报告缺口,不能把未查到当作从未发生。固定字段数也不保证字段内容有界,本文不承诺任意任务的上下文恒定大小或无损压缩。 + +这一目标为换人、重试、融合与调度提供共同基础:会话复用可以降低成本,正确性不能依赖复用才存在的隐含信息。它服务于现有黑板协作路径,不增加独立工具或启用门控;实际接续能力仍须通过下文实验验证。 + ## 最小合法任务单元 合法粒度由边界决定,不由 token 数、文件数或一句指令的长度决定。一个单元至少满足以下义务: -| 义务 | 可检查的表达 | 不满足时的处理 | -| --- | --- | --- | -| 输入闭合 | 所有必要数据绑定版本化引用;前置产物明确来自哪个任务 | 缺输入就等待或回推,不靠共享对话补齐 | -| 产物可交接 | 有限大小、有类型、可定位的提案或证据引用 | “我已经想好了”不能解锁后继 | -| 权限闭合 | 可读资源、提案写集、工具效果受宿主约束 | 未知效果按冲突处理;不能凭 worker 自报放行 | -| 验收明确 | 宿主选定验证器和义务;accept/reject/undecidable 分开 | 无法判定保持未接受,不能当作成功 | -| 可停止与作废 | attempt、租约、预算、取消边界明确 | 迟到结果只能记录,不能生效 | -| 组合保真 | 子任务产物经明确组合仍覆盖父任务的全部义务 | 子任务各自通过不等于父任务完成 | +| 义务 | 可检查的表达 | 不满足时的处理 | +| ------------ | ---------------------------------------------------- | ------------------------------------------ | +| 输入闭合 | 所有必要数据绑定版本化引用;前置产物明确来自哪个任务 | 缺输入就等待或回推,不靠共享对话补齐 | +| 产物可交接 | 有限大小、有类型、可定位的提案或证据引用 | “我已经想好了”不能解锁后继 | +| 权限闭合 | 可读资源、提案写集、工具效果受宿主约束 | 未知效果按冲突处理;不能凭 worker 自报放行 | +| 验收明确 | 宿主选定验证器和义务;accept/reject/undecidable 分开 | 无法判定保持未接受,不能当作成功 | +| 可停止与作废 | attempt、租约、预算、取消边界明确 | 迟到结果只能记录,不能生效 | +| 组合保真 | 子任务产物经明确组合仍覆盖父任务的全部义务 | 子任务各自通过不等于父任务完成 | 允许粒度小到“从固定源码提取一个接口事实并引用行与摘要”,前提是后继确实需要这个可检查事实。“先思考错误处理”若没有可交接结果,应留在一个单元内部。 @@ -58,19 +88,19 @@ CPU 类比还暴露出两个研究缺口:自然语言任务没有现成 ISA Task IR 是从现有声明与记录计算出的只读视图;不接收另一份 `inputs/outputs/effects/accept` JSON,也不落一份可独立修改的 IR。下表中的概念名用于解释分析结果,输入与校验仍使用现有字段。需要新语义时扩展对应 owner,不增加同义字段。 -| 分析概念 | 当前字段与唯一类型 owner | 编译与边界 | -| --- | --- | --- | -| identity / intent | [BoardTicket、PatchTaskSpec](../../src/integration/ooo-board.ts):`runId/taskId/revision/attempt/inputDigest`、`instruction`;[PatchWork](../../src/integration/ooo-patch.ts) | 复用现有身份与 `instruction`,不增加 `id/contractRevision/intent` 别名。board entry ID 与逻辑 taskId 不等同,绑定键包含 runId | -| inputs | `PatchTaskSpec.files` → `FrozenPatchTask.files`;`ProbePlan` 依赖列表 → `BoardTicket.dependencies` | 文件内容冻结;依赖产物由宿主在认领时绑定。编译视图只派生引用与摘要;当前没有多端口输入语言 | -| outputs | [PatchSubmission、ConclusionKind](../../src/integration/ooo-patch.ts);`PatchTaskSpec.admittedConclusions`;board `deliverableDigest/Ref` | 沿用 patch/conclusion 和单次交付。`cannot-complete` 不因结构合法而成为成功。多输出端口与任意 artifact kind 尚不支持,不能偷偷序列化进说明文字 | -| effects | `ProbePlan` 的 effect / [DispatchTask.effect](../../src/integration/ooo-execution.ts);`PatchTaskSpec.visible/editable` | 现有 effect 是粗类别 `read-only/isolated-artifact`;worker 读范围来自 `visible`,提案写范围来自 `editable`,宿主验证还消费完整 `files`。它们不是完整的语义读写集;不透明工具效果按冲突处理 | -| accept | `PatchTaskSpec.verify`;[CycleOptions.checks、CaseRule、Requirement](../../src/integration/ooo-cycle.ts);board 独立 `verdict/judgedDigest` | 沿用宿主验证和 board 判定,不新增第二个 accepted。当前 `verify` 是进程内函数,不能持久化;恢复所需的可序列化验证描述必须从既有 checks/cases/mutations 等配置冻结,并由共享宿主重建,不存函数或任意 checker 字符串 | -| budget | [PatchBudget](../../src/integration/ooo-patch.ts):`perFile/output` | 限制输出字节,不是 tokens;取消独立的 `outputs.maxBytes` 别名。模型 token 预算若要成为可强制限制,应先扩展其 owner 与执行协议,当前视图不得假称支持 | -| limits | `PatchLimits.turns/reads/timeoutMs` → `FrozenPatchTask.limits` | 保留原单位与执行时限,不另设 `deadlineMs`;不等于 board claim 的租约时限,也不等于轮次总预算 | -| placement | 无现有等价字段 | 仅由共享调度策略产生临时融合/分配建议,不进入当前票据,不影响验收;`visible/editable` 不承担 placement 语义。持久化执行关联用记录中的任务/attempt/执行者引用,不复制任务声明 | -| deps / requires | `ProbePlan` 依赖列表;`CycleOptions.requires: Requirement[]` | deps 是派发与产物依赖;requires 是 cycle 对已记录产物的 `verified/mutant-killed/test-title` 条件。两者不等同,当前 requires 不是通用授权/分支谓词注册表 | -| external fact / assumptions | `ProbePlan` wait event、[外部检查票据](../../src/integration/ooo-check.ts);无通用 assumptions 类型 | 检查事件保留原身份;本文的有限事实推测仍需在共享协议显式扩展,不能把缺失功能伪装成 `requires` 或票据已有字段 | -| refinement | 无现有父义务映射类型 | 离线模型可验证预先给定的拆分关系;生产表示须扩展共享任务契约,不能成为 board 新 kind 或第二份任务正文 | +| 分析概念 | 当前字段与唯一类型 owner | 编译与边界 | +| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| identity / intent | [BoardTicket、PatchTaskSpec](../../src/integration/ooo-board.ts):`runId/taskId/revision/attempt/inputDigest`、`instruction`;[PatchWork](../../src/integration/ooo-patch.ts) | 复用现有身份与 `instruction`,不增加 `id/contractRevision/intent` 别名。board entry ID 与逻辑 taskId 不等同,绑定键包含 runId | +| inputs | `PatchTaskSpec.files` → `FrozenPatchTask.files`;`ProbePlan` 依赖列表 → `BoardTicket.dependencies` | 文件内容冻结;依赖产物由宿主在认领时绑定。编译视图只派生引用与摘要;当前没有多端口输入语言 | +| outputs | [PatchSubmission、ConclusionKind](../../src/integration/ooo-patch.ts);`PatchTaskSpec.admittedConclusions`;board `deliverableDigest/Ref` | 沿用 patch/conclusion 和单次交付。`cannot-complete` 不因结构合法而成为成功。多输出端口与任意 artifact kind 尚不支持,不能偷偷序列化进说明文字 | +| effects | `ProbePlan` 的 effect / [DispatchTask.effect](../../src/integration/ooo-execution.ts);`PatchTaskSpec.visible/editable` | 现有 effect 是粗类别 `read-only/isolated-artifact`;worker 读范围来自 `visible`,提案写范围来自 `editable`,宿主验证还消费完整 `files`。它们不是完整的语义读写集;不透明工具效果按冲突处理 | +| accept | `PatchTaskSpec.verify`;[Requirement](../../src/integration/task-semantics.ts)(原与 `CycleOptions.checks`、`CaseRule` 同处 `ooo-cycle.ts`,随轮次退役后留在语义模块);board 独立 `verdict/judgedDigest` | 沿用宿主验证和 board 判定,不新增第二个 accepted。当前 `verify` 是进程内函数,不能持久化;恢复所需的可序列化验证描述必须从既有 checks/cases/mutations 等配置冻结,并由共享宿主重建,不存函数或任意 checker 字符串 | +| budget | [PatchBudget](../../src/integration/ooo-patch.ts):`perFile/output` | 限制输出字节,不是 tokens;取消独立的 `outputs.maxBytes` 别名。模型 token 预算若要成为可强制限制,应先扩展其 owner 与执行协议,当前视图不得假称支持 | +| limits | `PatchLimits.turns/reads/timeoutMs` → `FrozenPatchTask.limits` | 保留原单位与执行时限,不另设 `deadlineMs`;不等于 board claim 的租约时限,也不等于轮次总预算 | +| placement | 无现有等价字段 | 仅由共享调度策略产生临时融合/分配建议,不进入当前票据,不影响验收;`visible/editable` 不承担 placement 语义。持久化执行关联用记录中的任务/attempt/执行者引用,不复制任务声明 | +| deps / requires | `ProbePlan` 依赖列表;`CycleOptions.requires: Requirement[]` | deps 是派发与产物依赖;requires 是 cycle 对已记录产物的 `verified/mutant-killed/test-title` 条件。两者不等同,当前 requires 不是通用授权/分支谓词注册表 | +| external fact / assumptions | `ProbePlan` wait event、[外部检查票据](../../src/integration/check-ticket.ts);无通用 assumptions 类型 | 检查事件保留原身份;本文的有限事实推测仍需在共享协议显式扩展,不能把缺失功能伪装成 `requires` 或票据已有字段 | +| refinement | 无现有父义务映射类型 | 离线模型可验证预先给定的拆分关系;生产表示须扩展共享任务契约,不能成为 board 新 kind 或第二份任务正文 | board `kind` 的唯一 owner 是 [TaskBoardKind](../../src/core/types.ts)。它描述协作消息用途:`handoff` 交接、`result` 返回、`decision` 判定;不是 patch/conclusion 类型,也不是 effect。OoO 行内的 patch/snapshot `kind` 又是内部任务类别。三者不得互相转换或新增 `task-ir` kind、工具、频道来承载重复定义。 @@ -127,11 +157,11 @@ W(B) ∩ (R(A) ∪ W(A)) = ∅ 把“猜 Agent 接下来想做什么”缩小为“对一个已声明、有限取值的事实提前准备候选”。必须分别表达三种门: -| 门 | 能否提前越过 | 提交时要求 | -| --- | --- | --- | -| 数据输入 | 默认不能;缺失内容不能用一个 true 代替 | 精确输入与产物版本绑定 | -| 分支事实 | 仅可丢弃、无外部写入的候选准备可研究 | 有权威事件证明假设,且版本仍有效 | -| 权限/安全前提 | 不能作为可猜测分支 | 在执行动作之前已满足 | +| 门 | 能否提前越过 | 提交时要求 | +| ------------- | -------------------------------------- | -------------------------------- | +| 数据输入 | 默认不能;缺失内容不能用一个 true 代替 | 精确输入与产物版本绑定 | +| 分支事实 | 仅可丢弃、无外部写入的候选准备可研究 | 有权威事件证明假设,且版本仍有效 | +| 权限/安全前提 | 不能作为可猜测分支 | 在执行动作之前已满足 | 例如 T 的测试提案只使用已冻结接口;发布 T 还需等待检查事件 g。若提案内容完全不依赖 g,移动纯准备工作到 g 之前属于依赖细化,不是推测。若 g 决定本轮是否需要 T,提前准备 T 才是可能白跑的控制推测。若 T 必须读取尚未产生的错误日志,猜“失败”仍不能补全输入,不能启动。 @@ -143,7 +173,7 @@ W(B) ∩ (R(A) ∪ W(A)) = ∅ ## 成本判断与旧提案的关系 -[推测提案](../decisions/proposed/2026-09-11-ooo-speculation.md)维持原生命周期;本文不以新研究自动启用它。其中两项推论不能作为普遍规则:无 token 仍消耗 CPU、I/O 和验证容量,收益不必非负;检查长期通过也不能证明可删依赖,删除需要契约层的不依赖证明。 +[推测提案](../decisions/implemented/2026-09-11-ooo-speculation.md)维持原生命周期;本文不以新研究自动启用它。其中两项推论不能作为普遍规则:无 token 仍消耗 CPU、I/O 和验证容量,收益不必非负;检查长期通过也不能证明可删依赖,删除需要契约层的不依赖证明。 推测应同时报告延迟、额外费用和质量,而不是混为一个“收益”。若确需单一门控,可预先声明成本换算系数,用下式作估计而非保证: @@ -160,12 +190,12 @@ estimatedUtility = p * savedCriticalPath 复用现有计算与推演能力,不为 OoO 另建可微引擎、激活系统或推理图存储。任务依赖图描述执行约束,autodiff 图描述数值运算,MGR 图描述记忆关联;它们可以通过版本化投影关联,但节点和边的语义不能直接等同。 -| 能力与 owner | 在任务协作中的复用方向 | 保留的边界 | -| --- | --- | --- | -| [autodiff](../../src/lab/autodiff.ts) 的 Tensor/UOp 与梯度计算 | 为明确构造的成本、收益或动作评分模型提供数值底座 | Agent 调用、离散调度和工具副作用不因表示成图就可微;不声称梯度穿过整个 Agent 任务 | -| [HA:Hierarchical Activation](hierarchical-activation.md) | 对任务所需上下文做激活与保留评分,研究上下文复用的价值信号 | 激活分数不改变语义置信度、访问权限、硬预算或任务依赖;时间状态仍归 session/branch 所有 | +| 能力与 owner | 在任务协作中的复用方向 | 保留的边界 | +| ------------------------------------------------------------------ | ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | +| [autodiff](../../src/lab/autodiff.ts) 的 Tensor/UOp 与梯度计算 | 为明确构造的成本、收益或动作评分模型提供数值底座 | Agent 调用、离散调度和工具副作用不因表示成图就可微;不声称梯度穿过整个 Agent 任务 | +| [HA:Hierarchical Activation](hierarchical-activation.md) | 对任务所需上下文做激活与保留评分,研究上下文复用的价值信号 | 激活分数不改变语义置信度、访问权限、硬预算或任务依赖;时间状态仍归 session/branch 所有 | | [MGR:MemoryGraphReasoner](../../src/lab/memory-graph-reasoner.ts) | 利用已有遍历与 what-if 推演提出有来源的上下文、路径或有限假设候选 | `MemoryNode.requires` 的软门控不等于任务依赖成立;what-if 结果不等于真实检查结果,也不能自行开启推测执行 | -| 共享任务语义与协调器 | 生成合法动作集合,校验建议并执行原子认领与提交 | 合法性和接受权不交给评分模型;建议失效、不可用或关闭时仍能确定性执行 | +| 共享任务语义与协调器 | 生成合法动作集合,校验建议并执行原子认领与提交 | 合法性和接受权不交给评分模型;建议失效、不可用或关闭时仍能确定性执行 | 未来接线顺序是:共享语义计算合法候选 → 可选 HA/MGR 提供上下文或建议 → 共享策略在合法集合内排序 → 认领/提交点重新校验。模型提出集合外动作时拒绝,而不是提高分数后放行。需要新任务或新依赖的建议须走显式计划修订,不能隐式改写冻结计划。关联使用现有 run/task/attempt 和 AG 投影身份,不引入第二套任务 ID 或 board kind。 @@ -177,19 +207,31 @@ estimatedUtility = p * savedCriticalPath 本节记录复用方向,不把已有实验引擎标成已接入任务运行时,不改变现有启用门控,也不新增开关。未来接线与门控变化仍由相应能力 owner 和隐藏能力登记管理;harness 只承担薄适配,不各自实现评分或推演策略。 +**已落地的只是这条接线顺序本身**([task-advisers](../../src/integration/task-advisers.ts)):`selectableTasks` 给出合法候选集合(规则仍只有一份,`nextTask` 是它的头),可选来源只能在该集合内排序,共享策略定序,认领点重新校验。来源看到的是有界投影,建议只能是“排序集合内已有的任务”——它无法新增任务或依赖(没有这样的字段),`assumptions` 只作来源记录、永不参与合法性,所以软前提不能把任何东西变合法;`fuse`/`prepare` 这类未建模动作在动作字段上就被拒绝;参数/投影版本、观察顺序或初始状态对不上的分数被重新评分并标明是新建议;跨 session/branch 的来源被拒绝;关闭或失败的来源退回规则策略并留下理由。默认配置不带任何来源,也就是规则策略本身,所以上句仍然成立:没有 HA/MGR 实现被接入,门控与开关未变;四个对比臂的预算仍属 F 组。 + ## 共享层与黑板 -| 所有者 | 职责 | -| --- | --- | -| 共享任务语义模块 | 解析/规范化 Task IR、依赖与效果检查、拆分义务检查、合法融合候选 | -| 共享运行时 | 计划冻结、选择/派发、票据、验证、发布、取消、费用与事件记录 | -| 黑板及协调存储 | 跨进程发现、交接、认领、结果引用;经所有者操作更新权威状态 | -| 共享 Agent Surface | 同一工具目录、参数/结果语义与可发现入口 | -| harness 薄适配 | 原生工具 schema 编码、会话启动/恢复/终止、模型调用、能力与用量回报 | +| 所有者 | 职责 | +| ------------------ | ------------------------------------------------------------------ | +| 共享任务语义模块 | 解析/规范化 Task IR、依赖与效果检查、拆分义务检查、合法融合候选 | +| 共享运行时 | 计划冻结、选择/派发、票据、验证、发布、取消、费用与事件记录 | +| 黑板及协调存储 | 跨进程发现、交接、认领、结果引用;经所有者操作更新权威状态 | +| 共享 Agent Surface | 同一工具目录、参数/结果语义与可发现入口 | +| harness 薄适配 | 原生工具 schema 编码、会话启动/恢复/终止、模型调用、能力与用量回报 | 黑板是工具和载体,不凭自然语言条目推导依赖,也不充当裁判。共享运行时通过 board 的租约、attempt 与 deliver/judge 操作执行状态转移,具体持久化和写入权按下节固定。 -共享协议不意味着每个 harness 都有同等执行能力。适配器报告 capability;缺少会话复用或隔离能力时降级为独立执行,不能把正确性规则留给适配器自行解释。现有 Pi `ooo_round` 经研究 CLI 接线的形状不能作为最终共享入口。 +共享协议不意味着每个 harness 都有同等执行能力。适配器报告 capability;缺少会话复用或隔离能力时降级为独立执行,不能把正确性规则留给适配器自行解释。 + +### 产品入口:由现有协作设施吸收 OoO + +用户确定的产品方向是:OoO 作为协作内部的执行能力,被黑板、共享任务语义、执行生命周期与验收设施吸收;Agent 通过普通任务交接使用它,不需要选择或调用独立的 OoO 工具。黑板负责交接、依赖/产物引用、认领和交付;共享语义派生合法动作;执行设施负责分配、等待、会话复用和取消;任务指定的检查负责产物验收,协调器按协议提交。黑板本身不承担通用执行器或验证器的全部职责。 + +普通黑板文本不会自动变成可执行任务。只有通过现有操作进入、具有明确任务契约与执行授权的交接,才参与上述派生和调度;元数据缺口在既有 owner 扩展。不得通过新增 `ooo` 工具、专用频道、另一份任务正文或 `board.runRound` 包装,保留一套独立计划、状态和生命周期。研究 round 是实验组织单位,不要求所有日常协作都先创建一个独立 round。 + +现有 Pi `ooo_round` 仅作为迁移期间的兼容入口,最终从产品工具目录退出。兼容期间复用同一共享操作与权威事实,不发展新功能或维持双写;研究 CLI 可继续做受控顺序/OoO 对照,但不成为产品功能的必经入口。本次只确定目标,不宣称兼容接线或工具删除已经完成。实际移除时同步工具目录、适配层、操作说明及隐藏能力登记。 + +退出前必须走通一条普通协作路径:Agent 经现有黑板交接一个契约明确的父任务,任务语义识别其中独立工作,执行设施按资源条件重排/执行,产物通过既有交付与独立验收返回;全程不调用 `ooo_round` 或研究 CLI,也不生成平行任务状态。验证真实 accepted 产物及依赖解锁,并覆盖失败、取消和无独立工作时的顺序回退;同时确认原入口承担的必要查询/取消能力可从既有设施到达,再删除独立工具。没有推测授权不执行推测,没有多槽或融合能力时使用合法退化路径。 ## 运行记录的物理归属与唯一写入者 @@ -199,19 +241,66 @@ estimatedUtility = p * savedCriticalPath 新增持久化只承担两类事实:不可变的运行 manifest(冻结现有计划、配置与版本),以及 board 没有的追加运行事实。可以分别用 manifest 表与事件表实现,但不得增加与 board 竞争的每任务当前状态表。下面固定事实 owner;物理表名以迁移实现为准,同库与权威分工不变: -| 事实 | 权威位置 | 写入与读取规则 | -| --- | --- | --- | -| runId、冻结计划/验证配置、输入及摘要、保留策略 | 不可变运行 manifest | 协调服务经 daemon Store 注册;依赖图从计划派生。新计划产生新版本,不能覆盖旧输入 | -| 逻辑任务与 board entry 的绑定、检查签发与终态、候选字节或内容寻址引用、取消事实 | 同库追加运行事实,键含 runId/sequence,任务事实另带 taskId/attempt | 只保存不能从 manifest 或 board 推导的事实;取消事件与 board 围栏操作原子提交。候选存在不等于接受 | -| claim owner、lease、attempt、deliverable、独立 verdict、resolve | 现有 board 记录 | board 生命周期操作是唯一写入逻辑;受管理条目的通用写请求也必须经过同一协调事务,禁止直接 judge/resolve 绕开运行围栏 | -| 费用、执行尝试观察与恢复证据 | 同一追加运行事实 | 外部结果由宿主报告、协调服务验证后写;worker 不批准自己的产物。重复结果幂等,旧代次拒收;board 转移的历史可在事务内记录,但当前 board 状态仍只读 board | -| ready/blocked/accepted 任务集合、依赖满足、取消后的有效性、融合候选 | 无权威存储;从上述来源计算 | 同一共享派生函数服务 status、next 与依赖解锁;可缓存,不允许独立写入或修复缓存来改事实 | -| `round.jsonl`、`run.json`、工作目录日志 | 导出或执行产物 | JSONL 可导出作研究重放,文件不决定当前完成/取消状态;导出失败不回滚已提交判定 | +| 事实 | 权威位置 | 写入与读取规则 | +| ------------------------------------------------------------------------------- | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| runId、冻结计划/验证配置、输入及摘要、保留策略 | 不可变运行 manifest | 协调服务经 daemon Store 注册;依赖图从计划派生。新计划产生新版本,不能覆盖旧输入 | +| 逻辑任务与 board entry 的绑定、检查签发与终态、候选字节或内容寻址引用、取消事实 | 同库追加运行事实,键含 runId/sequence,任务事实另带 taskId/attempt | 只保存不能从 manifest 或 board 推导的事实;取消事件与 board 围栏操作原子提交。候选存在不等于接受 | +| claim owner、lease、attempt、deliverable、独立 verdict、resolve | 现有 board 记录 | board 生命周期操作是唯一写入逻辑;受管理条目的通用写请求也必须经过同一协调事务,禁止直接 judge/resolve 绕开运行围栏 | +| 费用、执行尝试观察与恢复证据 | 同一追加运行事实 | 外部结果由宿主报告、协调服务验证后写;worker 不批准自己的产物。重复结果幂等,旧代次拒收;board 转移的历史可在事务内记录,但当前 board 状态仍只读 board | +| ready/blocked/accepted 任务集合、依赖满足、取消后的有效性、融合候选 | 无权威存储;从上述来源计算 | 同一共享派生函数服务 status、next 与依赖解锁;可缓存,不允许独立写入或修复缓存来改事实 | +| `round.jsonl`、`run.json`、工作目录日志 | 导出或执行产物 | JSONL 可导出作研究重放,文件不决定当前完成/取消状态;导出失败不回滚已提交判定 | 事务内同时完成版本/租约/取消校验、board deliver/judge/resolve、产物关联与事件追加;失败则全部回滚。外部验证收据先生成,入事务后重核绑定,不能在持有写事务时等待模型或检查进程。断线重试以 run/task/attempt 和事件身份去重;未知检查结果保持无法判定,不从 PID 消失推断成功。 接受查询与依赖解锁必须调用同一谓词:绑定 entry 对当前产物 digest 的有效 board verdict,加当前运行版本与围栏检查。不得一条路径看 board verdict,另一条仅检查 artifact 非空。历史 event 中曾 accepted 不表示当前仍 accepted。 +### 事务参与与连接生命周期 + +同一个连接不等于已经处于同一个事务。Store 拥有事务边界;round 加入某一次同步状态转移,不能把整轮或整个异步 RPC 包进数据库事务。以下是接入契约,不表示当前 Store 已提供这些接口。 + +| 能力 | 持有者 | 责任 | +| --------------------------------------------- | ------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------- | +| 连接创建、schema migration、checkpoint、close | daemon 的 Store owner;离线 fixture 则由创建它的外层宿主拥有 | 启动时完成初始化;关闭前停止并收束所有使用者;创建者负责异常清理 | +| 同步写事务入口 | Store | 唯一执行 `BEGIN IMMEDIATE/COMMIT/ROLLBACK`;无外层事务时建立事务,已有事务时须显式提供有效事务作用域 | +| 已有事务的操作端口 | round 与 board 内部操作 | 只调用绑定当前 Store 的类型化操作;不暴露原始连接、SQL 事务控制或 `close()` | +| round 执行作用域 | 共享协调器 | 管理本轮 worker/check、订阅和计时器;释放这些资源不关闭借用的 Store,也不删除非本轮所有的目录 | + +事务端口是 Store 签发、仅在同步回调期间有效的能力,绑定连接与当前事务代次。其他 Store 的端口、退出回调后的端口,以及未提供端口却在已有事务内再次进入的写事务调用,都明确拒绝。不能仅凭 `isTransaction` 或全局深度猜测调用方是否有权加入。 + +board 的独立写入口和组合写入口复用同一实现:独立调用由 Store 包装事务,round 内调用显式使用当前事务端口,不再执行第二次 BEGIN。凡在一次转移内触及的 put、claim、deliver、judge、resolve、保留关系和运行事实写入都遵循此规则;不能只改 round 的事务包装,遗漏 `putTaskBoardEntry()` 自带的 BEGIN。清理、串行晋升等伴随写入也在同一边界内,或明确由独立维护事务承担。 + +初版采用整笔原子事务,不增加 savepoint 级局部恢复。内层操作抛错时将当前事务标记为 rollback-only;即使调用方捕获异常,最外层也不得 COMMIT。预期的 stale/conflict 应在该操作首次写入前返回,或作为失败使整笔回滚;不得先写一部分,再用普通返回值掩盖失败。事务是否已提交只由最外层决定。ROLLBACK 自身失败时保留原错误并隔离该连接,不能假定连接仍可继续使用。 + +回调必须同步且不接受 Promise/thenable;类型约束之外,端口在回调结束即撤销,防止延迟回调继续写入。外部验证、模型调用和通知都不得持有端口跨越 `await`。提交路径保持:读取并冻结验证输入 → 事务外验证 → 新的短事务内重读 fence/lease/cancel 并原子写入。不能通过“复用事务”取消验证后的复核。 + +共享运行时在事务成功后才返回已提交结果并发送通知。必须可靠送达的通知先把待发送事实写进同一事务,再在事务外发送;瞬时通知失败不改写已经提交的事实。调用方未收到响应时按同一身份查询/重试,得到已提交或 duplicate;不重新执行 worker。`afterCommit` 一类 hook 只具有提交后观察语义,不能把其异常解释成数据库已回滚。 + +生命周期采用外层拥有、内层借用的单一路径:daemon 创建 Store,构造不含 close 的操作端口传入 round;离线宿主也先创建自己的 Store,再调用相同 round 入口,并由外层 finally 关闭。round 内不使用 `ownsStore` 分支分散关闭逻辑。正常返回、初始化失败、提前取消、检查抛错都经过各资源 owner 的统一清理;资源每成功创建一步就纳入清理范围。status 的借用视图构造不迁移 schema、不发布 handoff、不因查询触发 round 初始化写入。 + +daemon 关闭顺序为:停止接收新工作,取消/围栏在途尝试,撤销订阅并等待使用 Store 的异步任务退出,完成已开始的同步事务,最后 checkpoint/close 一次。无法及时终止的外部进程不得保留数据库操作能力;其迟到结果由已关闭的运行作用域拒绝。此顺序需要 daemon 生命周期 owner 配合,不能只保证 round 不调用 close,却让其他后台使用者在关库后写入。 + +### 运行面的形状 + +另一个进程到达运行事实的唯一入口是 daemon 的 `taskRun`:`register`(注册运行身份,同一计划只能注册一次)、`freeze`(一次转移冻结整份计划,位置取数组顺序,悬空依赖与自依赖在冻结前按名字拒绝)、`bind`(把已存在条目绑到已冻结任务)、`cancel`(取消整个运行,或计划中的某个任务)、`status`(读回运行记录)。`taskBoard` 的 `put` 增加可选 `adopt`,使“创建条目”与“绑定”落在同一转移里:先建后绑是两次调用,中间崩溃会留下一个没有运行管理的条目,正是本围栏要消除的洞。取消事实的唯一写入者是 `cancelRun`:围栏与派发派生都读它,而在这之前 `src/` 里没有任何写入者,即“运行已取消”这个状态只能由测试直接 append——真实客户端不可达。`status` 是纯读:不注册、不追加、不迁移、不发布,也不派生 ready/blocked/accepted——那是共享纯函数在同一批事实上的结果,在这里再算一遍就是设计禁止的第二份真相。 + +这些加法在兼容 epoch 内,不推进 epoch:旧同 epoch daemon 缺 `taskRun` 时由客户端报“能力不可用”,而它会静默忽略 `put` 的 `adopt`(该字段与 `taskRun` 同一次变更引入,描述符目录在 daemon 生命周期内冻结)——所以客户端只在 `hello.methods` 含 `taskRun` 时才发这个字段,epoch 规则因此无需变动(见 [design.md](design.md) 的兼容性段落)。CLI 只暴露操作者需要的一半(`nmg run status`、`nmg run cancel`);运行自己的转移由 runner 用 daemon 客户端调用,不为它们另设人类入口。 + +客户端的边界按“服务者可能被堵住”来设计,而不是按“它总会回应”:同一进程既服务端点又同步等待子进程时,它永远回不了那个请求,这不是事务或锁能修的问题(写入者、`BEGIN IMMEDIATE`、围栏都已经是最悲观的),而是回复机制本身被占用。因此 `taskRun`/`taskBoard` 的调用都有上界,超时按名字报出上界、端点与持有该端点的 pid,而不是把传输层自己的超时抛给调用者;并且**客户端不调用自己服务的端点**:租约记录里的 pid 就是服务者的 pid,所以这是可判定的,宿主自己的调用走进程内的 round 入口(`round-host.ts` 的 `host.call(...)`),需要走线的客户端则由独立进程的宿主服务。 + +### status 借用视图的形态 + +共享 daemon 的 status 使用 owner 从现有 Store 提供的窄查询端口,复用当前连接和共享状态派生函数。端口只有类型化查询方法,不暴露写方法、原始连接、任意 SQL、事务控制或 close;构造端口不创建数据库连接。只读是调用方持有的能力范围,不是将共享连接切换为只读模式,因此不得在该连接上设置 `query_only`,影响正常写请求。 + +查询不得调用带迁移、懒清理、租约晋升或 handoff 发布副作用的 Store 方法。到期状态在查询视图中计算,必要的持久清理由独立写入口负责。status 与调度读取同一套事实和判定规则,但 status 不因发现待处理事项而执行它们。 + +离线宿主查看已有私有轮次库时,允许由该宿主通过 Store owner 的只读打开路径创建独立连接,并负责释放。该路径使用真正的只读数据库句柄;跳过迁移、初始化、写型 PRAGMA 和关闭时的 checkpoint。独立连接可额外设置 `query_only`,但它不能代替真正的只读打开。共享层可提供这个 owner 工厂,`src/integration` 的 status 消费者不能自行按路径创建句柄。两条路径复用查询实现,不复制状态逻辑。 + +不以 `BoardAdmissionOptions.view` 将初始化/执行类改成多模式查询类;采用 owner 提供查询端口、外层宿主拥有连接的组合。`NmgStoreOptions.readOnly` 若用于实现独立只读打开,属于 owner 工厂配置,不进入 round 票据,也不成为每个调用方切换连接的开关。cancel 走正常可写操作端口;它不能通过重新构造 BoardAdmission 顺带初始化不存在的 run。 + +库不存在、schema 缺失或不支持、指定 run 不存在均明确拒绝,并区分错误原因;不得创建库、迁移旧库或返回“空运行”。只有存在有效 manifest 且 schema 可识别、run 身份匹配时,才能报告“运行存在但尚无事件”。旧 schema 若已有明确支持的只读解析规则可读取,否则报告不支持;不能通过默认空字段掩盖未知格式。 + +接入验证分开覆盖两种形态:共享端口构造/查询不写入且不暴露写入或关闭能力,释放视图后 owner 仍能读写;离线只读打开不创建缺失文件、不迁移旧 schema、不执行写入或 checkpoint,且宿主关闭自己创建的句柄。两者在相同事实快照与求值时间下给出一致结果。独立只读连接的变异检查需单独把只读打开退回可写,证明测试能抓住它,即使 `query_only` 仍在;只测试最终写入被拒绝不足以区分这两种保护。 + ### 重算的范围与代价 视图可写为 `derive(manifest, boardSnapshot, runFactsThroughSequence, now)`。一次求值使用一致的数据库读快照与固定的 `now`,否则可能把旧 claim 和新判定拼在一起。租约是否到期依赖时间,不能只按事件序号缓存;缓存至少绑定事实版本,并在相关租约边界失效。缓存丢失只影响性能。 @@ -226,15 +315,23 @@ board TTL 需要显式的运行保留关系:运行记录引用的交付/判定 ### 当前实现与接入前置条件 -当前 [round-runner](../../evals/ooo-execution/round-runner.ts)为每轮建立私有 store;[BoardAdmission](../../src/integration/ooo-board.ts)在该库里同时建 OoO 表并使用通用 board 表,另一进程的 status/cancel 也会打开该文件。这是每轮同库,不是上面选定的共享 daemon 模式;其 meta 单行、taskId 主键和固定 channel 也不能原样搬入多轮共享库。 +当前为每轮建立私有 store 的 runner 只剩 [live-continuation](../../evals/ooo-execution/live-continuation.ts);`round-runner.ts` 曾做同样的事,已随轮次于 2026-09-18 退役([决策](../decisions/implemented/2026-09-18-retire-the-round-instrument.md))。[BoardAdmission](../../src/integration/ooo-board.ts)在该库里同时建 OoO 表并使用通用 board 表。这是每轮同库,不是上面选定的共享 daemon 模式;其 meta 单行、taskId 主键和固定 channel 也不能原样搬入多轮共享库。三个证据驱动([board-worker](../../evals/ooo-execution/board-worker.ts)、[board-deliver](../../evals/ooo-execution/board-deliver.ts)、[board-judge](../../evals/ooo-execution/board-judge.ts))已经不再打开该文件:它们经 [round-client](../../evals/ooo-execution/round-client.ts) 调用服务该 store 的 daemon,无 daemon 时按名字拒绝([round-host](../../evals/ooo-execution/round-host.ts) 是把它服务起来的宿主);仍未接入的是 runner 自身——它既创建 store、又构造 `BoardAdmission`,所以研究轮次自己的计划/候选/探针状态还在私有 schema 里,而不在上文选定的运行命名空间。 + +当前 `accepted()` 联查 board verdict 与 digest,而部分依赖路径依据 artifact 存在;board 被 TTL 清除可能使两种观察分离。接入前必须统一派生函数并实现保留关系。现有 `ooo_probe_tasks` 混合了不可变输入、候选字节和可派生状态,不能整表复制为新权威。现有 `RoundLog`(JSONL 记录,随轮次退役)不能宣称已经具备同库原子历史,也不能作为新权威。 + +事务与生命周期静态审查定位到以下接入依据;它们尚未由本次文档修改修复: + +- `BoardAdmission.transaction()` 与 `NmgStoreBase.putTaskBoardEntry()` 各有独立 BEGIN。当前 claim/commit 路径内使用的 board 方法不另开事务,发布又在外层提交后,因此不能把未来共享接入风险报告成当前已经发生的嵌套失败。共享接入须同时改两侧事务边界。 +- `runCycle`(已随轮次退役)曾是这一依据的样本:拥有自建连接并在 finally 关闭;提前取消分支位于这个 finally 之前,只关闭 gate,遗漏默认临时数据库目录清理。退役删掉了样本本身,但这一依据仍针对剩下的 runner:借用模式不能照搬这些 close 路径,统一外层资源作用域也须覆盖初始化阶段。 +- `submit()` 在 `commitArtifact()` 后等待 `afterCommit` 并发布:后续异常发生时提交已经生效。接入接口需要明确提交结果与通知失败,不假定一次抛错证明数据库未提交。 -当前 `accepted()` 联查 board verdict 与 digest,而部分依赖路径依据 artifact 存在;board 被 TTL 清除可能使两种观察分离。接入前必须统一派生函数并实现保留关系。现有 `ooo_probe_tasks` 混合了不可变输入、候选字节和可派生状态,不能整表复制为新权威。现有 [RoundLog](../../src/integration/ooo-round-log.ts)是 JSONL 记录,不能宣称已经具备同库原子历史。 +这三条依据在本分支上的当前状态与逐项证据见 [task-unit-semantics-obligations.md](task-unit-semantics-obligations.md) 的 D 组。 迁移顺序是:先实现从既有契约与事实派生视图的纯函数;再注入共享事务 Store 与多轮命名空间,持久化最小 manifest/事实并接入受管理 board 生命周期与保留规则;最后让研究 runner 和薄适配经 daemon 调用。旧私有轮次只读保留作研究证据,不与新运行双写;需要重放时以新 runId 显式创建新轮次,不把旧 accepted 导入为新轮次接受。新运行身份与旧日志格式不兼容时明确拒绝,不能默默回放。 ## 下一步可执行切片与停止条件 -先做语义判定,不先搭建通用调度平台。下一实现切片限定为共享层的纯数据编译器与离线执行模型,复用已有身份类型;不接模型、不改 board 表、不启用多执行者或付费推测。 +先做语义判定,不先搭建通用调度平台。下一实现切片限定为共享层的纯数据编译器与离线执行模型,复用已有身份类型;不接模型、不改 board 表、不启用多执行者或付费推测。后续产品接入优先完成上面的普通黑板完整场景,仅补它实际需要的能力;全量研究调用点迁移不是检验任务拆分价值的先决条件。事务、权限和生命周期约束在该场景涉及的路径上仍必须满足,不能用缩小切片绕过。 输入为现有冻结任务契约、父契约与已记录事实,内部派生 Task IR;输出为合法计划或带任务/字段/义务位置的拒绝原因,以及允许动作的分析结果。离线模型不实际发布或另存运行状态;上述同库与唯一写入者是后续接入的确定目标,不是首切片建库的理由。实现时通过 repo context 定位拥有的模块和测试路由;研究枚举与成本模拟保持 advisory,成熟的版本/权限/错误发布不变量才进入产品阻塞测试。 @@ -255,15 +352,25 @@ board TTL 需要显式的运行保留关系:运行记录引用的交付/判定 对至多四个单元枚举合法事件交错,核对每次发布的义务、输入与来源;用故意删除关键条件的变异证明用例能抓错。通过仅说明该有限模型满足所列性质,不证明任意 Agent 程序正确。 -后续性能实验按顺序隔离变量: +### 辅助验证:真实任务接续 + +普通黑板产品路径走通后,先在一个真实父任务的合法交接边界更换 Agent 会话。接手者不接收原始对话,只接收上述任务视图并可检索必要证据,继续完成同一个父任务。记录遗漏约束、重复劳动、误认完成、过期依据导致的动作,以及恢复耗时和上下文成本;由固定的父级验收检查最终产物。检查数量与台账完成率不替代这个结果。 + +需要定位上下文交接问题时,对同一交接点保存受控输入,比较完整可见历史、同预算摘要、任务视图加按需证据三种上下文策略,固定模型、任务权限、工具与父级验收,计入检索和视图构造费用;可见历史不包含隐藏推理。这项辅助对照不作为下表全部实验的前置门槛。单个场景只能暴露缺口,不能证明通用长任务收益。真实模型调用仍遵循下文预算与重复次数要求,本设计记录不启动付费实验。 + +### 主验证:粒度、并发与融合 + +选择一个确有独立工作的父任务,明确哪条数据或控制依赖因合法细化而不再阻塞独立工作,并保留组合验收。仅把测试分成普通与边界两组、串行换会话,不能证明发现并发。已有[接续实验](../experiments/execution/ooo-real-continuation-2026-09-14.md)及[后续分析](../experiments/execution/ooo-real-continuation-comparison-2026-09-14.md)用于交接与运行器缺陷诊断,不作为下列主假设已成立的证据。 + +优先完成 A–D,E 在确定并发与融合得到证据后独立评估: -| 实验臂 | 改变的内容 | 要回答的问题 | -| --- | --- | --- | -| A 粗任务基线 | 原父任务与原验收 | 真实成本与质量基线是什么 | -| B 细任务、同一执行槽 | 拆分,独立会话 | 拆分本身增加多少交接与验证成本 | -| C 同一细计划、多槽 | 仅改变执行槽数/合法顺序 | 确定并发能减少多少关键路径 | -| D 同一细计划/槽数、融合 | 仅改变合法会话复用 | 节省是否超过上下文负担与等待 | -| E 同 D、有限事实推测 | 仅加入有预算的候选准备 | 额外费用与失败恢复后是否仍有收益 | +| 实验臂 | 改变的内容 | 要回答的问题 | +| ----------------------- | ----------------------- | -------------------------------- | +| A 粗任务基线 | 原父任务与原验收 | 真实成本与质量基线是什么 | +| B 细任务、同一执行槽 | 拆分,独立会话 | 拆分本身增加多少交接与验证成本 | +| C 同一细计划、多槽 | 仅改变执行槽数/合法顺序 | 确定并发能减少多少关键路径 | +| D 同一细计划/槽数、融合 | 仅改变合法会话复用 | 节省是否超过上下文负担与等待 | +| E 同 D、有限事实推测 | 仅加入有预算的候选准备 | 额外费用与失败恢复后是否仍有收益 | A/B 是任务分解比较,不能声称只改派发顺序;B/C、C/D、D/E 各自控制其余变量。父级验收固定,另记分解所需检查与规划成本。离线模型先覆盖不同粒度、依赖密度、共享上下文、事实命中率及验证成本;它只能发现逻辑错误和成本转折点,不能预测真实模型质量。 diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/README.md b/docs/experiments/execution/archive/ooo-arms-2026-09-19/README.md new file mode 100644 index 00000000..b13d8656 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/README.md @@ -0,0 +1,62 @@ +# Archive: the arms' paid runs (2026-09-19) + +**Why this is here.** The D arm (fusion) and the E arm (bounded speculation) were run from a worktree +whose scratch lives in `.temp/`, which `.gitignore` excludes. The records before this directory quoted +numbers - 22 498 against 22 533 tokens, 12 948 against 11 048 ms, the E arm's quality failures - and the +files those numbers came from would have gone with the next temp cleanup. This is the second time on +this branch: `archive/ooo-continuation-2026-09-14/README.md` was written for the same failure mode after +row G7, and the lesson did not survive the session it was learned in. Every sample a claim in +`ooo-arms-pilot-2026-09-18.md` depends on is now inside the repository. + +**What was run.** Provider `deepseek`, model `deepseek-v4-flash`, `--live` required in both entry points. +Every run's own report is kept verbatim; nothing here was edited after the fact. + +| Directory | Entry point | Runs | What it is | +| ---------------------------- | ---------------------------------------- | ---- | ----------------------------------------------------------------------------- | +| `fusion-darm/` | `plan-driver.ts run --session-runner` | 6 | D arm: `fusion.unitsPerSession` 1 (control) against 2 (fused), 3 reps each | +| `fusion-darm/spec-1.json` | — | — | The spec the control arm ran (bound 1), frozen envelope limits included | +| `fusion-darm/spec-2.json` | — | — | The spec the fused arm ran (bound 2); `spec-1` differs only in that bound | +| `fusion-darm/aggregate.json` | `node .temp/run-darm.mjs` | — | The six runs plus per-arm medians, which is what the record quotes | +| `smoke/` | `plan-driver.ts run --session-runner` | 4 | The mechanism smoke: two units in one session, and the runs that failed first | +| `speculation-earm/` | `evals/ooo-execution/speculation-pilot.ts --live` | 8 | E arm: baseline against speculation, fact true and false, 2 reps each | +| `speculation-earm/aggregate.json` | — | — | The eight runs plus per-(arm, fact) totals | +| `speculation-earm/run2-2026-09-19T05-40-04/` | `evals/ooo-execution/speculation-pilot.ts --live` (3 reps) | 9 | Second E-arm run: every attempt's artifact bytes, the candidate tree its check ran in, the check's own output, one row per run, the aggregate, and a `CLEANABLE.md` saying the directory is scratch | +| `harness-three-way.json` | `node .temp/p1-harness.mjs` | 6 | The harness validation that had to come first: frozen stub, the fixture's canned answer and a wrong answer, through the same check | + +**What is missing, and why that is now a plan item.** The E arm's first run stored no artifact bytes: +`speculation-pilot.ts` returned each candidate's verdict but deleted the candidate tree on failure, so +the run that reported "quality false in all four verified candidates" cannot be asked *why*. The +instrument no longer deletes a tree, and +`docs/experiments/execution/ooo-arm-plan-2026-09-19.md` fixes the fields every later run must write +before it is allowed to run - artifact bytes, check output, exit code and the frozen digest - so the +next paid run is diagnosable without paying twice. + +**The rule this directory exists to keep.** A run's evidence is written where git tracks it, in the same +change that reports its numbers, and no run deletes its own evidence. Scratch under `.temp/` is for +working copies; a result that a sentence in a record depends on is not scratch. + +**Correction (2026-09-19, after the second run's evidence).** The paragraph above recorded a defect +that was not one. `artifactEnvelope` builds two legitimate shapes - a patch (`digest, files`) and a +conclusion (`digest, kind, conclusion, summary, evidence, citations`) - and the E arm's first harness +fed *every* artifact to `patchCandidate`, which reads patches only. The run's quality failures were +therefore reported through the wrong reader: eight of nine attempts in the second run answered with a +conclusion, which this unit's check cannot pass and the board would refuse, and the one attempt that +submitted a patch failed for a real reason (it wrote `rows: [...]` where the frozen interface requires +`lines: [...]`). The instrument now reads an artifact by its kind and records which reader was used. + +## cap4-darm/ - the cap experiment (2026-09-19) + +Tests the offline ceiling's prediction that one session carrying four units saves about 5 700 ms. +Entry point: `aggregate.json` (per-bound medians and the measured saving); the four runs are +`bound{1,4}-rep{1,2}.json` and the two specs are `spec-{1,4}.json`, built from +`evals/ooo-execution/fixtures/pipeline/fine.spec.json` with the canned answers stripped, so the units +really run. Measured: 5 796 ms saved against the predicted 5 700 ms, with tokens up 1.3-1.9x. + +## cap-cache/ - the cap experiment with cache accounting (2026-09-19) + +Re-runs the cap experiment recording cache reads beside tokens, because a chain carries its context +forward and most of what it resends is served from cache. Entry point: `aggregate.json`. +Measured: cap 1 to cap 2 saves 8 753 ms (predicted 3 800 ms, so the startup constant is plan-dependent), +fresh input stays flat at 7 925 / 7 942 / 9 142 tokens, and cap 2 is the knee - it takes most of the +available wall clock at the fewest tokens. The earlier `cap4-darm/` reading of a 1.3-1.9x token +multiplier came from unpaired medians and is superseded. diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/aggregate.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/aggregate.json new file mode 100644 index 00000000..0387d77f --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/aggregate.json @@ -0,0 +1,140 @@ +{ + "aggregate": { + "runs": 6, + "perBound": [ + { + "bound": 1, + "reps": 2, + "sessions": [ + 4, + 4 + ], + "medianWallMs": 26619.5, + "medianTokens": 45684.5, + "medianCacheRead": 37760, + "medianCacheWrite": 0, + "medianFreshInput": 7924.5 + }, + { + "bound": 2, + "reps": 2, + "sessions": [ + 2, + 2 + ], + "medianWallMs": 17866.5, + "medianTokens": 39749.5, + "medianCacheRead": 31808, + "medianCacheWrite": 0, + "medianFreshInput": 7941.5 + }, + { + "bound": 4, + "reps": 2, + "sessions": [ + 1, + 1 + ], + "medianWallMs": 16645, + "medianTokens": 55286, + "medianCacheRead": 46144, + "medianCacheWrite": 0, + "medianFreshInput": 9142 + } + ], + "measuredSavingMs": 9974.5, + "note": "freshInput = tokens - cacheRead, shown because cache reads are priced below fresh input; the reports do not state whether the two are nested, so the column is a reading aid, not a price" + }, + "runs": [ + { + "bound": 1, + "rep": 1, + "wallMs": 26186, + "tokens": 45482, + "cacheRead": 38528, + "cacheWrite": 0, + "sessions": 4, + "units": [ + "normalize:11722t/9728c/0w", + "scale:11300t/9472c/0w", + "total:10890t/9344c/0w", + "summarize:11570t/9984c/0w" + ] + }, + { + "bound": 2, + "rep": 1, + "wallMs": 17735, + "tokens": 41806, + "cacheRead": 33536, + "cacheWrite": 0, + "sessions": 2, + "units": [ + "normalize:11172t/9344c/0w", + "scale:9897t/7424c/0w", + "total:10915t/9344c/0w", + "summarize:9822t/7424c/0w" + ] + }, + { + "bound": 4, + "rep": 1, + "wallMs": 16480, + "tokens": 54834, + "cacheRead": 45824, + "cacheWrite": 0, + "sessions": 1, + "units": [ + "normalize:11314t/9472c/0w", + "scale:10046t/7680c/0w", + "total:14457t/12032c/0w", + "summarize:19017t/16640c/0w" + ] + }, + { + "bound": 1, + "rep": 2, + "wallMs": 27053, + "tokens": 45887, + "cacheRead": 36992, + "cacheWrite": 0, + "sessions": 4, + "units": [ + "normalize:11521t/8064c/0w", + "scale:11499t/9600c/0w", + "total:11374t/9472c/0w", + "summarize:11493t/9856c/0w" + ] + }, + { + "bound": 2, + "rep": 2, + "wallMs": 17998, + "tokens": 37693, + "cacheRead": 30080, + "cacheWrite": 0, + "sessions": 2, + "units": [ + "normalize:11246t/9472c/0w", + "scale:9885t/7680c/0w", + "total:7402t/6272c/0w", + "summarize:9160t/6656c/0w" + ] + }, + { + "bound": 4, + "rep": 2, + "wallMs": 16810, + "tokens": 55738, + "cacheRead": 46464, + "cacheWrite": 0, + "sessions": 1, + "units": [ + "normalize:11632t/9600c/0w", + "scale:10293t/7936c/0w", + "total:14650t/12288c/0w", + "summarize:19163t/16640c/0w" + ] + } + ] +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound1-rep1.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound1-rep1.json new file mode 100644 index 00000000..acd1a2a1 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound1-rep1.json @@ -0,0 +1,104 @@ +{ + "measuredAt": "2026-09-19T06:54:14.863Z", + "spec": ".temp/darm5/spec-1.json", + "report": { + "plan": [ + "normalize", + "scale", + "total", + "summarize" + ], + "order": [ + "normalize", + "scale", + "total", + "summarize" + ], + "units": [ + { + "taskId": "normalize", + "verdict": "accepted", + "workerMs": 7244, + "hostMs": 964, + "tokens": 11722, + "attempt": 1, + "cacheRead": 9728, + "cacheWrite": 0, + "sessionId": "session:normalize" + }, + { + "taskId": "scale", + "verdict": "accepted", + "workerMs": 5433, + "hostMs": 1024, + "tokens": 11300, + "attempt": 1, + "cacheRead": 9472, + "cacheWrite": 0, + "sessionId": "session:scale" + }, + { + "taskId": "total", + "verdict": "accepted", + "workerMs": 4291, + "hostMs": 938, + "tokens": 10890, + "attempt": 1, + "cacheRead": 9344, + "cacheWrite": 0, + "sessionId": "session:total" + }, + { + "taskId": "summarize", + "verdict": "accepted", + "workerMs": 5353, + "hostMs": 925, + "tokens": 11570, + "attempt": 1, + "cacheRead": 9984, + "cacheWrite": 0, + "sessionId": "session:summarize" + } + ], + "accepted": { + "normalize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(steps: readonly Step[]): Step[] {\\n return steps\\n .filter((step) => step.ms > 0)\\n .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "scale": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(steps: readonly Step[], factor: number): Step[] {\\n return steps.map((step) => ({ name: step.name, ms: Math.floor(step.ms * factor) }));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summarize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import { renderStep, type Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(normalized: readonly Step[], totalMs: number): string {\\n return [...normalized, { name: \\\"total\\\", ms: totalMs }]\\n .map(renderStep)\\n .join(\\\"\\\\n\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "total": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(steps: readonly Step[]): number {\\n return steps.reduce((sum, step) => sum + step.ms, 0);\\n}\\n\"}}" + }, + "wallMs": 26186, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "normalize" + ], + [ + "scale" + ], + [ + "total" + ], + [ + "summarize" + ] + ], + "hostMs": 3851, + "hostChecks": 4, + "tokens": 45482, + "cacheRead": 38528, + "cacheWrite": 0, + "failures": 0, + "parent": { + "verdict": "accept", + "files": [ + "normalize", + "scale", + "total", + "summarize" + ], + "ms": 1102 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound1-rep2.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound1-rep2.json new file mode 100644 index 00000000..256ed358 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound1-rep2.json @@ -0,0 +1,104 @@ +{ + "measuredAt": "2026-09-19T06:55:29.010Z", + "spec": ".temp/darm5/spec-1.json", + "report": { + "plan": [ + "normalize", + "scale", + "total", + "summarize" + ], + "order": [ + "normalize", + "scale", + "total", + "summarize" + ], + "units": [ + { + "taskId": "normalize", + "verdict": "accepted", + "workerMs": 6993, + "hostMs": 1011, + "tokens": 11521, + "attempt": 1, + "cacheRead": 8064, + "cacheWrite": 0, + "sessionId": "session:normalize" + }, + { + "taskId": "scale", + "verdict": "accepted", + "workerMs": 5456, + "hostMs": 975, + "tokens": 11499, + "attempt": 1, + "cacheRead": 9600, + "cacheWrite": 0, + "sessionId": "session:scale" + }, + { + "taskId": "total", + "verdict": "accepted", + "workerMs": 5904, + "hostMs": 943, + "tokens": 11374, + "attempt": 1, + "cacheRead": 9472, + "cacheWrite": 0, + "sessionId": "session:total" + }, + { + "taskId": "summarize", + "verdict": "accepted", + "workerMs": 4789, + "hostMs": 970, + "tokens": 11493, + "attempt": 1, + "cacheRead": 9856, + "cacheWrite": 0, + "sessionId": "session:summarize" + } + ], + "accepted": { + "normalize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(steps: readonly Step[]): Step[] {\\n return steps\\n .filter((step) => step.ms > 0)\\n .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "scale": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(steps: readonly Step[], factor: number): Step[] {\\n return steps.map((step) => ({\\n name: step.name,\\n ms: Math.floor(step.ms * factor),\\n }));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summarize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import { renderStep, type Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(normalized: readonly Step[], totalMs: number): string {\\n const lines = normalized.map((step) => renderStep(step));\\n lines.push(renderStep({ name: \\\"total\\\", ms: totalMs }));\\n return lines.join(\\\"\\\\n\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "total": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(steps: readonly Step[]): number {\\n return steps.reduce((sum, step) => sum + step.ms, 0);\\n}\\n\"}}" + }, + "wallMs": 27053, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "normalize" + ], + [ + "scale" + ], + [ + "total" + ], + [ + "summarize" + ] + ], + "hostMs": 3899, + "hostChecks": 4, + "tokens": 45887, + "cacheRead": 36992, + "cacheWrite": 0, + "failures": 0, + "parent": { + "verdict": "accept", + "files": [ + "normalize", + "scale", + "total", + "summarize" + ], + "ms": 1169 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound2-rep1.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound2-rep1.json new file mode 100644 index 00000000..16195747 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound2-rep1.json @@ -0,0 +1,100 @@ +{ + "measuredAt": "2026-09-19T06:54:33.934Z", + "spec": ".temp/darm5/spec-2.json", + "report": { + "plan": [ + "normalize", + "scale", + "total", + "summarize" + ], + "order": [ + "normalize", + "scale", + "total", + "summarize" + ], + "units": [ + { + "taskId": "normalize", + "verdict": "accepted", + "workerMs": 6384, + "hostMs": 957, + "tokens": 11172, + "attempt": 1, + "cacheRead": 9344, + "cacheWrite": 0, + "sessionId": "session:normalize" + }, + { + "taskId": "scale", + "verdict": "accepted", + "workerMs": 1815, + "hostMs": 950, + "tokens": 9897, + "attempt": 1, + "cacheRead": 7424, + "cacheWrite": 0, + "sessionId": "session:normalize" + }, + { + "taskId": "total", + "verdict": "accepted", + "workerMs": 3684, + "hostMs": 973, + "tokens": 10915, + "attempt": 1, + "cacheRead": 9344, + "cacheWrite": 0, + "sessionId": "session:total" + }, + { + "taskId": "summarize", + "verdict": "accepted", + "workerMs": 1974, + "hostMs": 984, + "tokens": 9822, + "attempt": 1, + "cacheRead": 7424, + "cacheWrite": 0, + "sessionId": "session:total" + } + ], + "accepted": { + "normalize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(steps: readonly Step[]): Step[] {\\n return steps\\n .filter((step) => step.ms > 0)\\n .map((step) => ({ name: step.name, ms: step.ms }))\\n .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "scale": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(steps: readonly Step[], factor: number): Step[] {\\n return steps.map((step) => ({\\n name: step.name,\\n ms: Math.floor(step.ms * factor),\\n }));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summarize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\nimport { renderStep } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(normalized: readonly Step[], totalMs: number): string {\\n const lines = normalized.map(renderStep);\\n lines.push(renderStep({ name: \\\"total\\\", ms: totalMs }));\\n return lines.join(\\\"\\\\n\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "total": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(steps: readonly Step[]): number {\\n return steps.reduce((sum, step) => sum + step.ms, 0);\\n}\\n\"}}" + }, + "wallMs": 17735, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "normalize", + "scale" + ], + [ + "total", + "summarize" + ] + ], + "hostMs": 3864, + "hostChecks": 4, + "tokens": 41806, + "cacheRead": 33536, + "cacheWrite": 0, + "failures": 0, + "parent": { + "verdict": "accept", + "files": [ + "normalize", + "scale", + "total", + "summarize" + ], + "ms": 1057 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound2-rep2.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound2-rep2.json new file mode 100644 index 00000000..a2d61cd8 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound2-rep2.json @@ -0,0 +1,100 @@ +{ + "measuredAt": "2026-09-19T06:55:48.331Z", + "spec": ".temp/darm5/spec-2.json", + "report": { + "plan": [ + "normalize", + "scale", + "total", + "summarize" + ], + "order": [ + "normalize", + "scale", + "total", + "summarize" + ], + "units": [ + { + "taskId": "normalize", + "verdict": "accepted", + "workerMs": 7148, + "hostMs": 1003, + "tokens": 11246, + "attempt": 1, + "cacheRead": 9472, + "cacheWrite": 0, + "sessionId": "session:normalize" + }, + { + "taskId": "scale", + "verdict": "accepted", + "workerMs": 2156, + "hostMs": 956, + "tokens": 9885, + "attempt": 1, + "cacheRead": 7680, + "cacheWrite": 0, + "sessionId": "session:normalize" + }, + { + "taskId": "total", + "verdict": "accepted", + "workerMs": 2867, + "hostMs": 947, + "tokens": 7402, + "attempt": 1, + "cacheRead": 6272, + "cacheWrite": 0, + "sessionId": "session:total" + }, + { + "taskId": "summarize", + "verdict": "accepted", + "workerMs": 1918, + "hostMs": 991, + "tokens": 9160, + "attempt": 1, + "cacheRead": 6656, + "cacheWrite": 0, + "sessionId": "session:total" + } + ], + "accepted": { + "normalize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(steps: readonly Step[]): Step[] {\\n return steps\\n .filter((step) => step.ms > 0)\\n .slice()\\n .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "scale": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(steps: readonly Step[], factor: number): Step[] {\\n return steps.map((step) => ({ name: step.name, ms: Math.floor(step.ms * factor) }));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summarize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import { renderStep, type Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(normalized: readonly Step[], totalMs: number): string {\\n const lines = normalized.map(renderStep);\\n lines.push(renderStep({ name: \\\"total\\\", ms: totalMs }));\\n return lines.join(\\\"\\\\n\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "total": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(steps: readonly Step[]): number {\\n return steps.reduce((sum, step) => sum + step.ms, 0);\\n}\\n\"}}" + }, + "wallMs": 17998, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "normalize", + "scale" + ], + [ + "total", + "summarize" + ] + ], + "hostMs": 3897, + "hostChecks": 4, + "tokens": 37693, + "cacheRead": 30080, + "cacheWrite": 0, + "failures": 0, + "parent": { + "verdict": "accept", + "files": [ + "normalize", + "scale", + "total", + "summarize" + ], + "ms": 1072 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound4-rep1.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound4-rep1.json new file mode 100644 index 00000000..26f0d84d --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound4-rep1.json @@ -0,0 +1,98 @@ +{ + "measuredAt": "2026-09-19T06:54:52.694Z", + "spec": ".temp/darm5/spec-4.json", + "report": { + "plan": [ + "normalize", + "scale", + "total", + "summarize" + ], + "order": [ + "normalize", + "scale", + "total", + "summarize" + ], + "units": [ + { + "taskId": "normalize", + "verdict": "accepted", + "workerMs": 6660, + "hostMs": 953, + "tokens": 11314, + "attempt": 1, + "cacheRead": 9472, + "cacheWrite": 0, + "sessionId": "session:normalize" + }, + { + "taskId": "scale", + "verdict": "accepted", + "workerMs": 1974, + "hostMs": 981, + "tokens": 10046, + "attempt": 1, + "cacheRead": 7680, + "cacheWrite": 0, + "sessionId": "session:normalize" + }, + { + "taskId": "total", + "verdict": "accepted", + "workerMs": 1740, + "hostMs": 995, + "tokens": 14457, + "attempt": 1, + "cacheRead": 12032, + "cacheWrite": 0, + "sessionId": "session:normalize" + }, + { + "taskId": "summarize", + "verdict": "accepted", + "workerMs": 2128, + "hostMs": 1035, + "tokens": 19017, + "attempt": 1, + "cacheRead": 16640, + "cacheWrite": 0, + "sessionId": "session:normalize" + } + ], + "accepted": { + "normalize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(steps: readonly Step[]): Step[] {\\n return steps\\n .filter((step) => step.ms > 0)\\n .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "scale": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(steps: readonly Step[], factor: number): Step[] {\\n return steps.map((step) => ({ name: step.name, ms: Math.floor(step.ms * factor) }));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summarize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import { type Step, renderStep } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(normalized: readonly Step[], totalMs: number): string {\\n const lines = normalized.map((step) => renderStep(step));\\n lines.push(renderStep({ name: \\\"total\\\", ms: totalMs }));\\n return lines.join(\\\"\\\\n\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "total": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(steps: readonly Step[]): number {\\n return steps.reduce((sum, step) => sum + step.ms, 0);\\n}\\n\"}}" + }, + "wallMs": 16480, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "normalize", + "scale", + "total", + "summarize" + ] + ], + "hostMs": 3964, + "hostChecks": 4, + "tokens": 54834, + "cacheRead": 45824, + "cacheWrite": 0, + "failures": 0, + "parent": { + "verdict": "accept", + "files": [ + "normalize", + "scale", + "total", + "summarize" + ], + "ms": 2005 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound4-rep2.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound4-rep2.json new file mode 100644 index 00000000..9f8346d9 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/bound4-rep2.json @@ -0,0 +1,98 @@ +{ + "measuredAt": "2026-09-19T06:56:06.485Z", + "spec": ".temp/darm5/spec-4.json", + "report": { + "plan": [ + "normalize", + "scale", + "total", + "summarize" + ], + "order": [ + "normalize", + "scale", + "total", + "summarize" + ], + "units": [ + { + "taskId": "normalize", + "verdict": "accepted", + "workerMs": 7656, + "hostMs": 1004, + "tokens": 11632, + "attempt": 1, + "cacheRead": 9600, + "cacheWrite": 0, + "sessionId": "session:normalize" + }, + { + "taskId": "scale", + "verdict": "accepted", + "workerMs": 1930, + "hostMs": 964, + "tokens": 10293, + "attempt": 1, + "cacheRead": 7936, + "cacheWrite": 0, + "sessionId": "session:normalize" + }, + { + "taskId": "total", + "verdict": "accepted", + "workerMs": 1296, + "hostMs": 939, + "tokens": 14650, + "attempt": 1, + "cacheRead": 12288, + "cacheWrite": 0, + "sessionId": "session:normalize" + }, + { + "taskId": "summarize", + "verdict": "accepted", + "workerMs": 1973, + "hostMs": 1037, + "tokens": 19163, + "attempt": 1, + "cacheRead": 16640, + "cacheWrite": 0, + "sessionId": "session:normalize" + } + ], + "accepted": { + "normalize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(steps: readonly Step[]): Step[] {\\n return steps\\n .filter((step) => step.ms > 0)\\n .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "scale": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(steps: readonly Step[], factor: number): Step[] {\\n return steps.map((step) => ({ name: step.name, ms: Math.floor(step.ms * factor) }));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summarize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\nimport { renderStep } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(normalized: readonly Step[], totalMs: number): string {\\n return [...normalized, { name: \\\"total\\\", ms: totalMs }]\\n .map(renderStep)\\n .join(\\\"\\\\n\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "total": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(steps: readonly Step[]): number {\\n return steps.reduce((sum, step) => sum + step.ms, 0);\\n}\\n\"}}" + }, + "wallMs": 16810, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "normalize", + "scale", + "total", + "summarize" + ] + ], + "hostMs": 3944, + "hostChecks": 4, + "tokens": 55738, + "cacheRead": 46464, + "cacheWrite": 0, + "failures": 0, + "parent": { + "verdict": "accept", + "files": [ + "normalize", + "scale", + "total", + "summarize" + ], + "ms": 1069 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/spec-1.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/spec-1.json new file mode 100644 index 00000000..4149755d --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/spec-1.json @@ -0,0 +1,135 @@ +{ + "baseline": [ + "evals/ooo-execution/fixtures/pipeline/frozen.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.ts", + "evals/ooo-execution/fixtures/pipeline/scale.ts", + "evals/ooo-execution/fixtures/pipeline/total.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ], + "plan": [ + { + "id": "normalize", + "effect": "isolated-artifact" + }, + { + "id": "scale", + "effect": "isolated-artifact" + }, + { + "id": "total", + "effect": "isolated-artifact" + }, + { + "id": "summarize", + "effect": "isolated-artifact", + "dependencies": [ + "normalize", + "scale", + "total" + ] + } + ], + "units": { + "normalize": { + "instruction": "Implement normalize in this directory so that normalize.test.ts passes: keep only the steps with a positive ms and return them in name order, without changing the input array. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/normalize.ts" + ], + "checks": [ + { + "label": "normalize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts" + ] + } + ] + }, + "scale": { + "instruction": "Implement scale in this directory so that scale.test.ts passes: multiply every step's ms by the factor it is given and round down to whole milliseconds. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/scale.ts" + ], + "checks": [ + { + "label": "scale", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts" + ] + } + ] + }, + "total": { + "instruction": "Implement total in this directory so that total.test.ts passes: sum the ms of the steps it is given, and answer 0 for no steps. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/total.ts" + ], + "checks": [ + { + "label": "total", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/total.test.ts" + ] + } + ] + }, + "summarize": { + "instruction": "Implement summarize in this directory so that summarize.test.ts passes: render every step it was given with renderStep, in the order given, then render one more step named \"total\" whose ms is the total it was given, and join the lines with newlines. The three builders it is handed are already accepted. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/summarize.ts" + ], + "checks": [ + { + "label": "summarize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts" + ] + } + ] + } + }, + "parentChecks": [ + { + "label": "composed pipeline", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ] + } + ], + "worker": { + "kind": "pi", + "provider": "deepseek", + "model": "deepseek-v4-flash" + }, + "limits": { + "turns": 6, + "reads": 3, + "timeoutMs": 120000 + }, + "fusion": { + "unitsPerSession": 1 + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/spec-2.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/spec-2.json new file mode 100644 index 00000000..bfa7d505 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/spec-2.json @@ -0,0 +1,135 @@ +{ + "baseline": [ + "evals/ooo-execution/fixtures/pipeline/frozen.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.ts", + "evals/ooo-execution/fixtures/pipeline/scale.ts", + "evals/ooo-execution/fixtures/pipeline/total.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ], + "plan": [ + { + "id": "normalize", + "effect": "isolated-artifact" + }, + { + "id": "scale", + "effect": "isolated-artifact" + }, + { + "id": "total", + "effect": "isolated-artifact" + }, + { + "id": "summarize", + "effect": "isolated-artifact", + "dependencies": [ + "normalize", + "scale", + "total" + ] + } + ], + "units": { + "normalize": { + "instruction": "Implement normalize in this directory so that normalize.test.ts passes: keep only the steps with a positive ms and return them in name order, without changing the input array. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/normalize.ts" + ], + "checks": [ + { + "label": "normalize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts" + ] + } + ] + }, + "scale": { + "instruction": "Implement scale in this directory so that scale.test.ts passes: multiply every step's ms by the factor it is given and round down to whole milliseconds. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/scale.ts" + ], + "checks": [ + { + "label": "scale", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts" + ] + } + ] + }, + "total": { + "instruction": "Implement total in this directory so that total.test.ts passes: sum the ms of the steps it is given, and answer 0 for no steps. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/total.ts" + ], + "checks": [ + { + "label": "total", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/total.test.ts" + ] + } + ] + }, + "summarize": { + "instruction": "Implement summarize in this directory so that summarize.test.ts passes: render every step it was given with renderStep, in the order given, then render one more step named \"total\" whose ms is the total it was given, and join the lines with newlines. The three builders it is handed are already accepted. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/summarize.ts" + ], + "checks": [ + { + "label": "summarize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts" + ] + } + ] + } + }, + "parentChecks": [ + { + "label": "composed pipeline", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ] + } + ], + "worker": { + "kind": "pi", + "provider": "deepseek", + "model": "deepseek-v4-flash" + }, + "limits": { + "turns": 6, + "reads": 3, + "timeoutMs": 120000 + }, + "fusion": { + "unitsPerSession": 2 + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/spec-4.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/spec-4.json new file mode 100644 index 00000000..b57db97d --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap-cache/spec-4.json @@ -0,0 +1,135 @@ +{ + "baseline": [ + "evals/ooo-execution/fixtures/pipeline/frozen.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.ts", + "evals/ooo-execution/fixtures/pipeline/scale.ts", + "evals/ooo-execution/fixtures/pipeline/total.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ], + "plan": [ + { + "id": "normalize", + "effect": "isolated-artifact" + }, + { + "id": "scale", + "effect": "isolated-artifact" + }, + { + "id": "total", + "effect": "isolated-artifact" + }, + { + "id": "summarize", + "effect": "isolated-artifact", + "dependencies": [ + "normalize", + "scale", + "total" + ] + } + ], + "units": { + "normalize": { + "instruction": "Implement normalize in this directory so that normalize.test.ts passes: keep only the steps with a positive ms and return them in name order, without changing the input array. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/normalize.ts" + ], + "checks": [ + { + "label": "normalize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts" + ] + } + ] + }, + "scale": { + "instruction": "Implement scale in this directory so that scale.test.ts passes: multiply every step's ms by the factor it is given and round down to whole milliseconds. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/scale.ts" + ], + "checks": [ + { + "label": "scale", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts" + ] + } + ] + }, + "total": { + "instruction": "Implement total in this directory so that total.test.ts passes: sum the ms of the steps it is given, and answer 0 for no steps. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/total.ts" + ], + "checks": [ + { + "label": "total", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/total.test.ts" + ] + } + ] + }, + "summarize": { + "instruction": "Implement summarize in this directory so that summarize.test.ts passes: render every step it was given with renderStep, in the order given, then render one more step named \"total\" whose ms is the total it was given, and join the lines with newlines. The three builders it is handed are already accepted. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/summarize.ts" + ], + "checks": [ + { + "label": "summarize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts" + ] + } + ] + } + }, + "parentChecks": [ + { + "label": "composed pipeline", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ] + } + ], + "worker": { + "kind": "pi", + "provider": "deepseek", + "model": "deepseek-v4-flash" + }, + "limits": { + "turns": 6, + "reads": 3, + "timeoutMs": 120000 + }, + "fusion": { + "unitsPerSession": 4 + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/aggregate.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/aggregate.json new file mode 100644 index 00000000..f4354731 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/aggregate.json @@ -0,0 +1,110 @@ +{ + "aggregate": { + "runs": 4, + "perBound": [ + { + "bound": 1, + "reps": 2, + "medianWallMs": 24258, + "medianTokens": 45158, + "sessions": [ + 4, + 4 + ], + "accepted": [ + 0, + 0 + ] + }, + { + "bound": 4, + "reps": 2, + "medianWallMs": 19431, + "medianTokens": 83865, + "sessions": [ + 1, + 1 + ], + "accepted": [ + 0, + 0 + ] + } + ], + "measuredSavingMs": 4827, + "predictedSavingMs": 5700, + "note": "predicted saving is the offline ceiling for this plan: 4 sessions at cap 1 against 1 at cap 4, at the D arm's measured ~1 900 ms startup" + }, + "runs": [ + { + "bound": 1, + "rep": 1, + "wallMs": 24258, + "tokens": 45158, + "failures": 0, + "sessions": [ + 1, + 1, + 1, + 1 + ], + "units": [ + "normalize:accepted:11413t", + "scale:accepted:11557t", + "total:accepted:10683t", + "summarize:accepted:11505t" + ] + }, + { + "bound": 4, + "rep": 1, + "wallMs": 19431, + "tokens": 83865, + "failures": 0, + "sessions": [ + 4 + ], + "units": [ + "normalize:accepted:11741t", + "scale:accepted:16827t", + "total:accepted:23958t", + "summarize:accepted:31339t" + ] + }, + { + "bound": 1, + "rep": 2, + "wallMs": 22280, + "tokens": 41673, + "failures": 0, + "sessions": [ + 1, + 1, + 1, + 1 + ], + "units": [ + "normalize:accepted:11277t", + "scale:accepted:7751t", + "total:accepted:10689t", + "summarize:accepted:11956t" + ] + }, + { + "bound": 4, + "rep": 2, + "wallMs": 15515, + "tokens": 55143, + "failures": 0, + "sessions": [ + 4 + ], + "units": [ + "normalize:accepted:11406t", + "scale:accepted:10102t", + "total:accepted:14526t", + "summarize:accepted:19109t" + ] + } + ] +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/bound1-rep1.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/bound1-rep1.json new file mode 100644 index 00000000..106cdbe5 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/bound1-rep1.json @@ -0,0 +1,94 @@ +{ + "measuredAt": "2026-09-19T06:43:37.977Z", + "spec": ".temp/darm4/spec-1.json", + "report": { + "plan": [ + "normalize", + "scale", + "total", + "summarize" + ], + "order": [ + "normalize", + "scale", + "total", + "summarize" + ], + "units": [ + { + "taskId": "normalize", + "verdict": "accepted", + "workerMs": 6885, + "hostMs": 932, + "tokens": 11413, + "attempt": 1, + "sessionId": "session:normalize" + }, + { + "taskId": "scale", + "verdict": "accepted", + "workerMs": 4802, + "hostMs": 1020, + "tokens": 11557, + "attempt": 1, + "sessionId": "session:scale" + }, + { + "taskId": "total", + "verdict": "accepted", + "workerMs": 4167, + "hostMs": 972, + "tokens": 10683, + "attempt": 1, + "sessionId": "session:total" + }, + { + "taskId": "summarize", + "verdict": "accepted", + "workerMs": 4515, + "hostMs": 950, + "tokens": 11505, + "attempt": 1, + "sessionId": "session:summarize" + } + ], + "accepted": { + "normalize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(steps: readonly Step[]): Step[] {\\n return steps\\n .filter((step) => step.ms > 0)\\n .map((step) => ({ name: step.name, ms: step.ms }))\\n .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "scale": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(steps: readonly Step[], factor: number): Step[] {\\n return steps.map((step) => ({ name: step.name, ms: Math.floor(step.ms * factor) }));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summarize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import { renderStep, type Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(normalized: readonly Step[], totalMs: number): string {\\n const lines = normalized.map((step) => renderStep(step));\\n lines.push(renderStep({ name: \\\"total\\\", ms: totalMs }));\\n return lines.join(\\\"\\\\n\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "total": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(steps: readonly Step[]): number {\\n return steps.reduce((sum, step) => sum + step.ms, 0);\\n}\\n\"}}" + }, + "wallMs": 24258, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "normalize" + ], + [ + "scale" + ], + [ + "total" + ], + [ + "summarize" + ] + ], + "hostMs": 3874, + "hostChecks": 4, + "tokens": 45158, + "failures": 0, + "parent": { + "verdict": "accept", + "files": [ + "normalize", + "scale", + "total", + "summarize" + ], + "ms": 1177 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/bound1-rep2.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/bound1-rep2.json new file mode 100644 index 00000000..4360ce97 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/bound1-rep2.json @@ -0,0 +1,94 @@ +{ + "measuredAt": "2026-09-19T06:44:10.651Z", + "spec": ".temp/darm4/spec-1.json", + "report": { + "plan": [ + "normalize", + "scale", + "total", + "summarize" + ], + "order": [ + "normalize", + "scale", + "total", + "summarize" + ], + "units": [ + { + "taskId": "normalize", + "verdict": "accepted", + "workerMs": 6350, + "hostMs": 1030, + "tokens": 11277, + "attempt": 1, + "sessionId": "session:normalize" + }, + { + "taskId": "scale", + "verdict": "accepted", + "workerMs": 3184, + "hostMs": 950, + "tokens": 7751, + "attempt": 1, + "sessionId": "session:scale" + }, + { + "taskId": "total", + "verdict": "accepted", + "workerMs": 3848, + "hostMs": 955, + "tokens": 10689, + "attempt": 1, + "sessionId": "session:total" + }, + { + "taskId": "summarize", + "verdict": "accepted", + "workerMs": 5005, + "hostMs": 940, + "tokens": 11956, + "attempt": 1, + "sessionId": "session:summarize" + } + ], + "accepted": { + "normalize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(steps: readonly Step[]): Step[] {\\n return steps\\n .filter((step) => step.ms > 0)\\n .sort((left, right) =>\\n left.name < right.name ? -1 : left.name > right.name ? 1 : 0,\\n );\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "scale": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(steps: readonly Step[], factor: number): Step[] {\\n return steps.map((step) => ({\\n name: step.name,\\n ms: Math.floor(step.ms * factor),\\n }));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summarize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\nimport { renderStep } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(normalized: readonly Step[], totalMs: number): string {\\n return [...normalized, { name: \\\"total\\\", ms: totalMs }]\\n .map(renderStep)\\n .join(\\\"\\\\n\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "total": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(steps: readonly Step[]): number {\\n return steps.reduce((sum, step) => sum + step.ms, 0);\\n}\\n\"}}" + }, + "wallMs": 22280, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "normalize" + ], + [ + "scale" + ], + [ + "total" + ], + [ + "summarize" + ] + ], + "hostMs": 3875, + "hostChecks": 4, + "tokens": 41673, + "failures": 0, + "parent": { + "verdict": "accept", + "files": [ + "normalize", + "scale", + "total", + "summarize" + ], + "ms": 1071 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/bound4-rep1.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/bound4-rep1.json new file mode 100644 index 00000000..d11aabdf --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/bound4-rep1.json @@ -0,0 +1,88 @@ +{ + "measuredAt": "2026-09-19T06:43:02.807Z", + "spec": ".temp/darm4/spec-4.json", + "report": { + "plan": [ + "normalize", + "scale", + "total", + "summarize" + ], + "order": [ + "normalize", + "scale", + "total", + "summarize" + ], + "units": [ + { + "taskId": "normalize", + "verdict": "accepted", + "workerMs": 7723, + "hostMs": 911, + "tokens": 11741, + "attempt": 1, + "sessionId": "session:normalize" + }, + { + "taskId": "scale", + "verdict": "accepted", + "workerMs": 2296, + "hostMs": 971, + "tokens": 16827, + "attempt": 1, + "sessionId": "session:normalize" + }, + { + "taskId": "total", + "verdict": "accepted", + "workerMs": 2808, + "hostMs": 921, + "tokens": 23958, + "attempt": 1, + "sessionId": "session:normalize" + }, + { + "taskId": "summarize", + "verdict": "accepted", + "workerMs": 2762, + "hostMs": 1026, + "tokens": 31339, + "attempt": 1, + "sessionId": "session:normalize" + } + ], + "accepted": { + "normalize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(steps: readonly Step[]): Step[] {\\n return steps\\n .filter((step) => step.ms > 0)\\n .map((step) => ({ name: step.name, ms: step.ms }))\\n .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "scale": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(steps: readonly Step[], factor: number): Step[] {\\n return steps.map((step) => ({\\n name: step.name,\\n ms: Math.floor(step.ms * factor),\\n }));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summarize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import { renderStep, type Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(normalized: readonly Step[], totalMs: number): string {\\n return [...normalized, { name: \\\"total\\\", ms: totalMs }]\\n .map(renderStep)\\n .join(\\\"\\\\n\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "total": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(steps: readonly Step[]): number {\\n return steps.reduce((sum, step) => sum + step.ms, 0);\\n}\\n\"}}" + }, + "wallMs": 19431, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "normalize", + "scale", + "total", + "summarize" + ] + ], + "hostMs": 3829, + "hostChecks": 4, + "tokens": 83865, + "failures": 0, + "parent": { + "verdict": "accept", + "files": [ + "normalize", + "scale", + "total", + "summarize" + ], + "ms": 1067 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/bound4-rep2.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/bound4-rep2.json new file mode 100644 index 00000000..eb2011d6 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/bound4-rep2.json @@ -0,0 +1,88 @@ +{ + "measuredAt": "2026-09-19T06:44:27.461Z", + "spec": ".temp/darm4/spec-4.json", + "report": { + "plan": [ + "normalize", + "scale", + "total", + "summarize" + ], + "order": [ + "normalize", + "scale", + "total", + "summarize" + ], + "units": [ + { + "taskId": "normalize", + "verdict": "accepted", + "workerMs": 6874, + "hostMs": 970, + "tokens": 11406, + "attempt": 1, + "sessionId": "session:normalize" + }, + { + "taskId": "scale", + "verdict": "accepted", + "workerMs": 1473, + "hostMs": 941, + "tokens": 10102, + "attempt": 1, + "sessionId": "session:normalize" + }, + { + "taskId": "total", + "verdict": "accepted", + "workerMs": 1364, + "hostMs": 951, + "tokens": 14526, + "attempt": 1, + "sessionId": "session:normalize" + }, + { + "taskId": "summarize", + "verdict": "accepted", + "workerMs": 1938, + "hostMs": 995, + "tokens": 19109, + "attempt": 1, + "sessionId": "session:normalize" + } + ], + "accepted": { + "normalize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(steps: readonly Step[]): Step[] {\\n return steps\\n .filter((step) => step.ms > 0)\\n .map((step) => ({ name: step.name, ms: step.ms }))\\n .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "scale": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(steps: readonly Step[], factor: number): Step[] {\\n return steps.map((step) => ({ name: step.name, ms: Math.floor(step.ms * factor) }));\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summarize": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\nimport { renderStep } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(normalized: readonly Step[], totalMs: number): string {\\n const lines = normalized.map(renderStep);\\n lines.push(renderStep({ name: \\\"total\\\", ms: totalMs }));\\n return lines.join(\\\"\\\\n\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(_steps: readonly Step[]): number {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "total": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/pipeline/frozen.ts\":\"/**\\n * The frozen interface of the pipeline report this task family builds.\\n *\\n * Nothing here is anyone's unit: it is the contract the builders are written to, and the only\\n * renderer. A finer plan is the same work precisely because this file stays as it is.\\n */\\nexport interface Step {\\n readonly name: string;\\n readonly ms: number;\\n}\\n\\n/** The one renderer: a step is its name and its duration. */\\nexport function renderStep(step: Step): string {\\n return `${step.name}: ${step.ms}ms`;\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\n\\ntest(\\\"normalize keeps the steps that took time, in name order\\\", () => {\\n assert.deepEqual(\\n normalize([\\n { name: \\\"b\\\", ms: 2 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 1 },\\n ]),\\n [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(normalize([]), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/normalize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The steps the report is built from: only the ones that took time, in name order. */\\nexport function normalize(_steps: readonly Step[]): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/pipeline.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { normalize } from \\\"./normalize.ts\\\";\\nimport { scale } from \\\"./scale.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\nimport { total } from \\\"./total.ts\\\";\\n\\n/** The composition's own acceptance: the four builders over one input, rendered. */\\ntest(\\\"the composed report is the four builders over one input\\\", () => {\\n const raw = [\\n { name: \\\"b\\\", ms: 4 },\\n { name: \\\"x\\\", ms: 0 },\\n { name: \\\"a\\\", ms: 3 },\\n ];\\n const scaled = scale(normalize(raw), 2);\\n assert.equal(summarize(scaled, total(scaled)), \\\"a: 6ms\\\\nb: 8ms\\\\ntotal: 14ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { scale } from \\\"./scale.ts\\\";\\n\\ntest(\\\"scale multiplies every step and rounds down to whole milliseconds\\\", () => {\\n assert.deepEqual(scale([{ name: \\\"a\\\", ms: 3 }], 1.5), [{ name: \\\"a\\\", ms: 4 }]);\\n assert.deepEqual(\\n scale(\\n [\\n { name: \\\"a\\\", ms: 2 },\\n { name: \\\"b\\\", ms: 1 },\\n ],\\n 2,\\n ),\\n [\\n { name: \\\"a\\\", ms: 4 },\\n { name: \\\"b\\\", ms: 2 },\\n ],\\n );\\n assert.deepEqual(scale([], 3), []);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/scale.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The same steps at another size: whole milliseconds, rounded down. */\\nexport function scale(_steps: readonly Step[], _factor: number): Step[] {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Step } from \\\"./frozen.ts\\\";\\nimport { summarize } from \\\"./summarize.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which\\n * total it is handed is the composition's business, and the composed check is where the real\\n * builders meet. */\\nconst steps: Step[] = [\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n];\\n\\ntest(\\\"the report lists the steps it was given, then the total it was given\\\", () => {\\n assert.equal(summarize(steps, 3), \\\"a: 1ms\\\\nb: 2ms\\\\ntotal: 3ms\\\");\\n assert.equal(summarize([], 0), \\\"total: 0ms\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/summarize.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The report: every step it was given, and the total it was given, rendered. Its input is the other\\n * three builders' results, which is why it can only be built once they exist. */\\nexport function summarize(_normalized: readonly Step[], _totalMs: number): string {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/pipeline/total.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { total } from \\\"./total.ts\\\";\\n\\ntest(\\\"total adds the steps it was given, and an empty pipeline is zero\\\", () => {\\n assert.equal(\\n total([\\n { name: \\\"a\\\", ms: 1 },\\n { name: \\\"b\\\", ms: 2 },\\n ]),\\n 3,\\n );\\n assert.equal(total([]), 0);\\n});\\n\",\"evals/ooo-execution/fixtures/pipeline/total.ts\":\"import type { Step } from \\\"./frozen.ts\\\";\\n\\n/** The whole pipeline in one number: the sum of the steps it was given. */\\nexport function total(steps: readonly Step[]): number {\\n return steps.reduce((sum, step) => sum + step.ms, 0);\\n}\\n\"}}" + }, + "wallMs": 15515, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "normalize", + "scale", + "total", + "summarize" + ] + ], + "hostMs": 3857, + "hostChecks": 4, + "tokens": 55143, + "failures": 0, + "parent": { + "verdict": "accept", + "files": [ + "normalize", + "scale", + "total", + "summarize" + ], + "ms": 1027 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/spec-1.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/spec-1.json new file mode 100644 index 00000000..4149755d --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/spec-1.json @@ -0,0 +1,135 @@ +{ + "baseline": [ + "evals/ooo-execution/fixtures/pipeline/frozen.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.ts", + "evals/ooo-execution/fixtures/pipeline/scale.ts", + "evals/ooo-execution/fixtures/pipeline/total.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ], + "plan": [ + { + "id": "normalize", + "effect": "isolated-artifact" + }, + { + "id": "scale", + "effect": "isolated-artifact" + }, + { + "id": "total", + "effect": "isolated-artifact" + }, + { + "id": "summarize", + "effect": "isolated-artifact", + "dependencies": [ + "normalize", + "scale", + "total" + ] + } + ], + "units": { + "normalize": { + "instruction": "Implement normalize in this directory so that normalize.test.ts passes: keep only the steps with a positive ms and return them in name order, without changing the input array. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/normalize.ts" + ], + "checks": [ + { + "label": "normalize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts" + ] + } + ] + }, + "scale": { + "instruction": "Implement scale in this directory so that scale.test.ts passes: multiply every step's ms by the factor it is given and round down to whole milliseconds. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/scale.ts" + ], + "checks": [ + { + "label": "scale", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts" + ] + } + ] + }, + "total": { + "instruction": "Implement total in this directory so that total.test.ts passes: sum the ms of the steps it is given, and answer 0 for no steps. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/total.ts" + ], + "checks": [ + { + "label": "total", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/total.test.ts" + ] + } + ] + }, + "summarize": { + "instruction": "Implement summarize in this directory so that summarize.test.ts passes: render every step it was given with renderStep, in the order given, then render one more step named \"total\" whose ms is the total it was given, and join the lines with newlines. The three builders it is handed are already accepted. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/summarize.ts" + ], + "checks": [ + { + "label": "summarize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts" + ] + } + ] + } + }, + "parentChecks": [ + { + "label": "composed pipeline", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ] + } + ], + "worker": { + "kind": "pi", + "provider": "deepseek", + "model": "deepseek-v4-flash" + }, + "limits": { + "turns": 6, + "reads": 3, + "timeoutMs": 120000 + }, + "fusion": { + "unitsPerSession": 1 + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/spec-4.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/spec-4.json new file mode 100644 index 00000000..b57db97d --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/cap4-darm/spec-4.json @@ -0,0 +1,135 @@ +{ + "baseline": [ + "evals/ooo-execution/fixtures/pipeline/frozen.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.ts", + "evals/ooo-execution/fixtures/pipeline/scale.ts", + "evals/ooo-execution/fixtures/pipeline/total.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ], + "plan": [ + { + "id": "normalize", + "effect": "isolated-artifact" + }, + { + "id": "scale", + "effect": "isolated-artifact" + }, + { + "id": "total", + "effect": "isolated-artifact" + }, + { + "id": "summarize", + "effect": "isolated-artifact", + "dependencies": [ + "normalize", + "scale", + "total" + ] + } + ], + "units": { + "normalize": { + "instruction": "Implement normalize in this directory so that normalize.test.ts passes: keep only the steps with a positive ms and return them in name order, without changing the input array. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/normalize.ts" + ], + "checks": [ + { + "label": "normalize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts" + ] + } + ] + }, + "scale": { + "instruction": "Implement scale in this directory so that scale.test.ts passes: multiply every step's ms by the factor it is given and round down to whole milliseconds. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/scale.ts" + ], + "checks": [ + { + "label": "scale", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts" + ] + } + ] + }, + "total": { + "instruction": "Implement total in this directory so that total.test.ts passes: sum the ms of the steps it is given, and answer 0 for no steps. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/total.ts" + ], + "checks": [ + { + "label": "total", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/total.test.ts" + ] + } + ] + }, + "summarize": { + "instruction": "Implement summarize in this directory so that summarize.test.ts passes: render every step it was given with renderStep, in the order given, then render one more step named \"total\" whose ms is the total it was given, and join the lines with newlines. The three builders it is handed are already accepted. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/summarize.ts" + ], + "checks": [ + { + "label": "summarize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts" + ] + } + ] + } + }, + "parentChecks": [ + { + "label": "composed pipeline", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ] + } + ], + "worker": { + "kind": "pi", + "provider": "deepseek", + "model": "deepseek-v4-flash" + }, + "limits": { + "turns": 6, + "reads": 3, + "timeoutMs": 120000 + }, + "fusion": { + "unitsPerSession": 4 + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/aggregate.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/aggregate.json new file mode 100644 index 00000000..9f9d9a1d --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/aggregate.json @@ -0,0 +1,123 @@ +{ + "aggregate": [ + { + "bound": 1, + "arm": "unfused (one unit per session)", + "sessions": [ + "1/1", + "1/1", + "1/1" + ], + "medianTokens": 22498, + "medianWallMs": 12948, + "failures": 0 + }, + { + "bound": 2, + "arm": "fused (two units one session)", + "sessions": [ + "2", + "2", + "2" + ], + "medianTokens": 22533, + "medianWallMs": 11048, + "failures": 0 + } + ], + "runs": [ + { + "bound": 1, + "rep": 1, + "elapsedMs": 13178, + "wallMs": 11902, + "tokens": 22498, + "failures": 0, + "sessions": [ + 1, + 1 + ], + "units": [ + "alpha:accepted:11211t", + "summary:accepted:11287t" + ] + }, + { + "bound": 1, + "rep": 2, + "elapsedMs": 14212, + "wallMs": 12948, + "tokens": 22435, + "failures": 0, + "sessions": [ + 1, + 1 + ], + "units": [ + "alpha:accepted:10714t", + "summary:accepted:11721t" + ] + }, + { + "bound": 1, + "rep": 3, + "elapsedMs": 15328, + "wallMs": 14066, + "tokens": 22922, + "failures": 0, + "sessions": [ + 1, + 1 + ], + "units": [ + "alpha:accepted:11297t", + "summary:accepted:11625t" + ] + }, + { + "bound": 2, + "rep": 1, + "elapsedMs": 13099, + "wallMs": 11812, + "tokens": 22533, + "failures": 0, + "sessions": [ + 2 + ], + "units": [ + "alpha:accepted:11828t", + "summary:accepted:10705t" + ] + }, + { + "bound": 2, + "rep": 2, + "elapsedMs": 10801, + "wallMs": 9584, + "tokens": 17592, + "failures": 0, + "sessions": [ + 2 + ], + "units": [ + "alpha:accepted:7903t", + "summary:accepted:9689t" + ] + }, + { + "bound": 2, + "rep": 3, + "elapsedMs": 12266, + "wallMs": 11048, + "tokens": 26299, + "failures": 0, + "sessions": [ + 2 + ], + "units": [ + "alpha:accepted:15502t", + "summary:accepted:10797t" + ] + } + ] +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound1-rep1.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound1-rep1.json new file mode 100644 index 00000000..a2659024 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound1-rep1.json @@ -0,0 +1,62 @@ +{ + "measuredAt": "2026-09-19T05:15:42.766Z", + "spec": ".temp/darm/spec-1.json", + "report": { + "plan": [ + "alpha", + "summary" + ], + "order": [ + "alpha", + "summary" + ], + "units": [ + { + "taskId": "alpha", + "verdict": "accepted", + "workerMs": 6215, + "hostMs": 945, + "tokens": 11211, + "attempt": 1, + "sessionId": "session:alpha" + }, + { + "taskId": "summary", + "verdict": "accepted", + "workerMs": 3839, + "hostMs": 901, + "tokens": 11287, + "attempt": 1, + "sessionId": "session:summary" + } + ], + "accepted": { + "alpha": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(rows: readonly string[]): Section {\\n return {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: rows.filter((row) => row !== \\\"\\\").sort(),\\n };\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(_sections: readonly Section[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summary": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(sections: readonly Section[]): Section {\\n return {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: sections.map((section) => `- ${section.title}`),\\n };\\n}\\n\"}}" + }, + "wallMs": 11902, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "alpha" + ], + [ + "summary" + ] + ], + "hostMs": 1846, + "hostChecks": 2, + "tokens": 22498, + "failures": 0, + "parent": { + "verdict": "reject", + "files": [ + "alpha", + "summary" + ], + "ms": 1054 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound1-rep2.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound1-rep2.json new file mode 100644 index 00000000..b040461f --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound1-rep2.json @@ -0,0 +1,62 @@ +{ + "measuredAt": "2026-09-19T05:15:56.986Z", + "spec": ".temp/darm/spec-1.json", + "report": { + "plan": [ + "alpha", + "summary" + ], + "order": [ + "alpha", + "summary" + ], + "units": [ + { + "taskId": "alpha", + "verdict": "accepted", + "workerMs": 5680, + "hostMs": 930, + "tokens": 10714, + "attempt": 1, + "sessionId": "session:alpha" + }, + { + "taskId": "summary", + "verdict": "accepted", + "workerMs": 5445, + "hostMs": 891, + "tokens": 11721, + "attempt": 1, + "sessionId": "session:summary" + } + ], + "accepted": { + "alpha": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(rows: readonly string[]): Section {\\n const lines = rows.filter((row) => row.trim() !== \\\"\\\").slice().sort();\\n return { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines };\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(_sections: readonly Section[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summary": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(sections: readonly Section[]): Section {\\n return {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: sections.map((section) => `- ${section.title}`),\\n };\\n}\\n\"}}" + }, + "wallMs": 12948, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "alpha" + ], + [ + "summary" + ] + ], + "hostMs": 1821, + "hostChecks": 2, + "tokens": 22435, + "failures": 0, + "parent": { + "verdict": "reject", + "files": [ + "alpha", + "summary" + ], + "ms": 1019 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound1-rep3.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound1-rep3.json new file mode 100644 index 00000000..7dd72fd1 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound1-rep3.json @@ -0,0 +1,62 @@ +{ + "measuredAt": "2026-09-19T05:16:12.316Z", + "spec": ".temp/darm/spec-1.json", + "report": { + "plan": [ + "alpha", + "summary" + ], + "order": [ + "alpha", + "summary" + ], + "units": [ + { + "taskId": "alpha", + "verdict": "accepted", + "workerMs": 7204, + "hostMs": 933, + "tokens": 11297, + "attempt": 1, + "sessionId": "session:alpha" + }, + { + "taskId": "summary", + "verdict": "accepted", + "workerMs": 4985, + "hostMs": 941, + "tokens": 11625, + "attempt": 1, + "sessionId": "session:summary" + } + ], + "accepted": { + "alpha": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(rows: readonly string[]): Section {\\n return {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: rows.filter((row) => row.trim() !== \\\"\\\").sort(),\\n };\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(_sections: readonly Section[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summary": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(sections: readonly Section[]): Section {\\n return {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: sections.map((s) => `- ${s.title}`),\\n };\\n}\\n\"}}" + }, + "wallMs": 14066, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "alpha" + ], + [ + "summary" + ] + ], + "hostMs": 1874, + "hostChecks": 2, + "tokens": 22922, + "failures": 0, + "parent": { + "verdict": "reject", + "files": [ + "alpha", + "summary" + ], + "ms": 1029 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound2-rep1.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound2-rep1.json new file mode 100644 index 00000000..a39c5e0b --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound2-rep1.json @@ -0,0 +1,60 @@ +{ + "measuredAt": "2026-09-19T05:16:25.416Z", + "spec": ".temp/darm/spec-2.json", + "report": { + "plan": [ + "alpha", + "summary" + ], + "order": [ + "alpha", + "summary" + ], + "units": [ + { + "taskId": "alpha", + "verdict": "accepted", + "workerMs": 7697, + "hostMs": 932, + "tokens": 11828, + "attempt": 1, + "sessionId": "session:alpha" + }, + { + "taskId": "summary", + "verdict": "accepted", + "workerMs": 2283, + "hostMs": 895, + "tokens": 10705, + "attempt": 1, + "sessionId": "session:alpha" + } + ], + "accepted": { + "alpha": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(rows: readonly string[]): Section {\\n return {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: rows.filter((row) => row.trim() !== \\\"\\\").slice().sort(),\\n };\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(_sections: readonly Section[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summary": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(sections: readonly Section[]): Section {\\n return {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: sections.map((section) => `- ${section.title}`),\\n };\\n}\\n\"}}" + }, + "wallMs": 11812, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "alpha", + "summary" + ] + ], + "hostMs": 1827, + "hostChecks": 2, + "tokens": 22533, + "failures": 0, + "parent": { + "verdict": "reject", + "files": [ + "alpha", + "summary" + ], + "ms": 1052 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound2-rep2.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound2-rep2.json new file mode 100644 index 00000000..c824e8e0 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound2-rep2.json @@ -0,0 +1,60 @@ +{ + "measuredAt": "2026-09-19T05:16:36.219Z", + "spec": ".temp/darm/spec-2.json", + "report": { + "plan": [ + "alpha", + "summary" + ], + "order": [ + "alpha", + "summary" + ], + "units": [ + { + "taskId": "alpha", + "verdict": "accepted", + "workerMs": 6012, + "hostMs": 868, + "tokens": 7903, + "attempt": 1, + "sessionId": "session:alpha" + }, + { + "taskId": "summary", + "verdict": "accepted", + "workerMs": 1750, + "hostMs": 948, + "tokens": 9689, + "attempt": 1, + "sessionId": "session:alpha" + } + ], + "accepted": { + "alpha": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(rows: readonly string[]): Section {\\n const lines = rows.filter((row) => row.trim() !== \\\"\\\").slice().sort();\\n return { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines };\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(_sections: readonly Section[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summary": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(sections: readonly Section[]): Section {\\n return {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: sections.map((section) => `- ${section.title}`),\\n };\\n}\\n\"}}" + }, + "wallMs": 9584, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "alpha", + "summary" + ] + ], + "hostMs": 1816, + "hostChecks": 2, + "tokens": 17592, + "failures": 0, + "parent": { + "verdict": "reject", + "files": [ + "alpha", + "summary" + ], + "ms": 985 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound2-rep3.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound2-rep3.json new file mode 100644 index 00000000..4e421c59 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/bound2-rep3.json @@ -0,0 +1,60 @@ +{ + "measuredAt": "2026-09-19T05:16:48.472Z", + "spec": ".temp/darm/spec-2.json", + "report": { + "plan": [ + "alpha", + "summary" + ], + "order": [ + "alpha", + "summary" + ], + "units": [ + { + "taskId": "alpha", + "verdict": "accepted", + "workerMs": 7398, + "hostMs": 879, + "tokens": 15502, + "attempt": 1, + "sessionId": "session:alpha" + }, + { + "taskId": "summary", + "verdict": "accepted", + "workerMs": 1817, + "hostMs": 949, + "tokens": 10797, + "attempt": 1, + "sessionId": "session:alpha" + } + ], + "accepted": { + "alpha": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(rows: readonly string[]): Section {\\n const lines = rows.filter((row) => row.trim() !== \\\"\\\").slice().sort();\\n return { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines };\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(_sections: readonly Section[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summary": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(sections: readonly Section[]): Section {\\n return {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: sections.map((section) => `- ${section.title}`),\\n };\\n}\\n\"}}" + }, + "wallMs": 11048, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "alpha", + "summary" + ] + ], + "hostMs": 1828, + "hostChecks": 2, + "tokens": 26299, + "failures": 0, + "parent": { + "verdict": "reject", + "files": [ + "alpha", + "summary" + ], + "ms": 962 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/spec-1.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/spec-1.json new file mode 100644 index 00000000..65c113a2 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/spec-1.json @@ -0,0 +1,100 @@ +{ + "baseline": [ + "evals/ooo-execution/fixtures/report/interface.ts", + "evals/ooo-execution/fixtures/report/alpha.ts", + "evals/ooo-execution/fixtures/report/beta.ts", + "evals/ooo-execution/fixtures/report/gamma.ts", + "evals/ooo-execution/fixtures/report/summary.ts", + "evals/ooo-execution/fixtures/report/alpha.test.ts", + "evals/ooo-execution/fixtures/report/beta.test.ts", + "evals/ooo-execution/fixtures/report/gamma.test.ts", + "evals/ooo-execution/fixtures/report/summary.test.ts", + "evals/ooo-execution/fixtures/report/report.test.ts" + ], + "plan": [ + { + "id": "alpha", + "effect": "isolated-artifact" + }, + { + "id": "summary", + "effect": "isolated-artifact", + "dependencies": [ + "alpha" + ] + } + ], + "units": { + "alpha": { + "instruction": "Implement alphaSection in this directory so that alpha.test.ts passes. It drops blank rows and sorts the rest, and describes itself with id \"alpha\" and title \"Alpha\". The interface file is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/report/alpha.ts" + ], + "canned": { + "evals/ooo-execution/fixtures/report/alpha.ts": "evals/ooo-execution/fixtures/report/alpha.canned.ts" + }, + "checks": [ + { + "label": "alpha", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/alpha.test.ts" + ] + } + ] + }, + "summary": { + "instruction": "Implement summarySection in this directory so that summary.test.ts passes. It lists one line per section, in the order it was given, each as '- ' plus that section's title, and describes itself with id \"summary\" and title \"Summary\". The three sections it is handed are already accepted; the interface file is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/report/summary.ts" + ], + "canned": { + "evals/ooo-execution/fixtures/report/summary.ts": "evals/ooo-execution/fixtures/report/summary.canned.ts" + }, + "checks": [ + { + "label": "summary", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/summary.test.ts" + ] + } + ], + "dependencies": [ + "alpha" + ] + } + }, + "parentChecks": [ + { + "label": "composed report", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/alpha.test.ts", + "evals/ooo-execution/fixtures/report/beta.test.ts", + "evals/ooo-execution/fixtures/report/gamma.test.ts", + "evals/ooo-execution/fixtures/report/summary.test.ts", + "evals/ooo-execution/fixtures/report/report.test.ts" + ] + } + ], + "worker": { + "kind": "pi", + "provider": "deepseek", + "model": "deepseek-v4-flash" + }, + "fusion": { + "unitsPerSession": 1 + }, + "limits": { + "turns": 6, + "reads": 3, + "timeoutMs": 180000 + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/spec-2.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/spec-2.json new file mode 100644 index 00000000..5dce05db --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/fusion-darm/spec-2.json @@ -0,0 +1,100 @@ +{ + "baseline": [ + "evals/ooo-execution/fixtures/report/interface.ts", + "evals/ooo-execution/fixtures/report/alpha.ts", + "evals/ooo-execution/fixtures/report/beta.ts", + "evals/ooo-execution/fixtures/report/gamma.ts", + "evals/ooo-execution/fixtures/report/summary.ts", + "evals/ooo-execution/fixtures/report/alpha.test.ts", + "evals/ooo-execution/fixtures/report/beta.test.ts", + "evals/ooo-execution/fixtures/report/gamma.test.ts", + "evals/ooo-execution/fixtures/report/summary.test.ts", + "evals/ooo-execution/fixtures/report/report.test.ts" + ], + "plan": [ + { + "id": "alpha", + "effect": "isolated-artifact" + }, + { + "id": "summary", + "effect": "isolated-artifact", + "dependencies": [ + "alpha" + ] + } + ], + "units": { + "alpha": { + "instruction": "Implement alphaSection in this directory so that alpha.test.ts passes. It drops blank rows and sorts the rest, and describes itself with id \"alpha\" and title \"Alpha\". The interface file is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/report/alpha.ts" + ], + "canned": { + "evals/ooo-execution/fixtures/report/alpha.ts": "evals/ooo-execution/fixtures/report/alpha.canned.ts" + }, + "checks": [ + { + "label": "alpha", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/alpha.test.ts" + ] + } + ] + }, + "summary": { + "instruction": "Implement summarySection in this directory so that summary.test.ts passes. It lists one line per section, in the order it was given, each as '- ' plus that section's title, and describes itself with id \"summary\" and title \"Summary\". The three sections it is handed are already accepted; the interface file is frozen: do not change its shape, and do not edit any other file.", + "editable": [ + "evals/ooo-execution/fixtures/report/summary.ts" + ], + "canned": { + "evals/ooo-execution/fixtures/report/summary.ts": "evals/ooo-execution/fixtures/report/summary.canned.ts" + }, + "checks": [ + { + "label": "summary", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/summary.test.ts" + ] + } + ], + "dependencies": [ + "alpha" + ] + } + }, + "parentChecks": [ + { + "label": "composed report", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/alpha.test.ts", + "evals/ooo-execution/fixtures/report/beta.test.ts", + "evals/ooo-execution/fixtures/report/gamma.test.ts", + "evals/ooo-execution/fixtures/report/summary.test.ts", + "evals/ooo-execution/fixtures/report/report.test.ts" + ] + } + ], + "worker": { + "kind": "pi", + "provider": "deepseek", + "model": "deepseek-v4-flash" + }, + "fusion": { + "unitsPerSession": 2 + }, + "limits": { + "turns": 6, + "reads": 3, + "timeoutMs": 180000 + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/harness-three-way.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/harness-three-way.json new file mode 100644 index 00000000..62a274a3 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/harness-three-way.json @@ -0,0 +1,38 @@ +[ + { + "unit": "beta", + "input": "stub (frozen)", + "checkPassed": false, + "detail": " at Test.run (node:internal/test_runner/test:1382:25) at Test.start (node:internal/test_runner/test:1242:17) at startSubtestAfterBootstrap (node:internal/test_runner/harness:387:17) " + }, + { + "unit": "beta", + "input": "canned (known good)", + "checkPassed": true, + "detail": "" + }, + { + "unit": "beta", + "input": "wrong canned", + "checkPassed": false, + "detail": " todo 0 ℹ duration_ms 104.01 ✖ failing tests: test at .temp\\p1-harness\\beta-wrong-canned-6a0c18\\beta.test.ts:1:1 ✖ .temp\\p1-harness\\beta-wrong-canned-6a0c18\\beta.test.ts (98.2447ms) 'test failed' " + }, + { + "unit": "alpha", + "input": "stub (frozen)", + "checkPassed": false, + "detail": " at Test.run (node:internal/test_runner/test:1382:25) at Test.start (node:internal/test_runner/test:1242:17) at startSubtestAfterBootstrap (node:internal/test_runner/harness:387:17) " + }, + { + "unit": "alpha", + "input": "canned (known good)", + "checkPassed": true, + "detail": "" + }, + { + "unit": "alpha", + "input": "wrong canned", + "checkPassed": false, + "detail": ", actual: { id: 'alpha', title: 'Alpha', lines: [ 'b=2', 'a=1' ] }, expected: { id: 'alpha', title: 'Alpha', lines: [ 'a=1', 'b=2' ] }, operator: 'deepStrictEqual', diff: 'simple' } " + } +] diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/smoke/smoke2b.stdout.txt b/docs/experiments/execution/archive/ooo-arms-2026-09-19/smoke/smoke2b.stdout.txt new file mode 100644 index 00000000..1bca141f --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/smoke/smoke2b.stdout.txt @@ -0,0 +1,49 @@ +{ + "plan": [ + "alpha", + "summary" + ], + "order": [ + "alpha", + "summary" + ], + "units": [ + { + "taskId": "alpha", + "verdict": "accepted", + "workerMs": 5211, + "hostMs": 945, + "tokens": 7549, + "attempt": 1 + }, + { + "taskId": "summary", + "verdict": "accepted", + "workerMs": 3560, + "hostMs": 1029, + "tokens": 7340, + "attempt": 1 + } + ], + "accepted": { + "alpha": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(rows: readonly string[]): Section {\\n const lines = rows.filter((row) => row !== \\\"\\\").sort();\\n return { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines };\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(_sections: readonly Section[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summary": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(sections: readonly Section[]): Section {\\n return {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: sections.map((section) => `- ${section.title}`),\\n };\\n}\\n\"}}" + }, + "wallMs": 10750, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [], + "hostMs": 1974, + "hostChecks": 2, + "tokens": 14889, + "failures": 0, + "parent": { + "verdict": "reject", + "files": [ + "alpha", + "summary" + ], + "ms": 1051 + }, + "incomplete": [] +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/smoke/smoke2c.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/smoke/smoke2c.json new file mode 100644 index 00000000..e1bd594e --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/smoke/smoke2c.json @@ -0,0 +1,31 @@ +{ + "measuredAt": "2026-09-19T05:11:45.721Z", + "spec": ".temp/smoke2.spec.json", + "report": { + "plan": [ + "alpha", + "summary" + ], + "order": [ + "alpha" + ], + "units": [], + "accepted": {}, + "wallMs": 6188, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [], + "hostMs": 0, + "hostChecks": 0, + "tokens": 0, + "failures": 1, + "parent": { + "verdict": "reject", + "files": [], + "ms": 971 + }, + "incomplete": [ + "alpha: Pi snapshot task did not finish within its bounded contract: stopReason=error, turns=4, reads=1, artifact=no artifact" + ] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/smoke/smoke2e.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/smoke/smoke2e.json new file mode 100644 index 00000000..0918a5c8 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/smoke/smoke2e.json @@ -0,0 +1,60 @@ +{ + "measuredAt": "2026-09-19T05:12:21.532Z", + "spec": ".temp/smoke2.spec.json", + "report": { + "plan": [ + "alpha", + "summary" + ], + "order": [ + "alpha", + "summary" + ], + "units": [ + { + "taskId": "alpha", + "verdict": "accepted", + "workerMs": 5949, + "hostMs": 944, + "tokens": 11148, + "attempt": 1, + "sessionId": "session:alpha" + }, + { + "taskId": "summary", + "verdict": "accepted", + "workerMs": 1694, + "hostMs": 1021, + "tokens": 9945, + "attempt": 1, + "sessionId": "session:alpha" + } + ], + "accepted": { + "alpha": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(rows: readonly string[]): Section {\\n const lines = rows.filter((row) => row !== \\\"\\\").sort();\\n return { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines };\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(_sections: readonly Section[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summary": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(sections: readonly Section[]): Section {\\n return {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: sections.map((section) => `- ${section.title}`),\\n };\\n}\\n\"}}" + }, + "wallMs": 9615, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "alpha", + "summary" + ] + ], + "hostMs": 1965, + "hostChecks": 2, + "tokens": 21093, + "failures": 0, + "parent": { + "verdict": "reject", + "files": [ + "alpha", + "summary" + ], + "ms": 1109 + }, + "incomplete": [] + } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/smoke/smoke2e.stdout.txt b/docs/experiments/execution/archive/ooo-arms-2026-09-19/smoke/smoke2e.stdout.txt new file mode 100644 index 00000000..1d466426 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/smoke/smoke2e.stdout.txt @@ -0,0 +1,56 @@ +{ + "plan": [ + "alpha", + "summary" + ], + "order": [ + "alpha", + "summary" + ], + "units": [ + { + "taskId": "alpha", + "verdict": "accepted", + "workerMs": 5949, + "hostMs": 944, + "tokens": 11148, + "attempt": 1, + "sessionId": "session:alpha" + }, + { + "taskId": "summary", + "verdict": "accepted", + "workerMs": 1694, + "hostMs": 1021, + "tokens": 9945, + "attempt": 1, + "sessionId": "session:alpha" + } + ], + "accepted": { + "alpha": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(rows: readonly string[]): Section {\\n const lines = rows.filter((row) => row !== \\\"\\\").sort();\\n return { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines };\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(_sections: readonly Section[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\"}}", + "summary": "{\"kind\":\"patch\",\"files\":{\"evals/ooo-execution/fixtures/report/alpha.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\n\\ntest(\\\"alpha lists the rows it was given, sorted, without blanks\\\", () => {\\n assert.deepEqual(alphaSection([\\\"b=2\\\", \\\"\\\", \\\"a=1\\\"]), {\\n id: \\\"alpha\\\",\\n title: \\\"Alpha\\\",\\n lines: [\\\"a=1\\\", \\\"b=2\\\"],\\n });\\n assert.deepEqual(alphaSection([]), { id: \\\"alpha\\\", title: \\\"Alpha\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/alpha.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The alpha section: the rows it was given, sorted, with blanks dropped. */\\nexport function alphaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/beta.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { betaSection } from \\\"./beta.ts\\\";\\n\\ntest(\\\"beta numbers the rows from one, in the order given\\\", () => {\\n assert.deepEqual(betaSection([\\\"x\\\", \\\"y\\\"]), {\\n id: \\\"beta\\\",\\n title: \\\"Beta\\\",\\n lines: [\\\"1. x\\\", \\\"2. y\\\"],\\n });\\n assert.deepEqual(betaSection([\\\"only\\\"]), { id: \\\"beta\\\", title: \\\"Beta\\\", lines: [\\\"1. only\\\"] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/beta.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/gamma.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\n\\ntest(\\\"gamma upper-cases the rows and keeps the first of a duplicate\\\", () => {\\n assert.deepEqual(gammaSection([\\\"a\\\", \\\"b\\\", \\\"a\\\"]), {\\n id: \\\"gamma\\\",\\n title: \\\"Gamma\\\",\\n lines: [\\\"A\\\", \\\"B\\\"],\\n });\\n assert.deepEqual(gammaSection([]), { id: \\\"gamma\\\", title: \\\"Gamma\\\", lines: [] });\\n});\\n\",\"evals/ooo-execution/fixtures/report/gamma.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */\\nexport function gammaSection(_rows: readonly string[]): Section {\\n throw new Error(\\\"not implemented\\\");\\n}\\n\",\"evals/ooo-execution/fixtures/report/interface.ts\":\"/**\\n * The frozen interface of the report this task family builds.\\n *\\n * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only\\n * renderer. A refinement of the work is legal precisely because this file stays as it is - the units\\n * change bodies, never this shape.\\n */\\nexport interface Section {\\n readonly id: string;\\n readonly title: string;\\n readonly lines: readonly string[];\\n}\\n\\n/** The one renderer: a section is its title and its lines, and the composition is the join. */\\nexport function render(section: Section): string {\\n return [`## ${section.title}`, ...section.lines].join(\\\"\\\\n\\\") + \\\"\\\\n\\\";\\n}\\n\",\"evals/ooo-execution/fixtures/report/report.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport { alphaSection } from \\\"./alpha.ts\\\";\\nimport { betaSection } from \\\"./beta.ts\\\";\\nimport { gammaSection } from \\\"./gamma.ts\\\";\\nimport { render } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The composition's own acceptance: the frozen renderer over the composed sections. */\\ntest(\\\"the composed report renders the frozen interface's shape\\\", () => {\\n const report = summarySection([\\n alphaSection([\\\"b=2\\\", \\\"a=1\\\"]),\\n betaSection([\\\"x\\\"]),\\n gammaSection([\\\"a\\\", \\\"a\\\"]),\\n ]);\\n assert.equal(render(report), \\\"## Summary\\\\n- Alpha\\\\n- Beta\\\\n- Gamma\\\\n\\\");\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.test.ts\":\"import assert from \\\"node:assert/strict\\\";\\nimport test from \\\"node:test\\\";\\n\\nimport type { Section } from \\\"./interface.ts\\\";\\nimport { summarySection } from \\\"./summary.ts\\\";\\n\\n/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed\\n * is the composition's business, and the composed check is where the real builders meet. */\\nconst section = (id: string, title: string): Section => ({ id, title, lines: [] });\\n\\ntest(\\\"the summary lists the sections it was given, in the order they were given\\\", () => {\\n assert.deepEqual(summarySection([section(\\\"alpha\\\", \\\"Alpha\\\"), section(\\\"beta\\\", \\\"Beta\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Alpha\\\", \\\"- Beta\\\"],\\n });\\n assert.deepEqual(summarySection([section(\\\"gamma\\\", \\\"Gamma\\\"), section(\\\"alpha\\\", \\\"Alpha\\\")]), {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: [\\\"- Gamma\\\", \\\"- Alpha\\\"],\\n });\\n});\\n\",\"evals/ooo-execution/fixtures/report/summary.ts\":\"import type { Section } from \\\"./interface.ts\\\";\\n\\n/** The summary section: one line per section, in the order the sections were given. Its input is the\\n * other three sections, which is why it can only be built once they exist. */\\nexport function summarySection(sections: readonly Section[]): Section {\\n return {\\n id: \\\"summary\\\",\\n title: \\\"Summary\\\",\\n lines: sections.map((section) => `- ${section.title}`),\\n };\\n}\\n\"}}" + }, + "wallMs": 9615, + "slotsRequested": 1, + "slotsUsed": 1, + "sessions": [ + [ + "alpha", + "summary" + ] + ], + "hostMs": 1965, + "hostChecks": 2, + "tokens": 21093, + "failures": 0, + "parent": { + "verdict": "reject", + "files": [ + "alpha", + "summary" + ], + "ms": 1109 + }, + "incomplete": [] +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/aggregate.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/aggregate.json new file mode 100644 index 00000000..899d8253 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/aggregate.json @@ -0,0 +1,54 @@ +{ + "provider": "deepseek", + "model": "deepseek-v4-flash", + "aggregate": [ + { + "arm": "baseline", + "fact": true, + "runs": 2, + "tokens": 13544, + "workMs": 10358, + "postFactMs": 10358, + "quality": [ + false, + false + ] + }, + { + "arm": "speculation", + "fact": true, + "runs": 2, + "tokens": 17151, + "workMs": 15066, + "postFactMs": 186, + "quality": [ + false, + false + ] + }, + { + "arm": "baseline", + "fact": false, + "runs": 2, + "tokens": 0, + "workMs": 0, + "postFactMs": 0, + "quality": [ + null, + null + ] + }, + { + "arm": "speculation", + "fact": false, + "runs": 2, + "tokens": 12543, + "workMs": 15577, + "postFactMs": 0, + "quality": [ + null, + null + ] + } + ] +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/e-arm.stdout.txt b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/e-arm.stdout.txt new file mode 100644 index 00000000..16f96e19 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/e-arm.stdout.txt @@ -0,0 +1,59 @@ +{"arm":"baseline","fact":true,"rep":1,"tokens":9362,"workMs":6957,"postFactMs":6957,"quality":false} +{"arm":"speculation","fact":true,"rep":1,"outcome":"publish","sessionReusable":true,"tokens":10636,"workMs":8442,"postFactMs":186,"quality":false,"qualityDetail":"2:25)\n at Test.start (node:internal/test_runner/test:1242:17)\n at startSubtestAfterBootstrap (node:internal/test_runner/harness:387:17) {\n generatedMessage: true,\n code: 'ERR_ASSERTION',\n actual: { id: 'beta', title: 'Beta', rows: [ '1. x', '2. y' ] },\n expected: { id: 'beta', title: 'Beta', lines: [ '1. x', '2. y' ] },\n operator: 'deepStrictEqual',\n diff: 'simple'\n }\n"} +{"arm":"baseline","fact":true,"rep":2,"tokens":4182,"workMs":3401,"postFactMs":3401,"quality":false} +{"arm":"speculation","fact":true,"rep":2,"outcome":"publish","sessionReusable":true,"tokens":6515,"workMs":6624,"postFactMs":0,"quality":false,"qualityDetail":"invalid patch structure (artifact keys: digest,kind,conclusion,summary,evidence,citations)"} +{"arm":"baseline","fact":false,"rep":1,"tokens":0,"workMs":0,"postFactMs":0,"quality":null} +{"arm":"speculation","fact":false,"rep":1,"outcome":"discard","sessionReusable":false,"tokens":7677,"workMs":8070,"postFactMs":0,"quality":null} +{"arm":"baseline","fact":false,"rep":2,"tokens":0,"workMs":0,"postFactMs":0,"quality":null} +{"arm":"speculation","fact":false,"rep":2,"outcome":"discard","sessionReusable":false,"tokens":4866,"workMs":7507,"postFactMs":0,"quality":null} +AGGREGATE +[ + { + "arm": "baseline", + "fact": true, + "runs": 2, + "tokens": 13544, + "workMs": 10358, + "postFactMs": 10358, + "quality": [ + false, + false + ] + }, + { + "arm": "speculation", + "fact": true, + "runs": 2, + "tokens": 17151, + "workMs": 15066, + "postFactMs": 186, + "quality": [ + false, + false + ] + }, + { + "arm": "baseline", + "fact": false, + "runs": 2, + "tokens": 0, + "workMs": 0, + "postFactMs": 0, + "quality": [ + null, + null + ] + }, + { + "arm": "speculation", + "fact": false, + "runs": 2, + "tokens": 12543, + "workMs": 15577, + "postFactMs": 0, + "quality": [ + null, + null + ] + } +] diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-1.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-1.json new file mode 100644 index 00000000..db84e837 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-1.json @@ -0,0 +1 @@ +{"arm":"baseline","fact":true,"rep":1,"tokens":9362,"workMs":6957,"postFactMs":6957,"quality":false} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-2.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-2.json new file mode 100644 index 00000000..23daf684 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-2.json @@ -0,0 +1 @@ +{"arm":"speculation","fact":true,"rep":1,"outcome":"publish","sessionReusable":true,"tokens":10636,"workMs":8442,"postFactMs":186,"quality":false,"qualityDetail":"2:25)\n at Test.start (node:internal/test_runner/test:1242:17)\n at startSubtestAfterBootstrap (node:internal/test_runner/harness:387:17) {\n generatedMessage: true,\n code: 'ERR_ASSERTION',\n actual: { id: 'beta', title: 'Beta', rows: [ '1. x', '2. y' ] },\n expected: { id: 'beta', title: 'Beta', lines: [ '1. x', '2. y' ] },\n operator: 'deepStrictEqual',\n diff: 'simple'\n }\n"} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-3.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-3.json new file mode 100644 index 00000000..3b86b45b --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-3.json @@ -0,0 +1 @@ +{"arm":"baseline","fact":true,"rep":2,"tokens":4182,"workMs":3401,"postFactMs":3401,"quality":false} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-4.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-4.json new file mode 100644 index 00000000..67ba3bb5 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-4.json @@ -0,0 +1 @@ +{"arm":"speculation","fact":true,"rep":2,"outcome":"publish","sessionReusable":true,"tokens":6515,"workMs":6624,"postFactMs":0,"quality":false,"qualityDetail":"invalid patch structure (artifact keys: digest,kind,conclusion,summary,evidence,citations)"} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-5.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-5.json new file mode 100644 index 00000000..2b8cec86 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-5.json @@ -0,0 +1 @@ +{"arm":"baseline","fact":false,"rep":1,"tokens":0,"workMs":0,"postFactMs":0,"quality":null} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-6.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-6.json new file mode 100644 index 00000000..0c637561 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-6.json @@ -0,0 +1 @@ +{"arm":"speculation","fact":false,"rep":1,"outcome":"discard","sessionReusable":false,"tokens":7677,"workMs":8070,"postFactMs":0,"quality":null} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-7.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-7.json new file mode 100644 index 00000000..bcb17122 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-7.json @@ -0,0 +1 @@ +{"arm":"baseline","fact":false,"rep":2,"tokens":0,"workMs":0,"postFactMs":0,"quality":null} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-8.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-8.json new file mode 100644 index 00000000..75eea4ce --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run-8.json @@ -0,0 +1 @@ +{"arm":"speculation","fact":false,"rep":2,"outcome":"discard","sessionReusable":false,"tokens":4866,"workMs":7507,"postFactMs":0,"quality":null} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/CLEANABLE-scratch-2026-09-19.md b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/CLEANABLE-scratch-2026-09-19.md new file mode 100644 index 00000000..5ed9ee79 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/CLEANABLE-scratch-2026-09-19.md @@ -0,0 +1,9 @@ +# Scratch - cleanable + +This directory holds one paid run's own evidence: every attempt's artifact bytes, the candidate +tree each check ran in, and the check's output. + +It is scratch and may be deleted once a record quotes its numbers - the run's report is copied +into `docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/` when it is +quoted. It is *not* deleted at the end of the run: that is what made an earlier run's quality +failures unexplainable. diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/aggregate.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/aggregate.json new file mode 100644 index 00000000..48dfc1fb --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/aggregate.json @@ -0,0 +1,58 @@ +{ + "provider": "deepseek", + "model": "deepseek-v4-flash", + "aggregate": [ + { + "arm": "baseline", + "fact": true, + "runs": 3, + "tokens": 18183, + "workMs": 19200, + "postFactMs": 19200, + "quality": [ + false, + false, + false + ] + }, + { + "arm": "speculation", + "fact": true, + "runs": 3, + "tokens": 18602, + "workMs": 19506, + "postFactMs": 175, + "quality": [ + false, + false, + false + ] + }, + { + "arm": "baseline", + "fact": false, + "runs": 3, + "tokens": 0, + "workMs": 0, + "postFactMs": 0, + "quality": [ + null, + null, + null + ] + }, + { + "arm": "speculation", + "fact": false, + "runs": 3, + "tokens": 20332, + "workMs": 23920, + "postFactMs": 0, + "quality": [ + null, + null, + null + ] + } + ] +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/baseline-1-artifact.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/baseline-1-artifact.json new file mode 100644 index 00000000..28a80ed8 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/baseline-1-artifact.json @@ -0,0 +1 @@ +{"digest":"2daa70a8229d241aa3a44ad72a22008ef8cde216c9a4c9b10a756254b72778d2","kind":"conclusion","conclusion":"cannot-complete","summary":"The task requires implementing betaSection so beta.test.ts passes while keeping the frozen interface.ts shape intact. The admission-bounded snapshot exposes only the editable file evals/ooo-execution/fixtures/report/beta.ts; it does not include the contents of interface.ts (the frozen Section type) or beta.test.ts. No filesystem, shell, or search tool is available — only read_snapshot, which returns the same immutable bound blob on every call, and submit_artifact. Consequently the exact Section shape (field names for id/title and the row entry structure for numbered rows) and the passing test expectations are unknowable without guessing. Writing an implementation would require inventing the interface shape, which the task forbids, and a no-change conclusion cannot be supported because no existing test title is resolvable from the provided data. Reporting cannot-complete rather than fabricating an answer.","evidence":"read_snapshot returned a bounded input containing only the editable path beta.ts (currently `throw new Error(\"not implemented\")`). Three read attempts yielded identical content. No tool exists to read interface.ts or beta.test.ts, so the frozen Section interface shape and the test titles/expectations are not available. Budget limits reads to 3, all consumed with no new information.","citations":[]} \ No newline at end of file diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/baseline-2-artifact.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/baseline-2-artifact.json new file mode 100644 index 00000000..51c8d57f --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/baseline-2-artifact.json @@ -0,0 +1 @@ +{"digest":"554966941bc667c6a36731d1684cf846e1c08676372e892d76339feb2a1423e0","kind":"conclusion","conclusion":"cannot-complete","summary":"The task cannot be completed from the frozen snapshot alone. Implementing betaSection requires the exact shape of the frozen `Section` type from ./interface.ts (field names and the row value format) and the assertions of beta.test.ts, neither of which is present in the snapshot. The read_snapshot tool accepts no paths and exposes only beta.ts, so the interface cannot be retrieved. Writing beta.ts without knowing Section's shape would require inventing facts, which the task explicitly forbids, and would be a guess likely to fail beta.test.ts. No safe implementation is derivable, so I am reporting cannot-complete rather than fabricating a shape.","evidence":"Snapshot files map contains only \"evals/ooo-execution/fixtures/report/beta.ts\" (the stub importing `type { Section } from \"./interface.ts\"`). No interface.ts content, no beta.test.ts content, and no Section field definitions are provided. read_snapshot returns the same bounded payload on repeated calls and accepts no path argument, so the frozen interface and test cannot be read. Without the Section shape (e.g. whether it is {id,title,rows} and whether rows are numbered strings or objects) any implementation is an invention rather than a derivation.","citations":[]} \ No newline at end of file diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/baseline-3-artifact.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/baseline-3-artifact.json new file mode 100644 index 00000000..50e449dd --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/baseline-3-artifact.json @@ -0,0 +1 @@ +{"digest":"5405d5ac3392816855b4f404aac767c40f2221258a049243ee5ed64370c79c81","kind":"conclusion","conclusion":"cannot-complete","summary":"Cannot implement betaSection without inventing facts. The only file exposed by read_snapshot is the editable target evals/ooo-execution/fixtures/report/beta.ts. The frozen interface file ./interface.ts (defining the shape of `Section`) and the test file beta.test.ts are not present in the snapshot's `files`, and no paths/commands are accepted by read_snapshot (it returns the same bounded immutable input each call). Writing a correct implementation of betaSection requires knowing the exact `Section` shape (property names, types, and which fields hold `id`, `title`, and the numbered rows) and what beta.test.ts asserts. Any implementation would be a guess, which the task explicitly forbids (\"Do not invent facts\"). Additionally, a no-change/citation-based conclusion is impossible because no existing test title is resolvable from the provided data. Therefore the requirement cannot be satisfied given the admitted inputs.","evidence":"read_snapshot returned digest 5405d5ac3392816855b4f404aac767c40f2221258a049243ee5ed64370c79c81 with files containing exactly one entry: evals/ooo-execution/fixtures/report/beta.ts = `import type { Section } from \"./interface.ts\";\\n\\n/** The beta section: the rows it was given, numbered from one, in the order given. */\\nexport function betaSection(_rows: readonly string[]): Section {\\n throw new Error(\"not implemented\");\\n}\\n`. The imported ./interface.ts and beta.test.ts are absent from the snapshot. read_snapshot accepts no path or command parameters and is immutable, so no additional file contents (the frozen `Section` interface shape, or the test assertions/titles) can be obtained. Without the `Section` shape, betaSection cannot be implemented without inventing property names/types.","citations":[]} \ No newline at end of file diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/alpha-wrong.canned.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/alpha-wrong.canned.ts new file mode 100644 index 00000000..14d0dcf4 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/alpha-wrong.canned.ts @@ -0,0 +1,8 @@ +import type { Section } from "./interface.ts"; + +/** A wrong answer, not a stub: it sorts descending where the frozen check says ascending, so a run + * that accepts it would be accepting on something other than the check. */ +export function alphaSection(rows: readonly string[]): Section { + const kept = rows.filter((row) => row.trim() !== "").sort((a, b) => b.localeCompare(a)); + return { id: "alpha", title: "Alpha", lines: kept }; +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/alpha.canned.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/alpha.canned.ts new file mode 100644 index 00000000..32c3664d --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/alpha.canned.ts @@ -0,0 +1,13 @@ +/** + * The reference answer for the alpha unit, used by the canned worker: the instrument has to show the + * task family accepts a correct submission before a model is paid to produce one. + */ +import type { Section } from "./interface.ts"; + +export function alphaSection(rows: readonly string[]): Section { + return { + id: "alpha", + title: "Alpha", + lines: rows.filter((row) => row !== "").slice().sort(), + }; +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/alpha.test.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/alpha.test.ts new file mode 100644 index 00000000..dcaf24b0 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/alpha.test.ts @@ -0,0 +1,13 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { alphaSection } from "./alpha.ts"; + +test("alpha lists the rows it was given, sorted, without blanks", () => { + assert.deepEqual(alphaSection(["b=2", "", "a=1"]), { + id: "alpha", + title: "Alpha", + lines: ["a=1", "b=2"], + }); + assert.deepEqual(alphaSection([]), { id: "alpha", title: "Alpha", lines: [] }); +}); diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/alpha.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/alpha.ts new file mode 100644 index 00000000..8a829712 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/alpha.ts @@ -0,0 +1,6 @@ +import type { Section } from "./interface.ts"; + +/** The alpha section: the rows it was given, sorted, with blanks dropped. */ +export function alphaSection(_rows: readonly string[]): Section { + throw new Error("not implemented"); +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/beta.canned.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/beta.canned.ts new file mode 100644 index 00000000..b2a7db28 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/beta.canned.ts @@ -0,0 +1,5 @@ +import type { Section } from "./interface.ts"; + +export function betaSection(rows: readonly string[]): Section { + return { id: "beta", title: "Beta", lines: rows.map((row, index) => `${index + 1}. ${row}`) }; +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/beta.test.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/beta.test.ts new file mode 100644 index 00000000..d2e76e1f --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/beta.test.ts @@ -0,0 +1,13 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { betaSection } from "./beta.ts"; + +test("beta numbers the rows from one, in the order given", () => { + assert.deepEqual(betaSection(["x", "y"]), { + id: "beta", + title: "Beta", + lines: ["1. x", "2. y"], + }); + assert.deepEqual(betaSection(["only"]), { id: "beta", title: "Beta", lines: ["1. only"] }); +}); diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/beta.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/beta.ts new file mode 100644 index 00000000..05e0f707 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/beta.ts @@ -0,0 +1,10 @@ +import type { Section } from "./interface.ts"; + +/** The beta section: the rows it was given, numbered from one, in the order given. */ +export function betaSection(rows: readonly string[]): Section { + return { + id: "beta", + title: "Beta", + rows: rows.map((row, index) => `${index + 1}. ${row}`), + }; +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/coarse.spec.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/coarse.spec.json new file mode 100644 index 00000000..c5e2a725 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/coarse.spec.json @@ -0,0 +1,77 @@ +{ + "baseline": [ + "evals/ooo-execution/fixtures/report/interface.ts", + "evals/ooo-execution/fixtures/report/alpha.ts", + "evals/ooo-execution/fixtures/report/beta.ts", + "evals/ooo-execution/fixtures/report/gamma.ts", + "evals/ooo-execution/fixtures/report/summary.ts", + "evals/ooo-execution/fixtures/report/alpha.test.ts", + "evals/ooo-execution/fixtures/report/beta.test.ts", + "evals/ooo-execution/fixtures/report/gamma.test.ts", + "evals/ooo-execution/fixtures/report/summary.test.ts", + "evals/ooo-execution/fixtures/report/report.test.ts" + ], + "plan": [ + { "id": "report", "effect": "isolated-artifact" } + ], + "units": { + "report": { + "instruction": "Implement the three section builders and the summary in this directory so that all five frozen test files pass. The interface file is frozen: do not change its shape. alphaSection drops blank rows and sorts the rest. betaSection numbers the rows from one in the order given. gammaSection upper-cases the rows and keeps the first of a duplicate. summarySection lists one line per section, in the order it was given, each as '- ' plus that section's title, and describes itself with id \"summary\" and title \"Summary\".", + "editable": [ + "evals/ooo-execution/fixtures/report/alpha.ts", + "evals/ooo-execution/fixtures/report/beta.ts", + "evals/ooo-execution/fixtures/report/gamma.ts", + "evals/ooo-execution/fixtures/report/summary.ts" + ], + "canned": { + "evals/ooo-execution/fixtures/report/alpha.ts": "evals/ooo-execution/fixtures/report/alpha.canned.ts", + "evals/ooo-execution/fixtures/report/beta.ts": "evals/ooo-execution/fixtures/report/beta.canned.ts", + "evals/ooo-execution/fixtures/report/gamma.ts": "evals/ooo-execution/fixtures/report/gamma.canned.ts", + "evals/ooo-execution/fixtures/report/summary.ts": "evals/ooo-execution/fixtures/report/summary.canned.ts" + } + } + }, + "checks": [ + { + "label": "alpha", + "command": "node", + "args": ["--experimental-strip-types", "--test", "evals/ooo-execution/fixtures/report/alpha.test.ts"] + }, + { + "label": "beta", + "command": "node", + "args": ["--experimental-strip-types", "--test", "evals/ooo-execution/fixtures/report/beta.test.ts"] + }, + { + "label": "gamma", + "command": "node", + "args": ["--experimental-strip-types", "--test", "evals/ooo-execution/fixtures/report/gamma.test.ts"] + }, + { + "label": "summary", + "command": "node", + "args": ["--experimental-strip-types", "--test", "evals/ooo-execution/fixtures/report/summary.test.ts"] + }, + { + "label": "report", + "command": "node", + "args": ["--experimental-strip-types", "--test", "evals/ooo-execution/fixtures/report/report.test.ts"] + } + ], + "parentChecks": [ + { + "label": "composed report", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/alpha.test.ts", + "evals/ooo-execution/fixtures/report/beta.test.ts", + "evals/ooo-execution/fixtures/report/gamma.test.ts", + "evals/ooo-execution/fixtures/report/summary.test.ts", + "evals/ooo-execution/fixtures/report/report.test.ts" + ] + } + ], + "worker": { "kind": "canned" } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/fine.spec.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/fine.spec.json new file mode 100644 index 00000000..a5a1b4e3 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/fine.spec.json @@ -0,0 +1,110 @@ +{ + "baseline": [ + "evals/ooo-execution/fixtures/report/interface.ts", + "evals/ooo-execution/fixtures/report/alpha.ts", + "evals/ooo-execution/fixtures/report/beta.ts", + "evals/ooo-execution/fixtures/report/gamma.ts", + "evals/ooo-execution/fixtures/report/summary.ts", + "evals/ooo-execution/fixtures/report/alpha.test.ts", + "evals/ooo-execution/fixtures/report/beta.test.ts", + "evals/ooo-execution/fixtures/report/gamma.test.ts", + "evals/ooo-execution/fixtures/report/summary.test.ts", + "evals/ooo-execution/fixtures/report/report.test.ts" + ], + "plan": [ + { "id": "alpha", "effect": "isolated-artifact" }, + { "id": "beta", "effect": "isolated-artifact" }, + { "id": "gamma", "effect": "isolated-artifact" }, + { "id": "summary", "effect": "isolated-artifact", "dependencies": ["alpha", "beta", "gamma"] } + ], + "units": { + "alpha": { + "instruction": "Implement alphaSection in this directory so that alpha.test.ts passes. It drops blank rows and sorts the rest, and describes itself with id \"alpha\" and title \"Alpha\". The interface file is frozen: do not change its shape, and do not edit any other file.", + "editable": ["evals/ooo-execution/fixtures/report/alpha.ts"], + "canned": { + "evals/ooo-execution/fixtures/report/alpha.ts": "evals/ooo-execution/fixtures/report/alpha.canned.ts" + }, + "checks": [ + { + "label": "alpha", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/alpha.test.ts" + ] + } + ] + }, + "beta": { + "instruction": "Implement betaSection in this directory so that beta.test.ts passes. It numbers the rows from one in the order given, and describes itself with id \"beta\" and title \"Beta\". The interface file is frozen: do not change its shape, and do not edit any other file.", + "editable": ["evals/ooo-execution/fixtures/report/beta.ts"], + "canned": { + "evals/ooo-execution/fixtures/report/beta.ts": "evals/ooo-execution/fixtures/report/beta.canned.ts" + }, + "checks": [ + { + "label": "beta", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/beta.test.ts" + ] + } + ] + }, + "gamma": { + "instruction": "Implement gammaSection in this directory so that gamma.test.ts passes. It upper-cases the rows and keeps the first of a duplicate, and describes itself with id \"gamma\" and title \"Gamma\". The interface file is frozen: do not change its shape, and do not edit any other file.", + "editable": ["evals/ooo-execution/fixtures/report/gamma.ts"], + "canned": { + "evals/ooo-execution/fixtures/report/gamma.ts": "evals/ooo-execution/fixtures/report/gamma.canned.ts" + }, + "checks": [ + { + "label": "gamma", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/gamma.test.ts" + ] + } + ] + }, + "summary": { + "instruction": "Implement summarySection in this directory so that summary.test.ts passes. It lists one line per section, in the order it was given, each as '- ' plus that section's title, and describes itself with id \"summary\" and title \"Summary\". The three sections it is handed are already accepted; the interface file is frozen: do not change its shape, and do not edit any other file.", + "editable": ["evals/ooo-execution/fixtures/report/summary.ts"], + "canned": { + "evals/ooo-execution/fixtures/report/summary.ts": "evals/ooo-execution/fixtures/report/summary.canned.ts" + }, + "checks": [ + { + "label": "summary", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/summary.test.ts" + ] + } + ] + } + }, + "parentChecks": [ + { + "label": "composed report", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/alpha.test.ts", + "evals/ooo-execution/fixtures/report/beta.test.ts", + "evals/ooo-execution/fixtures/report/gamma.test.ts", + "evals/ooo-execution/fixtures/report/summary.test.ts", + "evals/ooo-execution/fixtures/report/report.test.ts" + ] + } + ], + "worker": { "kind": "canned" } +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/gamma.canned.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/gamma.canned.ts new file mode 100644 index 00000000..f642891a --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/gamma.canned.ts @@ -0,0 +1,10 @@ +import type { Section } from "./interface.ts"; + +export function gammaSection(rows: readonly string[]): Section { + const lines: string[] = []; + for (const row of rows) { + const upper = row.toUpperCase(); + if (!lines.includes(upper)) lines.push(upper); + } + return { id: "gamma", title: "Gamma", lines }; +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/gamma.test.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/gamma.test.ts new file mode 100644 index 00000000..763c04b0 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/gamma.test.ts @@ -0,0 +1,13 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { gammaSection } from "./gamma.ts"; + +test("gamma upper-cases the rows and keeps the first of a duplicate", () => { + assert.deepEqual(gammaSection(["a", "b", "a"]), { + id: "gamma", + title: "Gamma", + lines: ["A", "B"], + }); + assert.deepEqual(gammaSection([]), { id: "gamma", title: "Gamma", lines: [] }); +}); diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/gamma.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/gamma.ts new file mode 100644 index 00000000..e0317815 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/gamma.ts @@ -0,0 +1,6 @@ +import type { Section } from "./interface.ts"; + +/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */ +export function gammaSection(_rows: readonly string[]): Section { + throw new Error("not implemented"); +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/interface.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/interface.ts new file mode 100644 index 00000000..d96c3f07 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/interface.ts @@ -0,0 +1,17 @@ +/** + * The frozen interface of the report this task family builds. + * + * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only + * renderer. A refinement of the work is legal precisely because this file stays as it is - the units + * change bodies, never this shape. + */ +export interface Section { + readonly id: string; + readonly title: string; + readonly lines: readonly string[]; +} + +/** The one renderer: a section is its title and its lines, and the composition is the join. */ +export function render(section: Section): string { + return [`## ${section.title}`, ...section.lines].join("\n") + "\n"; +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/report.test.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/report.test.ts new file mode 100644 index 00000000..b79cd74d --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/report.test.ts @@ -0,0 +1,18 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { alphaSection } from "./alpha.ts"; +import { betaSection } from "./beta.ts"; +import { gammaSection } from "./gamma.ts"; +import { render } from "./interface.ts"; +import { summarySection } from "./summary.ts"; + +/** The composition's own acceptance: the frozen renderer over the composed sections. */ +test("the composed report renders the frozen interface's shape", () => { + const report = summarySection([ + alphaSection(["b=2", "a=1"]), + betaSection(["x"]), + gammaSection(["a", "a"]), + ]); + assert.equal(render(report), "## Summary\n- Alpha\n- Beta\n- Gamma\n"); +}); diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/summary.canned.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/summary.canned.ts new file mode 100644 index 00000000..4c1a4b66 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/summary.canned.ts @@ -0,0 +1,5 @@ +import type { Section } from "./interface.ts"; + +export function summarySection(sections: readonly Section[]): Section { + return { id: "summary", title: "Summary", lines: sections.map((section) => `- ${section.title}`) }; +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/summary.test.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/summary.test.ts new file mode 100644 index 00000000..2e468dd2 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/summary.test.ts @@ -0,0 +1,22 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import type { Section } from "./interface.ts"; +import { summarySection } from "./summary.ts"; + +/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed + * is the composition's business, and the composed check is where the real builders meet. */ +const section = (id: string, title: string): Section => ({ id, title, lines: [] }); + +test("the summary lists the sections it was given, in the order they were given", () => { + assert.deepEqual(summarySection([section("alpha", "Alpha"), section("beta", "Beta")]), { + id: "summary", + title: "Summary", + lines: ["- Alpha", "- Beta"], + }); + assert.deepEqual(summarySection([section("gamma", "Gamma"), section("alpha", "Alpha")]), { + id: "summary", + title: "Summary", + lines: ["- Gamma", "- Alpha"], + }); +}); diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/summary.ts b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/summary.ts new file mode 100644 index 00000000..fcfe9d3b --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/summary.ts @@ -0,0 +1,7 @@ +import type { Section } from "./interface.ts"; + +/** The summary section: one line per section, in the order the sections were given. Its input is the + * other three sections, which is why it can only be built once they exist. */ +export function summarySection(_sections: readonly Section[]): Section { + throw new Error("not implemented"); +} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-1.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-1.json new file mode 100644 index 00000000..68303acb --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-1.json @@ -0,0 +1 @@ +{"arm":"baseline","fact":true,"rep":1,"runId":"2026-09-19T05-40-04-311Z","tokens":6563,"turns":5,"workMs":6188,"postFactMs":6188,"frozenDigest":"2daa70a8229d241aa3a44ad72a22008ef8cde216c9a4c9b10a756254b72778d2","interfaceDigest":"fd651af7eefe0417","artifactKind":"conclusion","artifactPath":".temp/e-arm/evidence/2026-09-19T05-40-04-311Z/baseline-1-artifact.json","quality":false,"qualityKind":"conclusion","qualityDetail":"a conclusion carries no files: the unit's check cannot pass on it"} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-10.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-10.json new file mode 100644 index 00000000..708b1832 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-10.json @@ -0,0 +1 @@ +{"arm":"speculation","fact":false,"rep":2,"runId":"2026-09-19T05-40-04-311Z","outcome":"discard","sessionReusable":false,"ticketId":"speculation-2","tokens":4072,"turns":4,"workMs":2927,"frozenDigest":"e05c1f34829871f4f609b61e53acbf9bec7d8650266f631f27f31c4e73877cdb","interfaceDigest":"fd651af7eefe0417","artifactKind":"patch","artifactPath":".temp/e-arm/evidence/2026-09-19T05-40-04-311Z/speculation-2-artifact.json","postFactMs":0,"quality":null,"qualityKind":null} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-11.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-11.json new file mode 100644 index 00000000..94c0405c --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-11.json @@ -0,0 +1 @@ +{"arm":"baseline","fact":false,"rep":3,"runId":"2026-09-19T05-40-04-311Z","tokens":0,"turns":0,"workMs":0,"postFactMs":0,"quality":null,"qualityKind":null} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-12.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-12.json new file mode 100644 index 00000000..ad804118 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-12.json @@ -0,0 +1 @@ +{"arm":"speculation","fact":false,"rep":3,"runId":"2026-09-19T05-40-04-311Z","outcome":"discard","sessionReusable":false,"ticketId":"speculation-3","tokens":7285,"turns":5,"workMs":7315,"frozenDigest":"bb1687575ac4d21f2b7dbbb57ed2cc080f1b5bd41f7eb928be426f107acb361f","interfaceDigest":"fd651af7eefe0417","artifactKind":"patch","artifactPath":".temp/e-arm/evidence/2026-09-19T05-40-04-311Z/speculation-3-artifact.json","postFactMs":0,"quality":null,"qualityKind":null} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-2.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-2.json new file mode 100644 index 00000000..2babab59 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-2.json @@ -0,0 +1 @@ +{"arm":"speculation","fact":true,"rep":1,"runId":"2026-09-19T05-40-04-311Z","outcome":"publish","sessionReusable":true,"ticketId":"speculation-1","tokens":7529,"turns":5,"workMs":8332,"frozenDigest":"e8ed925c00af611acfec4f5726ac9da6a349b2e7f30c08f05162b989bcfcb8f0","interfaceDigest":"fd651af7eefe0417","artifactKind":"conclusion","artifactPath":".temp/e-arm/evidence/2026-09-19T05-40-04-311Z/speculation-1-artifact.json","postFactMs":0,"quality":false,"qualityKind":"conclusion","qualityDetail":"a conclusion carries no files: the unit's check cannot pass on it"} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-3.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-3.json new file mode 100644 index 00000000..0c88314d --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-3.json @@ -0,0 +1 @@ +{"arm":"baseline","fact":true,"rep":2,"runId":"2026-09-19T05-40-04-311Z","tokens":7035,"turns":5,"workMs":7083,"postFactMs":7083,"frozenDigest":"554966941bc667c6a36731d1684cf846e1c08676372e892d76339feb2a1423e0","interfaceDigest":"fd651af7eefe0417","artifactKind":"conclusion","artifactPath":".temp/e-arm/evidence/2026-09-19T05-40-04-311Z/baseline-2-artifact.json","quality":false,"qualityKind":"conclusion","qualityDetail":"a conclusion carries no files: the unit's check cannot pass on it"} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-4.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-4.json new file mode 100644 index 00000000..afdbb25a --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-4.json @@ -0,0 +1 @@ +{"arm":"speculation","fact":true,"rep":2,"runId":"2026-09-19T05-40-04-311Z","outcome":"publish","sessionReusable":true,"ticketId":"speculation-2","tokens":4354,"turns":4,"workMs":5027,"frozenDigest":"ed7aeaa4b4b312f004f93352123ef88a17a86c2d2831d77bc9ee4e2f1f2c605d","interfaceDigest":"fd651af7eefe0417","artifactKind":"conclusion","artifactPath":".temp/e-arm/evidence/2026-09-19T05-40-04-311Z/speculation-2-artifact.json","postFactMs":0,"quality":false,"qualityKind":"conclusion","qualityDetail":"a conclusion carries no files: the unit's check cannot pass on it"} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-5.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-5.json new file mode 100644 index 00000000..8bec1025 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-5.json @@ -0,0 +1 @@ +{"arm":"baseline","fact":true,"rep":3,"runId":"2026-09-19T05-40-04-311Z","tokens":4585,"turns":4,"workMs":5929,"postFactMs":5929,"frozenDigest":"5405d5ac3392816855b4f404aac767c40f2221258a049243ee5ed64370c79c81","interfaceDigest":"fd651af7eefe0417","artifactKind":"conclusion","artifactPath":".temp/e-arm/evidence/2026-09-19T05-40-04-311Z/baseline-3-artifact.json","quality":false,"qualityKind":"conclusion","qualityDetail":"a conclusion carries no files: the unit's check cannot pass on it"} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-6.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-6.json new file mode 100644 index 00000000..428bc637 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-6.json @@ -0,0 +1 @@ +{"arm":"speculation","fact":true,"rep":3,"runId":"2026-09-19T05-40-04-311Z","outcome":"publish","sessionReusable":true,"ticketId":"speculation-3","tokens":6719,"turns":5,"workMs":6147,"frozenDigest":"eed5e0d055532f9d5452c1be66873d12a814927cb66512964bd9360de546b82e","interfaceDigest":"fd651af7eefe0417","artifactKind":"patch","artifactPath":".temp/e-arm/evidence/2026-09-19T05-40-04-311Z/speculation-3-artifact.json","postFactMs":175,"quality":false,"qualityKind":"patch","candidatePath":".temp/e-arm/evidence/2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3","checkMs":175,"qualityDetail":"✖ beta numbers the rows from one, in the order given (1.5815ms)\nℹ tests 1\nℹ suites 0\nℹ pass 0\nℹ fail 1\nℹ cancelled 0\nℹ skipped 0\nℹ todo 0\nℹ duration_ms 102.2731\n\n✖ failing tests:\n\ntest at .temp\\e-arm\\evidence\\2026-09-19T05-40-04-311Z\\candidate-speculation-3-c39e7ba3\\beta.test.ts:6:1\n✖ beta numbers the rows from one, in the order given (1.5815ms)\n AssertionError [ERR_ASSERTION]: Expected values to be strictly deep-equal:\n + actual - expected\n \n {\n id: 'beta',\n + rows: [\n - lines: [\n '1. x',\n '2. y'\n ],\n title: 'Beta'\n }\n \n at TestContext. (file:///C:/Documents/GitHub/NodeMemoryGraph-ooo/.temp/e-arm/evidence/2026-09-19T05-40-04-311Z/candidate-speculation-3-c39e7ba3/beta.test.ts:7:10)\n at Test.runInAsyncScope (node:async_hooks:227:14)\n at Test.run (node:internal/test_runner/test:1382:25)\n at Test.start (node:internal/test_runner/test:1242:17)\n at startSubtestAfterBootstrap (node:internal/test_runner/harness:387:17) {\n generatedMessage: true,\n code: 'ERR_ASSERTION',\n actual: { id: 'beta', title: 'Beta', rows: [ '1. x', '2. y' ] },\n expected: { id: 'beta', title: 'Beta', lines: [ '1. x', '2. y' ] },\n operator: 'deepStrictEqual',\n diff: 'simple'\n }\n"} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-7.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-7.json new file mode 100644 index 00000000..182fb8a0 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-7.json @@ -0,0 +1 @@ +{"arm":"baseline","fact":false,"rep":1,"runId":"2026-09-19T05-40-04-311Z","tokens":0,"turns":0,"workMs":0,"postFactMs":0,"quality":null,"qualityKind":null} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-8.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-8.json new file mode 100644 index 00000000..89f86bce --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-8.json @@ -0,0 +1 @@ +{"arm":"speculation","fact":false,"rep":1,"runId":"2026-09-19T05-40-04-311Z","outcome":"discard","sessionReusable":false,"ticketId":"speculation-1","tokens":8975,"turns":5,"workMs":13678,"frozenDigest":"f171af8d65d14dfb47c58ceb290b0d8c653e6a95f42d37a8220d4f7cddfad20f","interfaceDigest":"fd651af7eefe0417","artifactKind":"conclusion","artifactPath":".temp/e-arm/evidence/2026-09-19T05-40-04-311Z/speculation-1-artifact.json","postFactMs":0,"quality":null,"qualityKind":null} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-9.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-9.json new file mode 100644 index 00000000..136db9b7 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/row-9.json @@ -0,0 +1 @@ +{"arm":"baseline","fact":false,"rep":2,"runId":"2026-09-19T05-40-04-311Z","tokens":0,"turns":0,"workMs":0,"postFactMs":0,"quality":null,"qualityKind":null} diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/speculation-1-artifact.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/speculation-1-artifact.json new file mode 100644 index 00000000..2a03c6c8 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/speculation-1-artifact.json @@ -0,0 +1 @@ +{"digest":"f171af8d65d14dfb47c58ceb290b0d8c653e6a95f42d37a8220d4f7cddfad20f","kind":"conclusion","conclusion":"cannot-complete","summary":"Cannot implement betaSection without inventing facts. The bounded snapshot provided only the editable file evals/ooo-execution/fixtures/report/beta.ts (a stub importing `type { Section } from \"./interface.ts\"`). The frozen interface file interface.ts and the test file beta.test.ts were NOT included in the snapshot, so the exact shape of `Section` is unknown: in particular I cannot determine (a) the property names other than the spec-stated id \"beta\" / title \"Beta\", (b) whether the rows field is named `rows` and (c) whether each numbered row is a string, a `{ number, text }` object, or some other struct, nor whether numbering is 1-based under a specific key. Any implementation would require guessing these field names, which is prohibited (\"Do not invent facts\") and would likely fail against the real frozen interface. No test titles are resolvable either (beta.test.ts absent), so a no-change conclusion cannot meet its citation requirement. Reads were bounded to 3 and returned identical content, so the interface/test shapes are unresolvable from the provided input.","evidence":"Snapshot (digest f171af8d65d14dfb47c58ceb290b0d8c653e6a95f42d37a8220d4f7cddfad20f) files map contains exactly one entry: \"evals/ooo-execution/fixtures/report/beta.ts\" with content `import type { Section } from \"./interface.ts\"; export function betaSection(_rows: readonly string[]): Section { throw new Error(\"not implemented\"); }`. The referenced \"./interface.ts\" and the test \"beta.test.ts\" are neither listed under files nor under editable, and the tool accepts no paths, so their contents are unavailable. Spec text gives id \"beta\", title \"Beta\", and rows numbered from one in given order, but not the interface's concrete property names/types needed to type-check a return value.","citations":[]} \ No newline at end of file diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/speculation-2-artifact.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/speculation-2-artifact.json new file mode 100644 index 00000000..f11159ba --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/speculation-2-artifact.json @@ -0,0 +1 @@ +{"digest":"e05c1f34829871f4f609b61e53acbf9bec7d8650266f631f27f31c4e73877cdb","files":[{"content":"import type { Section } from \"./interface.ts\";\n\n/** The beta section: the rows it was given, numbered from one, in the order given. */\nexport function betaSection(rows: readonly string[]): Section {\n return {\n id: \"beta\",\n title: \"Beta\",\n rows: rows.map((row, index) => `${index + 1}. ${row}`),\n };\n}\n","path":"evals/ooo-execution/fixtures/report/beta.ts"}]} \ No newline at end of file diff --git a/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/speculation-3-artifact.json b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/speculation-3-artifact.json new file mode 100644 index 00000000..7dac2be0 --- /dev/null +++ b/docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/run2-2026-09-19T05-40-04-311Z/speculation-3-artifact.json @@ -0,0 +1 @@ +{"digest":"bb1687575ac4d21f2b7dbbb57ed2cc080f1b5bd41f7eb928be426f107acb361f","files":[{"path":"evals/ooo-execution/fixtures/report/beta.ts","content":"import type { Section } from \"./interface.ts\";\n\n/** The beta section: the rows it was given, numbered from one, in the order given. */\nexport function betaSection(rows: readonly string[]): Section {\n return {\n id: \"beta\",\n title: \"Beta\",\n rows: rows.map((text, index) => ({ number: index + 1, text })),\n };\n}\n"}]} \ No newline at end of file diff --git a/docs/experiments/execution/archive/ooo-continuation-2026-09-14/README.md b/docs/experiments/execution/archive/ooo-continuation-2026-09-14/README.md index 528d1e17..89f5266d 100644 --- a/docs/experiments/execution/archive/ooo-continuation-2026-09-14/README.md +++ b/docs/experiments/execution/archive/ooo-continuation-2026-09-14/README.md @@ -55,7 +55,7 @@ the cost comparison in the sibling documents. - `../ooo-real-continuation-2026-09-14.md` - the G7 check itself: five tasks, one boundary each. - `../ooo-real-continuation-comparison-2026-09-14.md` - the comparison of the stable first stage against the continuations, and the failure classification this archive supports. -- `../../../../experiments/execution/ooo-contract-obligations.md` - the ledger; row G7. +- `../../../../design/task-unit-semantics-obligations.md` - the ledger; row G7. ## What this archive does not hold diff --git a/docs/experiments/execution/ooo-acceptance-predicate-2026-09-13.md b/docs/experiments/execution/ooo-acceptance-predicate-2026-09-13.md index bdeadf20..f0147e10 100644 --- a/docs/experiments/execution/ooo-acceptance-predicate-2026-09-13.md +++ b/docs/experiments/execution/ooo-acceptance-predicate-2026-09-13.md @@ -1,9 +1,9 @@ # One acceptance predicate, adopted by the round **Related:** [task-unit semantics design](../../design/task-unit-semantics.md) · -[proposed decision record](../../decisions/proposed/2026-09-13-task-unit-semantics.md) · +[proposed decision record](../../decisions/implemented/2026-09-13-task-unit-semantics.md) · [slice 1: compiler and model](task-semantics-slice1-2026-09-13.md) · -[board deliverable/verdict decision](../../decisions/proposed/2026-09-06-board-governance-addressing.md) +[board deliverable/verdict decision](../../decisions/implemented/2026-09-06-board-governance-addressing.md) Measured 2026-09-13. Zero model tokens: every number below comes from the deterministic round suites and the mutation teeth, with no provider contacted. diff --git a/docs/experiments/execution/ooo-arm-plan-2026-09-19.md b/docs/experiments/execution/ooo-arm-plan-2026-09-19.md new file mode 100644 index 00000000..8edb147d --- /dev/null +++ b/docs/experiments/execution/ooo-arm-plan-2026-09-19.md @@ -0,0 +1,127 @@ +# Plan: what is left of the arm programme, and the data each step needs + +**Status:** a plan, not a measurement. It exists because two paid runs were reported and their evidence +was either deleted or never written: the numbers in +[the arms record](./ooo-arms-pilot-2026-09-18.md) outlived nothing. + +**Related:** [the arms record](./ooo-arms-pilot-2026-09-18.md) · +[the rescued samples](./archive/ooo-arms-2026-09-19/README.md) · +[the obligation ledger](../../design/task-unit-semantics-obligations.md) + +## The rule this plan exists to keep + +A step below may run only once its data list is fixed: which fields, where they are written, and what is +read off them. A field a claim needs is written *before* the claim, into +`docs/experiments/execution/archive//`, which git tracks. No run deletes its own evidence; `.temp/` +holds working copies only, and a result a sentence depends on is not a working copy. + +This is the second time on this branch that the rule had to be learned. Row G7's samples were rescued +into [`archive/ooo-continuation-2026-09-14/`](./archive/ooo-continuation-2026-09-14/README.md), whose +README states the failure mode exactly; the D and E arm runs repeated it, and were rescued unread into +[`archive/ooo-arms-2026-09-19/`](./archive/ooo-arms-2026-09-19/README.md). A lesson recorded in prose +in one directory did not survive two sessions, which is why P4 below asks for a check instead. + +## P1 - Why did every published candidate fail its own check? (offline; one unit, ~11 k tokens) + +**Question.** The E arm's first run reported `quality: false` for all four verified candidates, so by the +design's own rule it could claim nothing. The report cannot be acted on, because the run stored no bytes. + +**Data to collect.** One fresh attempt through `evals/ooo-execution/speculation-pilot.ts`, writing per +attempt: `artifactId` (sha256 of the artifact bytes), `artifactPath`, `candidatePath` (the kept tree), +`checkStdout`, `checkStderr`, `checkExit`, `checkMs`, `frozenDigest`, `interfaceDigest`, `tokens`, +`turns`, `workMs`, `instruction`. + +**Analysis.** One three-way table, all three run by hand against the same check: the frozen stub, the +model's candidate, and the fixture's own canned answer (known good), plus a diff of the candidate against +the stub. + +**Decisive.** The table itself. Canned passes and the model's fails: `quality: false` is a result about +the model's work - an unverified guess produced a bad patch - and the arm's next question is about +verification, not about the harness. If the *canned* answer also fails, the harness is broken and nothing +else may be concluded from the first run. + +## P2 - The E arm's economics (paid; at most 150 k tokens, 3 reps per condition) + +**Data to collect.** Everything in P1, plus `arm` (baseline | speculation), `fact` (true | false), `rep`, +`outcome` (from `speculationOutcome`), `sessionReusable`, `ticketId`, `sessionId`, `reusedSession`, +`verifyMs`, `postFactMs`, `wasteTokens` (speculation's spend when the fact turned out false). Aggregates +are medians and ranges per (arm, fact), never a single run. + +**Analysis.** The design's three terms kept apart rather than netted: latency saved when the fact held, +tokens wasted when it did not, and the quality term for every published candidate. Break-even hit rate +`p* = waste / (saved + waste)` over the measured numbers, reported as a threshold with its sample size. + +**Decisive.** Quality parity inside the compared cells is a precondition, not a result: if a published +candidate fails its check, the arm reports that as its outcome and claims no speedup. Otherwise +feasibility is settled by one cell in which a published candidate *is* verified and its post-fact +latency (verification included) is below the control's post-fact work. The hit-rate threshold is the +analysis, and no further spend is needed to state it. + +## P3 - landed 2026-09-19, one half of it + +The rule that matters is now pinned in : an artifact is read +by its kind, and the patch reader refuses a conclusion by name. The mutant half is not done - the host +reader lives in , which is not a mutation target, and adding a target is a +sweep of its own rather than a line in this plan. The claim that the extension was laxer than the host +was wrong and is corrected in three places (the arms record, the ledger and the archive README). + +## P3 (original statement) - The envelope's two readers must agree (offline, no model) + +**The defect.** The extension accepted an artifact whose keys were +`digest, kind, conclusion, summary, evidence, citations` with no `files`; `patchCandidate` refused the +same bytes as "invalid patch structure". A submission the adapter admits and the board can never accept. + +**Data to collect.** An acceptance matrix in a test, no model call: artifact shapes (the refused one, a +minimal valid one, one with extra keys, one whose digest is wrong) against the two readers' verdicts. + +**Analysis.** The matrix must contain no cell in which the extension accepts and the host refuses. + +**Decisive.** The test fails on the current code and passes after the fix, and a named mutant that +loosens the extension's reader back is caught by it. + +## P1 and P2: answered (2026-09-19) + +The diagnosis is in the archive's README and the arms record: eight of nine attempts produced a +conclusion artifact, the one patch failed on `rows` against `lines`, and no envelope defect existed - +the instrument read by the wrong reader. The economics are measured: cost real, gain unrealised +(0 of 3 holding reps published), the candidate's admissibility the binding constraint. + +## P5 - A measured cost model: tried with autodiff, and the data is what fails (2026-09-19) + +The design names autodiff as the reuse target for "cost or action scoring", and the arms need a cost +model rather than a declared bound, so the terms were fitted to the paid runs with `src/lab/autodiff.ts`: +one stacked design-matrix matmul over all rows (never a loop of per-run subgraphs - 9.6x in earlier +work), no compiled tape (its O(graph) compile cost does not amortize for a one-shot fit), standardised +features and target, plain gradient descent. + +**Result: the fit is not a model.** Leave-one-out residuals across the 19 paid runs run to -15 955 and ++11 805 tokens against a mean of 11 184, so it predicts nothing about a row it has not seen. And the +session-startup term fitted out as **0 ms** where the D arm measured ~1 900 ms directly - which is the +more useful half of the finding: it says the term is *not identifiable* from these runs, because the +D arm holds `units` at 2 (bound 1 = 2 sessions, bound 2 = 1 session), leaving `units` collinear with the +intercept and only three runs per level. + +**The blocker is the data, not the optimiser.** What follows is therefore not a better fit but a better +design matrix, and the cheap half of that is offline: + +1. build the design matrix with canned workers (no model calls) over per-session unit counts 1, 2 and 3, + so `units` varies instead of being constant - the session term only becomes estimable when both + features move independently; +2. record the features that can carry signal, because the measured spread inside one arm was wider than + the difference between arms: the union tool surface's own token count (payable in every turn - the + D arm's first unit cost ~0.7 k more), turns, snapshot size, and per-unit tokens; +3. re-fit with the same autodiff call, and report leave-one-out residuals again. A term enters the + planner only when its confidence interval excludes zero. + +**Standing constraint.** Autodiff prices *legal* options; it never decides legality. Whether two units +may share a session stays a predicate over declared facts (`sharedSessionLegal`), and no fitted number +may widen it. + +## P4 - Retention, kept light (free) + +The user's reading, recorded here because it is the rule to follow: a run's evidence is needed *while the +work is being done*, so it is written to a marked scratch directory (`.temp/.../CLEANABLE.md`) and may be +deleted later - the failure was never that scratch existed, it was deleting the data before the record +that needed it was written. The instrument now writes every artifact, candidate tree and check output, +marks the directory cleanable, and copies the run into the tracked archive when a record quotes it. No +`docs:check` rule: this needs a place to keep things, not a gate. diff --git a/docs/experiments/execution/ooo-arms-pilot-2026-09-18.md b/docs/experiments/execution/ooo-arms-pilot-2026-09-18.md new file mode 100644 index 00000000..f3e03eac --- /dev/null +++ b/docs/experiments/execution/ooo-arms-pilot-2026-09-18.md @@ -0,0 +1,225 @@ +# The granularity arms against a real model - 2026-09-18 + +**Status:** paid measurement, 14 runs and 29 model calls, one model, two task families, one of them held +out of the instrument. **Directional only**: the sample cannot resolve a small difference, and no +quality difference appeared to resolve. + +The runs behind every number below are kept, unedited, in [the archive](archive/ooo-arms-2026-09-19/README.md) - they were rescued out of after the fact - and [the plan](ooo-arm-plan-2026-09-19.md) fixes what the next paid run must store before it is allowed to run. +**Related:** [task-unit semantics design](../../design/task-unit-semantics.md) · +[its obligation ledger](../../design/task-unit-semantics-obligations.md) · +[the offline sweep that ordered this](./ooo-cost-model-2026-09-17.md) + +The design's real-model stage "须另行明确模型、预算与重复次数" and reports **质量 / 墙钟 / token / +宿主成本 / 浪费成本 / 人工介入**, with "没有同等父产物质量,不得声称加快". This record fixes those +inputs, reports those six, and claims nothing the sample cannot carry. + +## What was fixed, and what was left to the arms + +| Input | Value | +| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Model | `PI_PROVIDER=deepseek`, `PI_MODEL=deepseek-v4-flash` (the session's own model; passed in, not defaulted) | +| Envelope limits | `turns: 6`, `reads: 3`, `timeoutMs: 120_000` - fixed for **every** arm, and above the host default of 3 | +| Repetitions | A 3, B 3, C 2 | +| Arm order | Drawn from a seeded shuffle (`--seed`), printed in the results, so the order is reproducible | +| Task family | `evals/ooo-execution/fixtures/pipeline` - **held out**: it was written after the driver and the report family and is not the family the instrument was built against | +| Arms (the only variable) | A = the whole task as one unit; B = the same work as four units at one slot; C = B with two slots | +| Not fixed | wall-clock time of day, host load, and the model's own sampling - none of which any arm controls | + +The arms' frozen envelope is identical: the same 10 baseline files, the same composed parent check, the +same per-unit checks, the same limits. Both specs are checked offline first, with the instrument's own +answers and a wrong answer, in `evals/ooo-execution/families.test.ts` (8 cases over the two families) - +a plan whose acceptance cannot be shown offline would not be worth paying for. + +## The measurement + +`node --experimental-strip-types evals/ooo-execution/pilot.ts --live --out `, 8 runs, +23 model calls, all 8 runs complete (no failure, no timeout, no retry, no refused submission). +Results merged with `pilot.ts --report a.json,b.json`, which makes no model call and refuses to merge +two different instruments. The D arm (fusion, 6 more runs and 6 more model calls) was added on +2026-09-19 and is reported in its own section below. + +| Arm | Plan | Slots | Runs | Accepted units | Parent | Wall total (median) | Host total | Tokens | +| --- | -------------- | ----- | ---- | -------------- | --------- | ------------------- | ---------- | ------ | +| A | coarse, 1 unit | 1 | 3 | 1 / 1 / 1 | accept ×3 | 25 654 ms (7 319) | 4 693 ms | 33 677 | +| B | fine, 4 units | 1 | 3 | 4 / 4 / 4 | accept ×3 | 64 385 ms (20 600) | 10 103 ms | 94 601 | +| C | fine, 4 units | 2 | 2 | 4 / 4 | accept ×2 | 32 461 ms (17 060) | 7 241 ms | 59 830 | + +Per run: A 8.6 s and 11.2 k tokens; B 21.5 s and 31.5 k tokens; C 16.2 s and 29.9 k tokens. Host cost per +run: A 1.6 s (one candidate), B 3.4 s and C 3.6 s (four candidates each). The arms have different run +counts, so the table's totals are sums and every ratio below is per run. + +**Quality.** Every unit of every run was accepted by its own frozen check, and the composed parent check +accepted in all 8 runs, in both plans. The arms are therefore comparable (`comparePlanSlots` would say +`comparable: true` here) and the family is _easy_ for this model - that is the honest reading, and it is +also why this pilot cannot say anything about quality differences between granularities: there were +none to measure. The design's rule is met in the narrow sense (equal parent quality), not in the strong +one (no speedup claim survives if the coarse arm is the one that degrades on a harder task). + +**Wasted cost.** Zero. No submission was rejected, no attempt was abandoned, no work had to be redone. +The `--live` pilot records `incomplete` per run and stores each run's own result file for exactly this +reason; a large `wasted` figure would have to come from those, not from a token total. + +**Human intervention.** None during the runs. Interventions _before_ the runs, both measured and both +recorded here rather than worked around: see the next section. + +## What the arms were predicted to do, and what they did + +The offline sweep fixed three expectations. All three held, on a held-out family and with a model that +is not the sweep: + +1. **"One slot never buys anything, and splitting on one slot is always a loss."** B cost 2.5× A in wall + time and 2.8× A in tokens for the same accepted work. Confirmed, and in the direction the sweep + ordered: a pilot reporting B cheaper than A would have had a measurement problem. +2. **"The split only pays when independence and slots are both real."** Per run, C recovered part of + B's loss (16.2 s against B's 21.5 s, 0.76×; 29.9 k tokens against 31.5 k, 0.95×) but nowhere near + the 2× a two-slot overlap would give if the + model call were the only cost: one unit waits for three dependencies, and the host's candidate checks + do not run concurrently. The sweep's own turning point - a saving that shrinks as a critical path + appears - is visible at this size. +3. **"The host check queue is what caps fine granularity."** The host's cost grew with the number of + candidates, not with the slots: 1.6 s per run for one candidate against 3.4 s and 3.6 s for four, with + C's extra slot buying no host saving at all. + +The sweep could not have said the token price: it models time, and the fine plan's four model calls each +carry the frozen snapshot again, which is where the 2.8× comes from. That is the one number here the +offline layer did not predict, and it is the number that matters most for a paid plan. + +## What this does not say + +- n = 8. A 25 % wall-time difference between B and C is one run's difference at this sample; the + direction is consistent with the sweep, but the size is not a result. +- One model. A model that answers in one turn instead of four would change the balance between model + time and host time, and that balance is what all three predictions rest on. +- One held-out family, of the same shape as the instrument's own (frozen interface, three independent + builders, one summary that depends on all three). A shape without a shared interface, or with a chain + instead of a fan-in, is outside what was measured. +- The families are small enough for the model to finish them in one attempt. Nothing here measures what + happens when an attempt fails, which is where a slot count would be expected to matter most. + +## Two environment findings, both fixed before any result above + +1. **The first model turn of an attempt frequently comes back aborted.** `pi turn error: This operation +was aborted` appeared in most attempts (a live probe of the pre-existing + `evals/ooo-execution/live-patch.ts` failed 2 of 6 attempts at the host default of 3 turns, and + succeeded when the nested session's own environment (`PI_SESSION_FILE`/`PI_SESSION_ID`) was cleared). + The attempt usually continues after that turn, so the pilot fixes `turns: 6` for every arm instead of + treating the abort as a worker failure. This is an instrument observation, not an arm difference, and + it is fixed for all arms together. +2. **A submitted patch carries the unit's whole frozen view.** `runParentCheck` composed the parent + candidate by overwriting it with each accepted submission, so the last unit's untouched copies of its + siblings - the stubs it was frozen with - landed over the work they had done. The parent check then + failed a plan whose units had all been accepted. Composition now takes only the files a unit changed + (`the-parent-takes-the-last-units-whole-view`, caught by the new family cases). + +## Commands + +``` +# offline first: both families, both plans, the instrument's answers and a wrong one +node --experimental-strip-types --test evals/ooo-execution/families.test.ts + +# the paid pilot (23 calls at A 3 / B 3 / C 2), then the same aggregate with no model call +node --experimental-strip-types evals/ooo-execution/pilot.ts --live --out .temp/pilot/pilot.json +node --experimental-strip-types evals/ooo-execution/pilot.ts --report .temp/pilot/pilot.json \ + --out .temp/pilot/pilot-summary.json +``` + +## D arm: fusion (added 2026-09-19) + +The D arm asks whether one session running two units in a row is cheaper than two sessions running one +each - the design's `fusion` bound - and it needed a mechanism the extension did not have: a session +held across units. `--session-runner` supplies it (one Pi session per driver session, re-pointed per +unit). The only difference between the two arms is `fusion.unitsPerSession`: 1 is the control the +driver already described, 2 is fused. Everything else is identical - the same spec, the same envelope +(`turns: 6`, `reads: 3`, `timeoutMs: 120 000`), `--slots 1`, the same model, and three reps each. + +| Arm | Bound | Sessions per run | Accepted units | Median tokens | Median wall | Failures | +| ------- | ----- | ---------------- | -------------- | ------------- | ----------- | -------- | +| unfused | 1 | `1/1` ×3 | 2/2 ×3 | 22 498 | 12 948 ms | 0 | +| fused | 2 | `2` ×3 | 2/2 ×3 | 22 533 | 11 048 ms | 0 | + +**What this sample carries.** Quality parity holds - every unit was accepted in every run of both arms, +so the wall-time comparison is admissible - and the fused arm was ~1.9 s faster per run, consistently: +11.8 / 9.6 / 11.0 s against 11.9 / 12.9 / 14.1 s. That is about 15 % of the unfused wall and wider than +either arm's own spread, and it prices the session-startup term the cost model carried as `unmeasured`: +one fused session removes exactly one startup, so the term is ~1.9 s at this model and plan size. + +**What it does not carry.** Tokens did not fall: 22 498 against 22 533 is 0.2 %, and the fused arm's own +spread (17.6k - 26.3k) is wider than the difference. Per unit the saving is real and smaller than +predicted - the second unit cost ~10.8k fused against ~11.6k unfused, about 8 % - and the first unit +cost ~0.7k more, which cancels it. The reason is a mechanism cost rather than noise: a chain's tool +surface is the union of its units' capabilities, because a session's surface is fixed when it is +created, so a unit can be offered a tool it has no use for and spend a turn on the refusal. Two units +is also where the design expected the saving to be smallest. + +**Rejected:** reading the token tie as "fusion does not pay". The two terms are separable and the +second-unit saving is real; what n = 6 at one plan shape and one model cannot say is how either scales. + +``` +node .temp/run-darm.mjs # 6 paid runs (3 per bound); writes .temp/darm/*.json + aggregate +``` + +## E arm: bounded speculation (first run, 2026-09-19) + +The E arm adds one declared fact to the D arm's machinery: whether this round needs the unit at all - +the case the design names as legitimate speculation, because the fact decides whether the work happens +rather than what the work reads. The instrument is `evals/ooo-execution/speculation-pilot.ts`: the +candidate is prepared before the fact, and the shared layer's `speculationOutcome` decides what happens +to it. A published candidate is still verified by the host with the unit's own frozen check, which is the +quality term; a discarded one closes its branch session. + +| Arm | Fact | Runs | Tokens | Work ms | Post-fact ms | Quality | +| ----------- | ------ | ---- | ------- | ------- | ------------ | ---------- | +| baseline | holds | 2 | 13 544 | 10 358 | 10 358 | false ×2 | +| speculation | holds | 2 | 17 151 | 15 066 | 186 | false ×2 | +| baseline | absent | 2 | 0 | 0 | 0 | n/a | +| speculation | absent | 2 | 12 543 | 15 577 | 0 | n/a | + +**No speedup may be claimed.** The quality term is false in all four verified candidates: each submitted +patch failed the unit's own frozen check. The design's rule is that latency and cost may not be reported +without equal quality, so the shape below is what the arm *would* measure, not a result. + +**What the shape is.** Speculation always pays for the unit - 12 543 tokens when the fact turned out +false, which is the whole point of measuring the waste - and buys the work's latency back when it holds: +186 ms of post-fact verification against 10 358 ms of work in the control. The arm's economics therefore +turn on the hit rate, which is what the design's utility formula says and what this sample cannot yet +price. + +**Two defects the first run found.** The extension accepted an artifact the host refuses: one candidate +carried `digest, kind, conclusion, summary, evidence, citations` and no `files`, which +`artifactEnvelope` admitted and `patchCandidate` then refused as "invalid patch structure" - a +submission the adapter can accept that the board can never accept. And the instrument's own quality +harness deleted the candidate tree on failure, so the first run could not say why every check failed; +it now keeps the tree, and the control rows record the check's own output. + +## E arm: second run, with evidence (2026-09-19) + +The first E-arm run reported quality failures it could not explain, because it kept no bytes. This run +keeps everything (artifacts, candidate trees, the check's own output, one row per attempt) and answers +both the diagnosis and the economics question. Three reps per condition, 9 paid units, ~62 k tokens. + +| Arm | Fact | Tokens (3 reps, total) | Work ms | Post-fact ms | Quality | +| ----------- | ------ | ---------------------- | ------- | ------------ | ----------------- | +| baseline | holds | 18 183 | 19 200 | 19 200 | 0 of 3 passed | +| speculation | holds | 18 602 | 19 506 | 175 | 0 of 3 passed | +| baseline | absent | 0 | 0 | 0 | n/a | +| speculation | absent | 20 332 | 23 920 | 0 | n/a | + +**Diagnosis first (P1).** The harness's check path was validated offline before any of this was read: +the frozen stub fails, the fixture's own canned answer passes, and a wrong answer fails, for both +units - so a failing check means what it says. What the *first* run could not see is that eight of nine +attempts answered with a **conclusion** artifact ("no change needed"), which is legitimate for a task +whose rule admits one, carries no files, and therefore cannot pass this unit's check - the board would +refuse it for the same reason. The single patch attempt failed on a real mistake: it wrote +`rows: [...]` where the frozen interface requires `lines: [...]`. Nothing here was an envelope +defect; the earlier claim that the extension was laxer than the host was wrong, and it is corrected in +the archive's README. + +**Economics (P2).** The mechanism works and its shape is the design's: when the fact holds, the +post-fact cost drops from the work itself (~6.2 s median) to verification of an already-prepared +candidate (175 ms in the one rep that produced one); when the fact is absent, the whole spend is waste - +20 332 tokens, more than doing the work when needed. **No benefit is claimed, and none is available:** +the prepared candidate was publishable in 0 of 3 reps where the fact held, so the latency term never +became real. In this shape - this model, this unit's instruction, this fact - bounded speculation has a +real cost and no realised gain, and the binding constraint is the candidate's admissibility rather than +the mechanism's speed. + diff --git a/docs/experiments/execution/ooo-cost-model-2026-09-17.md b/docs/experiments/execution/ooo-cost-model-2026-09-17.md new file mode 100644 index 00000000..eab07693 --- /dev/null +++ b/docs/experiments/execution/ooo-cost-model-2026-09-17.md @@ -0,0 +1,93 @@ +# A granularity and cost sweep before any paid call - 2026-09-17 + +**Status:** offline measurement. No model call was made; nothing here is a result about model quality. +**Related:** [task-unit semantics design](../../design/task-unit-semantics.md) · +[its obligation ledger](../../design/task-unit-semantics-obligations.md) + +The design orders the arms as "离线模型先覆盖不同粒度、依赖密度、共享上下文、事实命中率及验证成本;它只能 +发现逻辑错误和成本转折点,不能预测真实模型质量", and says the real-model stage "须另行明确模型、预算与 +重复次数". The repository had no such offline layer: `rg "simulat|离线|stub worker"` over `evals/` and +`docs/experiments/` found nothing. This record is the first pass of one, and what it changed about how +the paid pilot should be run. + +## The instrument + +`evals/ooo-execution/cost-model.ts` (advisory, `--sweep` and single-vector modes) plus +`evals/ooo-execution/cost-model.test.ts` (8 cases). + +Plan shape: `units` (granularity), `density` (the fraction of possible forward edges that exist, so 0 +is fully independent and 1 is a chain), `seed`. The graph is **derived** from the seed rather than +written by hand, so a gain cannot come from a shape picked to produce one. + +Cost vector, with what each term stands for: + +| Term | Stands for | Status | +| ----------------------- | ----------------------------------------------------------- | ------------------------------------------ | +| `workMs` | one agent call's latency | measured range in earlier rounds (seconds) | +| `rederiveMs` × ¬hitRate | re-deriving an artifact a fact hit would have removed | **assumed**, swept | +| `verifyMs` | one host candidate check, one at a time | measured order (~1.5 s) | +| `contextMsPerUnit` | a legal session boundary | **assumed**, swept | +| `coarseContextSaving` | what one session saves by not paying that boundary per unit | **assumed**, this is the term to argue | +| `slots` | execution slots | the arm's manipulated variable | + +The schedule is greedy list scheduling over `slots`, followed by **one host check queue**: a candidate +check never runs concurrently with another check. A dependency is satisfied by an _accepted_ artifact, +so a unit waits for its predecessor's check, not for its model call — which is the difference the +out-of-order round exists to exploit. + +## What the model got wrong first + +Its initial self-check demanded that four independent units on four slots finish in "one unit plus one +check". It did not, and the model was wrong, not the check: the host serialises four checks, so the +floor is one unit **plus four** checks. That number is now the check. The same instrument without the +queue would have reported a concurrency gain no round can have. + +## The sweep + +Base vector: work 8 s, re-derive 6 s, hit rate 0.5, verify 1.5 s, context 1.2 s per unit, coarse +saving 0.5, 4 slots. Milliseconds. + +| density | units | coarse | fine, 1 slot | fine, multi-slot | saved by slots | critical path | gain vs coarse | +| ------- | ----- | ------ | ------------ | ---------------- | -------------- | ------------- | -------------- | +| 0 | 1 | 11 900 | 13 700 | 13 700 | 0 | 13 700 | **−1 800** | +| 0 | 2 | 23 800 | 25 900 | 15 200 | 10 700 | 13 700 | +8 600 | +| 0 | 4 | 47 600 | 50 300 | 18 200 | 32 100 | 13 700 | +29 400 | +| 0 | 8 | 95 200 | 99 100 | 30 400 | 68 700 | 13 700 | +64 800 | +| 0.25 | 2 | 23 800 | 27 400 | 27 400 | 0 | 27 400 | **−3 600** | +| 0.25 | 4 | 47 600 | 51 800 | 30 400 | 21 400 | 27 400 | +17 200 | +| 0.5 | 4 | 47 600 | 51 800 | 41 100 | 10 700 | 41 100 | +6 500 | +| 1 | 4 | 47 600 | 54 800 | 54 800 | 0 | 54 800 | **−7 200** | +| 1 | 8 | 95 200 | 109 600 | 109 600 | 0 | 109 600 | **−14 400** | + +## What it says, and what it does not + +1. **One slot never buys anything, and splitting on one slot is always a loss.** This is the design's + B arm, and the model says B should be _worse_ than A on cost: it pays a session boundary per unit + and overlaps nothing. A pilot that reports B cheaper than A has a measurement problem, not a + finding. +2. **The split only pays when independence and slots are both real.** A single dependency edge among + two units (density 0.25, units 2) turns the gain negative; a chain (density 1) is negative at every + granularity (−14.4 s at 8 units), because the boundaries are added and nothing is overlapped. +3. **The host check queue is what caps fine granularity.** Every unit adds a serial check, so the + fine arm's floor is one unit plus _units_ checks; the coarse arm pays the same number of checks in + this vector, which is why the gain does not grow without limit with independence alone. +4. **It cannot say anything about quality, and it does not pretend to.** `cost-model.test.ts` fails if + the result object ever grows a key matching `/quality|pass|accept|correct|success/i`, so a simulated + pass rate cannot quietly become a reported one. The design's caveat stays true by construction. + +## Consequences for the paid pilot + +- Choose a parent task whose legal refinement yields **several** units (≥ 4) with **density well below + 0.5** and per-unit checks that do not dominate the unit's work. A refinement that yields two units or + a chain is predicted to lose, and running it would spend tokens to confirm the model. +- Expect A < B on cost and A ≈ B on quality, with the gain (if any) appearing only at C. A pilot that + shows a B-arm cost saving contradicts this sweep and needs its instrument checked first. +- Fix the model, budget and repetitions separately, as the design requires; this record fixes none of + them and used no paid call. + +## Commands + +``` +node --experimental-strip-types evals/ooo-execution/cost-model.ts --sweep --out .temp/cost-sweep.json +node --experimental-strip-types --test evals/ooo-execution/cost-model.test.ts # 8 pass / 0 fail +``` diff --git a/docs/experiments/execution/ooo-real-continuation-2026-09-14-g7-run.jsonl b/docs/experiments/execution/ooo-real-continuation-2026-09-14-g7-run.jsonl new file mode 100644 index 00000000..ae33498a --- /dev/null +++ b/docs/experiments/execution/ooo-real-continuation-2026-09-14-g7-run.jsonl @@ -0,0 +1,25 @@ +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk","entryId":"1789396212677_000001","at":"2026-09-14T14:30:12.678Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration","entryId":"1789396212679_000002","at":"2026-09-14T14:30:12.679Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge","entryId":"1789396212679_000003","at":"2026-09-14T14:30:12.679Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average","entryId":"1789396212680_000004","at":"2026-09-14T14:30:12.680Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths","entryId":"1789396212680_000005","at":"2026-09-14T14:30:12.680Z"} +{"role":"part1","task":"chunk","channel":"ooo-continuation:chunk","entryId":"1789396212677_000001","agentId":"worker-part1-23136","pid":23136,"stage":"part1","wallMs":4650,"turns":4,"reads":1,"tokens":4268,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"9ae8b2c48ead7d745ed8bffcdbd73c828ad58a776b4c58957a810f0faf8931b1","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\g7-run\\chunk\\part1\\chunk.ts","verdict":null,"at":"2026-09-14T14:30:19.126Z"} +{"role":"judge","task":"chunk","channel":"ooo-continuation:chunk","entryId":"1789396212677_000001","agentId":"judge-continuation-46744","pid":46744,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"9ae8b2c48ead7d745ed8bffcdbd73c828ad58a776b4c58957a810f0faf8931b1","continuation":"1789396220810_000006","at":"2026-09-14T14:30:20.810Z"} +{"role":"part2","task":"chunk","channel":"ooo-continuation:chunk","entryId":"1789396220810_000006","agentId":"worker-part2-56932","pid":56932,"stage":"part2","wallMs":12571,"turns":6,"reads":1,"tokens":14230,"cases":6,"passed":6,"failed":[],"conclusion":"promote-candidate","digest":"fc8bb609de5eacdb9c8e4dd5a9fa32d342bdfaa4b928db402fdfa6816dd4c00f","continuedFrom":"9ae8b2c48ead7d745ed8bffcdbd73c828ad58a776b4c58957a810f0faf8931b1","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\g7-run\\chunk\\part2\\chunk.ts","verdict":null,"at":"2026-09-14T14:30:35.104Z"} +{"role":"judge","task":"chunk","channel":"ooo-continuation:chunk","entryId":"1789396220810_000006","agentId":"judge-continuation-21908","pid":21908,"stage":"parent","verdict":"accepted","cases":6,"passed":6,"failed":[],"artifactDigest":"fc8bb609de5eacdb9c8e4dd5a9fa32d342bdfaa4b928db402fdfa6816dd4c00f","continuation":null,"at":"2026-09-14T14:30:36.834Z"} +{"role":"part1","task":"duration","channel":"ooo-continuation:duration","entryId":"1789396212679_000002","agentId":"worker-part1-21348","pid":21348,"stage":"part1","wallMs":4585,"turns":4,"reads":1,"tokens":4425,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"f4e5aad302937c654229b101fccf655789bc2352fdf47baf8880ac2388091a71","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\g7-run\\duration\\part1\\duration.ts","verdict":null,"at":"2026-09-14T14:30:43.159Z"} +{"role":"judge","task":"duration","channel":"ooo-continuation:duration","entryId":"1789396212679_000002","agentId":"judge-continuation-60728","pid":60728,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"f4e5aad302937c654229b101fccf655789bc2352fdf47baf8880ac2388091a71","continuation":"1789396244837_000007","at":"2026-09-14T14:30:44.838Z"} +{"role":"part2","task":"duration","channel":"ooo-continuation:duration","entryId":"1789396244837_000007","agentId":"worker-part2-57896","pid":57896,"stage":"part2","wallMs":26900,"turns":5,"reads":1,"tokens":22262,"cases":8,"passed":8,"failed":[],"conclusion":"promote-candidate","digest":"00f891c1b6a70b81a3f77a1dc192b312a05439cdaa69c083238893b7acbd0c11","continuedFrom":"f4e5aad302937c654229b101fccf655789bc2352fdf47baf8880ac2388091a71","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\g7-run\\duration\\part2\\duration.ts","verdict":null,"at":"2026-09-14T14:31:13.478Z"} +{"role":"judge","task":"duration","channel":"ooo-continuation:duration","entryId":"1789396244837_000007","agentId":"judge-continuation-24928","pid":24928,"stage":"parent","verdict":"accepted","cases":8,"passed":8,"failed":[],"artifactDigest":"00f891c1b6a70b81a3f77a1dc192b312a05439cdaa69c083238893b7acbd0c11","continuation":null,"at":"2026-09-14T14:31:15.251Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge","entryId":"1789396212679_000003","agentId":"worker-part1-58480","pid":58480,"stage":"part1","wallMs":6749,"turns":4,"reads":1,"tokens":5018,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"b75eef86deb4f2c3528e608f1b7600d649b026572ca156c48a99ef71b3936a97","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\g7-run\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T14:31:23.660Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge","entryId":"1789396212679_000003","agentId":"judge-continuation-73104","pid":73104,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"b75eef86deb4f2c3528e608f1b7600d649b026572ca156c48a99ef71b3936a97","continuation":"1789396285336_000008","at":"2026-09-14T14:31:25.337Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge","pid":52236,"stage":"part2","failed":["invalid patch structure"],"at":"2026-09-14T14:31:31.663Z"} +{"role":"part1","task":"average","channel":"ooo-continuation:average","entryId":"1789396212680_000004","agentId":"worker-part1-73816","pid":73816,"stage":"part1","wallMs":4772,"turns":4,"reads":1,"tokens":4342,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"673532a7116695691ae96e0c344c7c16e7f9fc7f9e6846570b9823a84d5685fd","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\g7-run\\average\\part1\\average.ts","verdict":null,"at":"2026-09-14T14:31:40.049Z"} +{"role":"judge","task":"average","channel":"ooo-continuation:average","entryId":"1789396212680_000004","agentId":"judge-continuation-56116","pid":56116,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"673532a7116695691ae96e0c344c7c16e7f9fc7f9e6846570b9823a84d5685fd","continuation":"1789396301854_000009","at":"2026-09-14T14:31:41.854Z"} +{"role":"part2","task":"average","channel":"ooo-continuation:average","entryId":"1789396301854_000009","agentId":"worker-part2-73520","pid":73520,"stage":"part2","wallMs":5367,"turns":4,"reads":1,"tokens":5177,"cases":6,"passed":6,"failed":[],"conclusion":"promote-candidate","digest":"2394506c6130bcd5a03e5c9b036ec394d427cd9d488980ee110eaa06c8e0b98e","continuedFrom":"673532a7116695691ae96e0c344c7c16e7f9fc7f9e6846570b9823a84d5685fd","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\g7-run\\average\\part2\\average.ts","verdict":null,"at":"2026-09-14T14:31:49.041Z"} +{"role":"judge","task":"average","channel":"ooo-continuation:average","entryId":"1789396301854_000009","agentId":"judge-continuation-57472","pid":57472,"stage":"parent","verdict":"accepted","cases":6,"passed":6,"failed":[],"artifactDigest":"2394506c6130bcd5a03e5c9b036ec394d427cd9d488980ee110eaa06c8e0b98e","continuation":null,"at":"2026-09-14T14:31:50.851Z"} +{"role":"part1","task":"paths","channel":"ooo-continuation:paths","entryId":"1789396212680_000005","agentId":"worker-part1-53948","pid":53948,"stage":"part1","wallMs":8781,"turns":5,"reads":2,"tokens":7216,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"55daf617a7c9de7d5e6bd94a4d0eb0e9c558f4ba7e07914a2257d5a0313516b5","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\g7-run\\paths\\part1\\paths.ts","verdict":null,"at":"2026-09-14T14:32:01.422Z"} +{"role":"judge","task":"paths","channel":"ooo-continuation:paths","entryId":"1789396212680_000005","agentId":"judge-continuation-24456","pid":24456,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"55daf617a7c9de7d5e6bd94a4d0eb0e9c558f4ba7e07914a2257d5a0313516b5","continuation":"1789396323213_000010","at":"2026-09-14T14:32:03.214Z"} +{"role":"part2","task":"paths","channel":"ooo-continuation:paths","entryId":"1789396323213_000010","agentId":"worker-part2-70756","pid":70756,"stage":"part2","wallMs":10020,"turns":6,"reads":3,"tokens":13143,"cases":7,"passed":7,"failed":[],"conclusion":"no-change-needed","digest":"55daf617a7c9de7d5e6bd94a4d0eb0e9c558f4ba7e07914a2257d5a0313516b5","continuedFrom":"55daf617a7c9de7d5e6bd94a4d0eb0e9c558f4ba7e07914a2257d5a0313516b5","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\g7-run\\paths\\part2\\paths.ts","verdict":null,"at":"2026-09-14T14:32:15.029Z"} +{"role":"judge","task":"paths","channel":"ooo-continuation:paths","entryId":"1789396323213_000010","agentId":"judge-continuation-62184","pid":62184,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"55daf617a7c9de7d5e6bd94a4d0eb0e9c558f4ba7e07914a2257d5a0313516b5","continuation":null,"at":"2026-09-14T14:32:16.882Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge","pid":42984,"stage":"part2","failed":["task board entry 1789396285336_000008 already claimed by worker-part2-52236"],"at":"2026-09-14T14:33:28.226Z"} diff --git a/docs/experiments/execution/ooo-real-continuation-2026-09-14-merge-retries.jsonl b/docs/experiments/execution/ooo-real-continuation-2026-09-14-merge-retries.jsonl new file mode 100644 index 00000000..f93ee748 --- /dev/null +++ b/docs/experiments/execution/ooo-real-continuation-2026-09-14-merge-retries.jsonl @@ -0,0 +1,12 @@ +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk","entryId":"1789396475299_000001","at":"2026-09-14T14:34:35.300Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration","entryId":"1789396475300_000002","at":"2026-09-14T14:34:35.301Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge","entryId":"1789396475301_000003","at":"2026-09-14T14:34:35.301Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average","entryId":"1789396475301_000004","at":"2026-09-14T14:34:35.302Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths","entryId":"1789396475302_000005","at":"2026-09-14T14:34:35.302Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge","entryId":"1789396475301_000003","agentId":"worker-part1-62552","pid":62552,"stage":"part1","wallMs":6276,"turns":4,"reads":1,"tokens":4758,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"ee0bbbaae9b09201c2662996a9acfd83e9226920c3d19eab9928f6b56cff69ec","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\merge-retry\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T14:34:43.348Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge","entryId":"1789396475301_000003","agentId":"judge-continuation-46236","pid":46236,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"ee0bbbaae9b09201c2662996a9acfd83e9226920c3d19eab9928f6b56cff69ec","continuation":"1789396485190_000006","at":"2026-09-14T14:34:45.191Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge","pid":66272,"stage":"part2","failed":["invalid patch structure"],"at":"2026-09-14T14:34:55.923Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge","pid":37812,"stage":"part2","failed":["task board entry 1789396485190_000006 already claimed by worker-part2-66272"],"at":"2026-09-14T14:36:04.643Z"} +{"role":"release","task":"merge","channel":"ooo-continuation:merge","entryId":"1789396485190_000006","released":"worker-part2-66272","pid":17004,"at":"2026-09-14T14:37:14.816Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge","entryId":"1789396485190_000006","agentId":"worker-part2-32436","pid":32436,"stage":"part2","wallMs":9049,"turns":4,"reads":1,"tokens":6964,"cases":7,"passed":7,"failed":[],"conclusion":"promote-candidate","digest":"81b75fa98b89875744428cefdd981c8e7de2403f8718348f39d1045489aae75a","continuedFrom":"ee0bbbaae9b09201c2662996a9acfd83e9226920c3d19eab9928f6b56cff69ec","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\merge-retry\\merge\\part2\\merge.ts","verdict":null,"at":"2026-09-14T14:37:25.749Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge","entryId":"1789396485190_000006","agentId":"judge-continuation-57108","pid":57108,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"81b75fa98b89875744428cefdd981c8e7de2403f8718348f39d1045489aae75a","continuation":null,"at":"2026-09-14T14:38:39.988Z"} diff --git a/docs/experiments/execution/ooo-real-continuation-comparison-2026-09-14-m6.jsonl b/docs/experiments/execution/ooo-real-continuation-comparison-2026-09-14-m6.jsonl new file mode 100644 index 00000000..9ac0fa85 --- /dev/null +++ b/docs/experiments/execution/ooo-real-continuation-comparison-2026-09-14-m6.jsonl @@ -0,0 +1,198 @@ +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk","entryId":"1789398763494_000001","at":"2026-09-14T15:12:43.495Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration","entryId":"1789398763496_000002","at":"2026-09-14T15:12:43.496Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge","entryId":"1789398763496_000003","at":"2026-09-14T15:12:43.496Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average","entryId":"1789398763496_000004","at":"2026-09-14T15:12:43.497Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths","entryId":"1789398763497_000005","at":"2026-09-14T15:12:43.497Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r1","entryId":"1789398765300_000006","at":"2026-09-14T15:12:45.309Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r1","entryId":"1789398765309_000007","at":"2026-09-14T15:12:45.310Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r1","entryId":"1789398765310_000008","at":"2026-09-14T15:12:45.310Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r1","entryId":"1789398765310_000009","at":"2026-09-14T15:12:45.311Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r1","entryId":"1789398765311_000010","at":"2026-09-14T15:12:45.311Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r1","entryId":"1789398765310_000008","agentId":"worker-part1-3664","pid":3664,"stage":"part1","wallMs":5553,"turns":4,"reads":1,"tokens":4363,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"5940035bba18791decb30c743d7cb50dae817053a22c1706502c63e172702729","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T15:12:52.567Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r1","entryId":"1789398765310_000008","agentId":"judge-continuation-47604","pid":47604,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"5940035bba18791decb30c743d7cb50dae817053a22c1706502c63e172702729","continuation":"1789398774428_000011","at":"2026-09-14T15:12:54.428Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r1","entryId":"1789398774428_000011","agentId":"worker-part2-53660","pid":53660,"stage":"part2","wallMs":6510,"turns":4,"reads":1,"tokens":6056,"cases":7,"passed":7,"failed":[],"conclusion":"promote-candidate","digest":"95333636068e7822b237ceca63c8342a990f2ddb080f63d38c6be20575f4da21","continuedFrom":"5940035bba18791decb30c743d7cb50dae817053a22c1706502c63e172702729","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2\\merge.ts","verdict":null,"at":"2026-09-14T15:13:02.712Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r1","entryId":"1789398774428_000011","agentId":"judge-continuation-69232","pid":69232,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"95333636068e7822b237ceca63c8342a990f2ddb080f63d38c6be20575f4da21","continuation":null,"at":"2026-09-14T15:13:04.612Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r2","entryId":"1789398786706_000012","at":"2026-09-14T15:13:06.708Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r2","entryId":"1789398786709_000013","at":"2026-09-14T15:13:06.709Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r2","entryId":"1789398786709_000014","at":"2026-09-14T15:13:06.709Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r2","entryId":"1789398786709_000015","at":"2026-09-14T15:13:06.710Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r2","entryId":"1789398786710_000016","at":"2026-09-14T15:13:06.710Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r2","entryId":"1789398786709_000014","agentId":"worker-part1-66512","pid":66512,"stage":"part1","wallMs":6047,"turns":5,"reads":2,"tokens":6375,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"5a9b4c2397e694dc6bb13579611f30077371096e2408c7805eaee7aa3c8ea20b","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T15:13:14.587Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r2","entryId":"1789398786709_000014","agentId":"judge-continuation-66936","pid":66936,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"5a9b4c2397e694dc6bb13579611f30077371096e2408c7805eaee7aa3c8ea20b","continuation":"1789398796381_000017","at":"2026-09-14T15:13:16.381Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r3","entryId":"1789398807322_000018","at":"2026-09-14T15:13:27.333Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r3","entryId":"1789398807333_000019","at":"2026-09-14T15:13:27.333Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r3","entryId":"1789398807334_000020","at":"2026-09-14T15:13:27.334Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r3","entryId":"1789398807334_000021","at":"2026-09-14T15:13:27.334Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r3","entryId":"1789398807335_000022","at":"2026-09-14T15:13:27.335Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r3","entryId":"1789398807334_000020","agentId":"worker-part1-12244","pid":12244,"stage":"part1","wallMs":5731,"turns":4,"reads":1,"tokens":4748,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"ee3338f60c6811a58ca03ba96bd68764136534a19d0948d9c8fa40a7119d8994","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T15:13:34.805Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r3","entryId":"1789398807334_000020","agentId":"judge-continuation-64736","pid":64736,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"ee3338f60c6811a58ca03ba96bd68764136534a19d0948d9c8fa40a7119d8994","continuation":"1789398816616_000023","at":"2026-09-14T15:13:36.616Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r4","entryId":"1789398831382_000024","at":"2026-09-14T15:13:51.384Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r4","entryId":"1789398831384_000025","at":"2026-09-14T15:13:51.385Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r4","entryId":"1789398831385_000026","at":"2026-09-14T15:13:51.385Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r4","entryId":"1789398831385_000027","at":"2026-09-14T15:13:51.386Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r4","entryId":"1789398831386_000028","at":"2026-09-14T15:13:51.386Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r4","entryId":"1789398831385_000026","agentId":"worker-part1-74840","pid":74840,"stage":"part1","wallMs":4230,"turns":4,"reads":1,"tokens":4222,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"889ae3c17d4fff8eb932c88c608941702e6b5443605b23d024540b50036cdd3e","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T15:13:57.315Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r4","entryId":"1789398831385_000026","agentId":"judge-continuation-14724","pid":14724,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"889ae3c17d4fff8eb932c88c608941702e6b5443605b23d024540b50036cdd3e","continuation":"1789398839058_000029","at":"2026-09-14T15:13:59.058Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r5","entryId":"1789398851885_000030","at":"2026-09-14T15:14:11.895Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r5","entryId":"1789398851896_000031","at":"2026-09-14T15:14:11.896Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r5","entryId":"1789398851896_000032","at":"2026-09-14T15:14:11.896Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r5","entryId":"1789398851897_000033","at":"2026-09-14T15:14:11.897Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r5","entryId":"1789398851897_000034","at":"2026-09-14T15:14:11.897Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r5","entryId":"1789398851896_000032","agentId":"worker-part1-72432","pid":72432,"stage":"part1","wallMs":9543,"turns":6,"reads":2,"tokens":9958,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"acb5eceeacfd259d1a1810d96d79b225f04cf149c786b92bf3917b42a3003258","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T15:14:23.169Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r5","entryId":"1789398851896_000032","agentId":"judge-continuation-65624","pid":65624,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"acb5eceeacfd259d1a1810d96d79b225f04cf149c786b92bf3917b42a3003258","continuation":"1789398864858_000035","at":"2026-09-14T15:14:24.858Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r5","entryId":"1789398864858_000035","agentId":"worker-part2-4992","pid":4992,"stage":"part2","wallMs":10108,"turns":4,"reads":1,"tokens":6922,"cases":7,"passed":7,"failed":[],"conclusion":"promote-candidate","digest":"c075333a49dc5337b079adbd5e47e33d1d7a2f8ed462022f3ad75fc405f4aacb","continuedFrom":"acb5eceeacfd259d1a1810d96d79b225f04cf149c786b92bf3917b42a3003258","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2\\merge.ts","verdict":null,"at":"2026-09-14T15:14:36.661Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r5","entryId":"1789398864858_000035","agentId":"judge-continuation-41436","pid":41436,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"c075333a49dc5337b079adbd5e47e33d1d7a2f8ed462022f3ad75fc405f4aacb","continuation":null,"at":"2026-09-14T15:14:38.373Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r6","entryId":"1789398880060_000036","at":"2026-09-14T15:14:40.071Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r6","entryId":"1789398880071_000037","at":"2026-09-14T15:14:40.072Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r6","entryId":"1789398880072_000038","at":"2026-09-14T15:14:40.072Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r6","entryId":"1789398880072_000039","at":"2026-09-14T15:14:40.073Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r6","entryId":"1789398880073_000040","at":"2026-09-14T15:14:40.073Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r6","entryId":"1789398880072_000038","agentId":"worker-part1-57028","pid":57028,"stage":"part1","wallMs":7545,"turns":4,"reads":1,"tokens":5456,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"6c42e59259b6b4e56adb51b83731b13131f87da87d5387a94a7e1741e38db6df","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T15:14:49.299Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r6","entryId":"1789398880072_000038","agentId":"judge-continuation-47484","pid":47484,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"6c42e59259b6b4e56adb51b83731b13131f87da87d5387a94a7e1741e38db6df","continuation":"1789398891093_000041","at":"2026-09-14T15:14:51.093Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r6","entryId":"1789398891093_000041","agentId":"worker-part2-75248","pid":75248,"stage":"part2","wallMs":9135,"turns":4,"reads":1,"tokens":7035,"cases":7,"passed":7,"failed":[],"conclusion":"promote-candidate","digest":"785d7f5e5beb3c73172cf2bc69ca804fc0aec3d85de79f1922e7551418ce1fcf","continuedFrom":"6c42e59259b6b4e56adb51b83731b13131f87da87d5387a94a7e1741e38db6df","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2\\merge.ts","verdict":null,"at":"2026-09-14T15:15:01.927Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r6","entryId":"1789398891093_000041","agentId":"judge-continuation-39356","pid":39356,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"785d7f5e5beb3c73172cf2bc69ca804fc0aec3d85de79f1922e7551418ce1fcf","continuation":null,"at":"2026-09-14T15:15:03.648Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r7","entryId":"1789399374049_000042","at":"2026-09-14T15:22:54.058Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r7","entryId":"1789399374058_000043","at":"2026-09-14T15:22:54.059Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r7","entryId":"1789399374059_000044","at":"2026-09-14T15:22:54.059Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r7","entryId":"1789399374059_000045","at":"2026-09-14T15:22:54.060Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r7","entryId":"1789399374060_000046","at":"2026-09-14T15:22:54.060Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r7","entryId":"1789399374059_000044","agentId":"worker-part1-60024","pid":60024,"stage":"part1","wallMs":4578,"turns":4,"reads":1,"tokens":4285,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"ee0bbbaae9b09201c2662996a9acfd83e9226920c3d19eab9928f6b56cff69ec","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T15:23:00.415Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r7","entryId":"1789399374059_000044","agentId":"judge-continuation-57428","pid":57428,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"ee0bbbaae9b09201c2662996a9acfd83e9226920c3d19eab9928f6b56cff69ec","continuation":"1789399382221_000047","at":"2026-09-14T15:23:02.221Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r8","entryId":"1789399391300_000048","at":"2026-09-14T15:23:11.301Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r8","entryId":"1789399391302_000049","at":"2026-09-14T15:23:11.302Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r8","entryId":"1789399391302_000050","at":"2026-09-14T15:23:11.303Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r8","entryId":"1789399391303_000051","at":"2026-09-14T15:23:11.303Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r8","entryId":"1789399391304_000052","at":"2026-09-14T15:23:11.304Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r8","entryId":"1789399391302_000050","agentId":"worker-part1-16208","pid":16208,"stage":"part1","wallMs":5747,"turns":4,"reads":1,"tokens":4742,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"b80069f4e8c0c6bcd1e2063250e375405b5ba7df4a1e6b9ba91dc0a4e5c3804a","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T15:23:18.837Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r8","entryId":"1789399391302_000050","agentId":"judge-continuation-28064","pid":28064,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"b80069f4e8c0c6bcd1e2063250e375405b5ba7df4a1e6b9ba91dc0a4e5c3804a","continuation":"1789399400637_000053","at":"2026-09-14T15:23:20.637Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r8","entryId":"1789399400637_000053","agentId":"worker-part2-17912","pid":17912,"stage":"part2","wallMs":12360,"turns":6,"reads":2,"tokens":14355,"cases":7,"passed":7,"failed":[],"conclusion":"promote-candidate","digest":"ba0934c8365dee152ab4b1796cacdad4db2a4b88ab4ec0ee5b13b2e0cea422ee","continuedFrom":"b80069f4e8c0c6bcd1e2063250e375405b5ba7df4a1e6b9ba91dc0a4e5c3804a","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2\\merge.ts","verdict":null,"at":"2026-09-14T15:23:34.678Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r8","entryId":"1789399400637_000053","agentId":"judge-continuation-65924","pid":65924,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"ba0934c8365dee152ab4b1796cacdad4db2a4b88ab4ec0ee5b13b2e0cea422ee","continuation":null,"at":"2026-09-14T15:23:36.484Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r9","entryId":"1789399418129_000054","at":"2026-09-14T15:23:38.140Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r9","entryId":"1789399418140_000055","at":"2026-09-14T15:23:38.140Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r9","entryId":"1789399418140_000056","at":"2026-09-14T15:23:38.141Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r9","entryId":"1789399418141_000057","at":"2026-09-14T15:23:38.141Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r9","entryId":"1789399418141_000058","at":"2026-09-14T15:23:38.142Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r9","entryId":"1789399418140_000056","agentId":"worker-part1-49476","pid":49476,"stage":"part1","wallMs":5539,"turns":4,"reads":1,"tokens":4490,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"ee0bbbaae9b09201c2662996a9acfd83e9226920c3d19eab9928f6b56cff69ec","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T15:23:45.423Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r9","entryId":"1789399418140_000056","agentId":"judge-continuation-42504","pid":42504,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"ee0bbbaae9b09201c2662996a9acfd83e9226920c3d19eab9928f6b56cff69ec","continuation":"1789399427113_000059","at":"2026-09-14T15:23:47.114Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r9","entryId":"1789399427113_000059","agentId":"worker-part2-71696","pid":71696,"stage":"part2","wallMs":8211,"turns":4,"reads":1,"tokens":6661,"cases":7,"passed":7,"failed":[],"conclusion":"promote-candidate","digest":"0d272f560da8cd54b1647ba90d2fb91621fc0f4fd275805683ab4030a56252da","continuedFrom":"ee0bbbaae9b09201c2662996a9acfd83e9226920c3d19eab9928f6b56cff69ec","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2\\merge.ts","verdict":null,"at":"2026-09-14T15:23:57.086Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r9","entryId":"1789399427113_000059","agentId":"judge-continuation-67508","pid":67508,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"0d272f560da8cd54b1647ba90d2fb91621fc0f4fd275805683ab4030a56252da","continuation":null,"at":"2026-09-14T15:23:58.775Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r10","entryId":"1789399440530_000060","at":"2026-09-14T15:24:00.540Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r10","entryId":"1789399440540_000061","at":"2026-09-14T15:24:00.540Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r10","entryId":"1789399440541_000062","at":"2026-09-14T15:24:00.541Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r10","entryId":"1789399440541_000063","at":"2026-09-14T15:24:00.541Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r10","entryId":"1789399440542_000064","at":"2026-09-14T15:24:00.542Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r10","entryId":"1789399440541_000062","agentId":"worker-part1-67644","pid":67644,"stage":"part1","wallMs":5941,"turns":4,"reads":1,"tokens":5019,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"22d190927cd2e21d8ac5fbbb0099c63b1a34043f1e51d1de83bc66c4882f0939","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T15:24:08.153Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r10","entryId":"1789399440541_000062","agentId":"judge-continuation-50040","pid":50040,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"22d190927cd2e21d8ac5fbbb0099c63b1a34043f1e51d1de83bc66c4882f0939","continuation":"1789399450059_000065","at":"2026-09-14T15:24:10.059Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r10","entryId":"1789399450059_000065","agentId":"worker-part2-16600","pid":16600,"stage":"part2","wallMs":8378,"turns":4,"reads":1,"tokens":6491,"cases":7,"passed":7,"failed":[],"conclusion":"promote-candidate","digest":"81dcfe681b1e997740e462205e611ffecc5951d7769b1f31426ad96c34ab56f4","continuedFrom":"22d190927cd2e21d8ac5fbbb0099c63b1a34043f1e51d1de83bc66c4882f0939","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2\\merge.ts","verdict":null,"at":"2026-09-14T15:24:20.166Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r10","entryId":"1789399450059_000065","agentId":"judge-continuation-61224","pid":61224,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"81dcfe681b1e997740e462205e611ffecc5951d7769b1f31426ad96c34ab56f4","continuation":null,"at":"2026-09-14T15:24:21.960Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r11","entryId":"1789399463796_000066","at":"2026-09-14T15:24:23.807Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r11","entryId":"1789399463808_000067","at":"2026-09-14T15:24:23.808Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r11","entryId":"1789399463808_000068","at":"2026-09-14T15:24:23.809Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r11","entryId":"1789399463809_000069","at":"2026-09-14T15:24:23.809Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r11","entryId":"1789399463809_000070","at":"2026-09-14T15:24:23.810Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r11","entryId":"1789399463808_000068","agentId":"worker-part1-40908","pid":40908,"stage":"part1","wallMs":4615,"turns":4,"reads":1,"tokens":4488,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"14a109fe026532e70f02ebd7246dae4eafcb0a05ee8fa83929a6b0ab61e899b9","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T15:24:30.220Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r11","entryId":"1789399463808_000068","agentId":"judge-continuation-49896","pid":49896,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"14a109fe026532e70f02ebd7246dae4eafcb0a05ee8fa83929a6b0ab61e899b9","continuation":"1789399472032_000071","at":"2026-09-14T15:24:32.032Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r11","entryId":"1789399472032_000071","agentId":"worker-part2-57256","pid":57256,"stage":"part2","wallMs":5979,"turns":4,"reads":1,"tokens":5816,"cases":7,"passed":7,"failed":[],"conclusion":"promote-candidate","digest":"6667df1c481132eedd71ba2c612d5ebb2afbfb60ec208310d702c826e9e3c131","continuedFrom":"14a109fe026532e70f02ebd7246dae4eafcb0a05ee8fa83929a6b0ab61e899b9","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2\\merge.ts","verdict":null,"at":"2026-09-14T15:24:39.846Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r11","entryId":"1789399472032_000071","agentId":"judge-continuation-16904","pid":16904,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"6667df1c481132eedd71ba2c612d5ebb2afbfb60ec208310d702c826e9e3c131","continuation":null,"at":"2026-09-14T15:24:41.634Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r12","entryId":"1789399483342_000072","at":"2026-09-14T15:24:43.353Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r12","entryId":"1789399483354_000073","at":"2026-09-14T15:24:43.354Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r12","entryId":"1789399483354_000074","at":"2026-09-14T15:24:43.355Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r12","entryId":"1789399483355_000075","at":"2026-09-14T15:24:43.355Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r12","entryId":"1789399483355_000076","at":"2026-09-14T15:24:43.356Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r12","entryId":"1789399483354_000074","agentId":"worker-part1-74184","pid":74184,"stage":"part1","wallMs":7623,"turns":5,"reads":1,"tokens":7408,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"782d82da89d39bf740097028a0b64b7a03bf9b99b8d4fbe1a1e4b1ad1b2fc4df","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T15:24:52.810Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r12","entryId":"1789399483354_000074","agentId":"judge-continuation-15856","pid":15856,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"782d82da89d39bf740097028a0b64b7a03bf9b99b8d4fbe1a1e4b1ad1b2fc4df","continuation":"1789399494622_000077","at":"2026-09-14T15:24:54.623Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r13","entryId":"1789399703288_000078","at":"2026-09-14T15:28:23.298Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r13","entryId":"1789399703298_000079","at":"2026-09-14T15:28:23.299Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r13","entryId":"1789399703299_000080","at":"2026-09-14T15:28:23.299Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r13","entryId":"1789399703299_000081","at":"2026-09-14T15:28:23.300Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r13","entryId":"1789399703300_000082","at":"2026-09-14T15:28:23.300Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r13","entryId":"1789399703299_000080","agentId":"worker-part1-52904","pid":52904,"stage":"part1","wallMs":5203,"turns":4,"reads":1,"tokens":4733,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"f3bd63acca92ddad7d881d189106beea18a9b6ddf2723f9c60f77fdbc0ca620e","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1\\merge.ts","verdict":null,"at":"2026-09-14T15:28:30.159Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r13","entryId":"1789399703299_000080","agentId":"judge-continuation-73288","pid":73288,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"f3bd63acca92ddad7d881d189106beea18a9b6ddf2723f9c60f77fdbc0ca620e","continuation":"1789399711943_000083","at":"2026-09-14T15:28:31.943Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r13","entryId":"1789399711943_000083","agentId":"worker-part2-48092","pid":48092,"stage":"part2","wallMs":11823,"turns":6,"reads":2,"tokens":14936,"cases":7,"passed":7,"failed":[],"conclusion":"promote-candidate","digest":"44b751c14a49bd141f577c1feef614530f2dc8e23ec9ae7ab72e8a2a9a1b4f2e","continuedFrom":"f3bd63acca92ddad7d881d189106beea18a9b6ddf2723f9c60f77fdbc0ca620e","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2\\merge.ts","verdict":null,"at":"2026-09-14T15:28:45.446Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r2","pid":72108,"stage":"part2","failed":["the artifact no longer matches the delivery record: f3bd63acca92ddad7d881d189106beea18a9b6ddf2723f9c60f77fdbc0ca620e != 5a9b4c2397e694dc6bb13579611f30077371096e2408c7805eaee7aa3c8ea20b"],"turns":null,"reads":null,"tokens":null,"sessionId":null,"wallMs":1789400035628,"at":"2026-09-14T15:33:55.628Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r3","pid":29436,"stage":"part2","failed":["the artifact no longer matches the delivery record: f3bd63acca92ddad7d881d189106beea18a9b6ddf2723f9c60f77fdbc0ca620e != ee3338f60c6811a58ca03ba96bd68764136534a19d0948d9c8fa40a7119d8994"],"turns":null,"reads":null,"tokens":null,"sessionId":null,"wallMs":1789400038987,"at":"2026-09-14T15:33:58.987Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r4","pid":71940,"stage":"part2","failed":["the artifact no longer matches the delivery record: f3bd63acca92ddad7d881d189106beea18a9b6ddf2723f9c60f77fdbc0ca620e != 889ae3c17d4fff8eb932c88c608941702e6b5443605b23d024540b50036cdd3e"],"turns":null,"reads":null,"tokens":null,"sessionId":null,"wallMs":1789400042365,"at":"2026-09-14T15:34:02.365Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r7","pid":66524,"stage":"part2","failed":["the artifact no longer matches the delivery record: f3bd63acca92ddad7d881d189106beea18a9b6ddf2723f9c60f77fdbc0ca620e != ee0bbbaae9b09201c2662996a9acfd83e9226920c3d19eab9928f6b56cff69ec"],"turns":null,"reads":null,"tokens":null,"sessionId":null,"wallMs":1789400045655,"at":"2026-09-14T15:34:05.655Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r12","pid":66976,"stage":"part2","failed":["the artifact no longer matches the delivery record: f3bd63acca92ddad7d881d189106beea18a9b6ddf2723f9c60f77fdbc0ca620e != 782d82da89d39bf740097028a0b64b7a03bf9b99b8d4fbe1a1e4b1ad1b2fc4df"],"turns":null,"reads":null,"tokens":null,"sessionId":null,"wallMs":1789400048969,"at":"2026-09-14T15:34:08.969Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r2","pid":45848,"stage":"part2","failed":["the artifact no longer matches the delivery record: f3bd63acca92ddad7d881d189106beea18a9b6ddf2723f9c60f77fdbc0ca620e != 5a9b4c2397e694dc6bb13579611f30077371096e2408c7805eaee7aa3c8ea20b"],"turns":null,"reads":null,"tokens":null,"sessionId":null,"wallMs":1789400160791,"at":"2026-09-14T15:36:00.791Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r3","pid":31084,"stage":"part2","failed":["the artifact no longer matches the delivery record: f3bd63acca92ddad7d881d189106beea18a9b6ddf2723f9c60f77fdbc0ca620e != ee3338f60c6811a58ca03ba96bd68764136534a19d0948d9c8fa40a7119d8994"],"turns":null,"reads":null,"tokens":null,"sessionId":null,"wallMs":1789400162513,"at":"2026-09-14T15:36:02.513Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r4","pid":5424,"stage":"part2","failed":["the artifact no longer matches the delivery record: f3bd63acca92ddad7d881d189106beea18a9b6ddf2723f9c60f77fdbc0ca620e != 889ae3c17d4fff8eb932c88c608941702e6b5443605b23d024540b50036cdd3e"],"turns":null,"reads":null,"tokens":null,"sessionId":null,"wallMs":1789400164283,"at":"2026-09-14T15:36:04.283Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r7","pid":41920,"stage":"part2","failed":["the artifact no longer matches the delivery record: f3bd63acca92ddad7d881d189106beea18a9b6ddf2723f9c60f77fdbc0ca620e != ee0bbbaae9b09201c2662996a9acfd83e9226920c3d19eab9928f6b56cff69ec"],"turns":null,"reads":null,"tokens":null,"sessionId":null,"wallMs":1789400166000,"at":"2026-09-14T15:36:06.000Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r12","pid":62148,"stage":"part2","failed":["the artifact no longer matches the delivery record: f3bd63acca92ddad7d881d189106beea18a9b6ddf2723f9c60f77fdbc0ca620e != 782d82da89d39bf740097028a0b64b7a03bf9b99b8d4fbe1a1e4b1ad1b2fc4df"],"turns":null,"reads":null,"tokens":null,"sessionId":null,"wallMs":1789400167814,"at":"2026-09-14T15:36:07.814Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r14","entryId":"1789400278154_000084","at":"2026-09-14T15:37:58.165Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r14","entryId":"1789400278165_000085","at":"2026-09-14T15:37:58.166Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r14","entryId":"1789400278166_000086","at":"2026-09-14T15:37:58.166Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r14","entryId":"1789400278167_000087","at":"2026-09-14T15:37:58.167Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r14","entryId":"1789400278167_000088","at":"2026-09-14T15:37:58.167Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r14","entryId":"1789400278166_000086","agentId":"worker-part1-53064","pid":53064,"stage":"part1","wallMs":9566,"turns":6,"reads":1,"tokens":10508,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"ee047cacb6c2def020433219429517180f7468988fe8b3c042b1bc711d84c9e1","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1-r14\\merge.ts","verdict":null,"at":"2026-09-14T15:38:09.435Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r14","entryId":"1789400278166_000086","agentId":"judge-continuation-39832","pid":39832,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"ee047cacb6c2def020433219429517180f7468988fe8b3c042b1bc711d84c9e1","continuation":"1789400291114_000089","at":"2026-09-14T15:38:11.114Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r14","pid":26156,"stage":"part2","failed":["invalid patch structure (submitted artifact kept at C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2-r14\\artifact.failed.txt)"],"turns":3,"reads":1,"tokens":2798,"sessionId":"01a0a091-9e40-7588-bdbc-0fdabcd71a42","wallMs":3386,"at":"2026-09-14T15:38:16.285Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r15","entryId":"1789400299633_000090","at":"2026-09-14T15:38:19.635Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r15","entryId":"1789400299635_000091","at":"2026-09-14T15:38:19.636Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r15","entryId":"1789400299636_000092","at":"2026-09-14T15:38:19.636Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r15","entryId":"1789400299636_000093","at":"2026-09-14T15:38:19.637Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r15","entryId":"1789400299637_000094","at":"2026-09-14T15:38:19.637Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r15","entryId":"1789400299636_000092","agentId":"worker-part1-37668","pid":37668,"stage":"part1","wallMs":4621,"turns":4,"reads":1,"tokens":4465,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"b68a1492d81e6549274f95c96f39ab5e6026208c18ccbdd580ac0b008a630f35","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1-r15\\merge.ts","verdict":null,"at":"2026-09-14T15:38:25.971Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r15","entryId":"1789400299636_000092","agentId":"judge-continuation-8128","pid":8128,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"b68a1492d81e6549274f95c96f39ab5e6026208c18ccbdd580ac0b008a630f35","continuation":"1789400307654_000095","at":"2026-09-14T15:38:27.654Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r15","entryId":"1789400307654_000095","agentId":"worker-part2-58232","pid":58232,"stage":"part2","wallMs":10644,"turns":5,"reads":1,"tokens":10493,"cases":7,"passed":7,"failed":[],"conclusion":"no-change-needed","digest":"b68a1492d81e6549274f95c96f39ab5e6026208c18ccbdd580ac0b008a630f35","continuedFrom":"b68a1492d81e6549274f95c96f39ab5e6026208c18ccbdd580ac0b008a630f35","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2-r15\\merge.ts","verdict":null,"at":"2026-09-14T15:38:40.016Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r15","entryId":"1789400307654_000095","agentId":"judge-continuation-28084","pid":28084,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"b68a1492d81e6549274f95c96f39ab5e6026208c18ccbdd580ac0b008a630f35","continuation":null,"at":"2026-09-14T15:38:41.779Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r16","entryId":"1789400323429_000096","at":"2026-09-14T15:38:43.438Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r16","entryId":"1789400323439_000097","at":"2026-09-14T15:38:43.439Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r16","entryId":"1789400323439_000098","at":"2026-09-14T15:38:43.440Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r16","entryId":"1789400323440_000099","at":"2026-09-14T15:38:43.440Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r16","entryId":"1789400323440_000100","at":"2026-09-14T15:38:43.441Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r16","entryId":"1789400323439_000098","agentId":"worker-part1-61932","pid":61932,"stage":"part1","wallMs":4275,"turns":4,"reads":1,"tokens":4233,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"03fb729b3f6690908e8fcf0e073f18ca6214655547d1ce1359148a582df84222","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1-r16\\merge.ts","verdict":null,"at":"2026-09-14T15:38:49.428Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r16","entryId":"1789400323439_000098","agentId":"judge-continuation-28876","pid":28876,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"03fb729b3f6690908e8fcf0e073f18ca6214655547d1ce1359148a582df84222","continuation":"1789400331140_000101","at":"2026-09-14T15:38:51.140Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r16","entryId":"1789400331140_000101","agentId":"worker-part2-69420","pid":69420,"stage":"part2","wallMs":7580,"turns":5,"reads":1,"tokens":7581,"cases":7,"passed":7,"failed":[],"conclusion":"promote-candidate","digest":"30bc27f523d2e7c938dd5f339c26c2b5901561b3ae54012eb6fb38d30084b783","continuedFrom":"03fb729b3f6690908e8fcf0e073f18ca6214655547d1ce1359148a582df84222","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2-r16\\merge.ts","verdict":null,"at":"2026-09-14T15:39:00.427Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r16","entryId":"1789400331140_000101","agentId":"judge-continuation-65392","pid":65392,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"30bc27f523d2e7c938dd5f339c26c2b5901561b3ae54012eb6fb38d30084b783","continuation":null,"at":"2026-09-14T15:39:02.127Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r17","entryId":"1789400343751_000102","at":"2026-09-14T15:39:03.753Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r17","entryId":"1789400343753_000103","at":"2026-09-14T15:39:03.754Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r17","entryId":"1789400343754_000104","at":"2026-09-14T15:39:03.754Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r17","entryId":"1789400343754_000105","at":"2026-09-14T15:39:03.755Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r17","entryId":"1789400343755_000106","at":"2026-09-14T15:39:03.755Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r17","entryId":"1789400343754_000104","agentId":"worker-part1-52884","pid":52884,"stage":"part1","wallMs":4620,"turns":4,"reads":1,"tokens":4410,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"354d6f2cda03ecb19ce7412e76350c0c5060c62a2e4bd48c548ea8555a60d903","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1-r17\\merge.ts","verdict":null,"at":"2026-09-14T15:39:10.102Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r17","entryId":"1789400343754_000104","agentId":"judge-continuation-60324","pid":60324,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"354d6f2cda03ecb19ce7412e76350c0c5060c62a2e4bd48c548ea8555a60d903","continuation":"1789400351787_000107","at":"2026-09-14T15:39:11.787Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r17","entryId":"1789400351787_000107","agentId":"worker-part2-11600","pid":11600,"stage":"part2","wallMs":8016,"turns":5,"reads":2,"tokens":8757,"cases":7,"passed":7,"failed":[],"conclusion":"promote-candidate","digest":"1b2994ba405d3f4ea95bacbfddfca329696c151abc6cee12e30acdffea53f1ed","continuedFrom":"354d6f2cda03ecb19ce7412e76350c0c5060c62a2e4bd48c548ea8555a60d903","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2-r17\\merge.ts","verdict":null,"at":"2026-09-14T15:39:21.599Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r17","entryId":"1789400351787_000107","agentId":"judge-continuation-16160","pid":16160,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"1b2994ba405d3f4ea95bacbfddfca329696c151abc6cee12e30acdffea53f1ed","continuation":null,"at":"2026-09-14T15:39:23.487Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r18","entryId":"1789400365292_000108","at":"2026-09-14T15:39:25.294Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r18","entryId":"1789400365294_000109","at":"2026-09-14T15:39:25.294Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r18","entryId":"1789400365294_000110","at":"2026-09-14T15:39:25.295Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r18","entryId":"1789400365295_000111","at":"2026-09-14T15:39:25.295Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r18","entryId":"1789400365295_000112","at":"2026-09-14T15:39:25.296Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r18","entryId":"1789400365294_000110","agentId":"worker-part1-47304","pid":47304,"stage":"part1","wallMs":4963,"turns":4,"reads":1,"tokens":4460,"cases":2,"passed":0,"failed":["interleaved: expected [1,2,3,4], observed [1,2,3]","one before the other: expected [1,2,3,4], observed [1,2]"],"conclusion":"promote-candidate","digest":"ada2773b1348f8f85cc3fdeb2930a99dcaf67e51b9b0aa490410fc5a573f03d2","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1-r18\\merge.ts","verdict":null,"at":"2026-09-14T15:39:32.079Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r18","entryId":"1789400365294_000110","agentId":"judge-continuation-51348","pid":51348,"stage":"part1","verdict":"rejected","cases":2,"passed":0,"failed":["interleaved: expected [1,2,3,4], observed [1,2,3]","one before the other: expected [1,2,3,4], observed [1,2]"],"artifactDigest":"ada2773b1348f8f85cc3fdeb2930a99dcaf67e51b9b0aa490410fc5a573f03d2","continuation":null,"at":"2026-09-14T15:39:33.957Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r18","pid":67384,"stage":"part2","failed":["no open part2 handoff in ooo-continuation:merge:r18"],"turns":null,"reads":null,"tokens":null,"sessionId":null,"wallMs":1789400375693,"at":"2026-09-14T15:39:35.693Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r19","entryId":"1789400379248_000113","at":"2026-09-14T15:39:39.254Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r19","entryId":"1789400379254_000114","at":"2026-09-14T15:39:39.255Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r19","entryId":"1789400379255_000115","at":"2026-09-14T15:39:39.255Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r19","entryId":"1789400379255_000116","at":"2026-09-14T15:39:39.256Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r19","entryId":"1789400379256_000117","at":"2026-09-14T15:39:39.256Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r19","entryId":"1789400379255_000115","agentId":"worker-part1-58480","pid":58480,"stage":"part1","wallMs":5689,"turns":4,"reads":1,"tokens":4704,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"a315ca06462a27462d8c34c20abd9d815acd4d58601b6a8c029cf7590db6361e","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1-r19\\merge.ts","verdict":null,"at":"2026-09-14T15:39:46.718Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r19","entryId":"1789400379255_000115","agentId":"judge-continuation-58428","pid":58428,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"a315ca06462a27462d8c34c20abd9d815acd4d58601b6a8c029cf7590db6361e","continuation":"1789400388707_000118","at":"2026-09-14T15:39:48.708Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r19","pid":35892,"stage":"part2","failed":["invalid patch structure (submitted artifact kept at C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2-r19\\artifact.failed.txt)"],"turns":3,"reads":1,"tokens":3023,"sessionId":"01a0a093-1c03-7cc5-be87-0e556575227f","wallMs":3866,"at":"2026-09-14T15:39:54.495Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r20","entryId":"1789400726279_000119","at":"2026-09-14T15:45:26.281Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r20","entryId":"1789400726281_000120","at":"2026-09-14T15:45:26.282Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r20","entryId":"1789400726282_000121","at":"2026-09-14T15:45:26.282Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r20","entryId":"1789400726282_000122","at":"2026-09-14T15:45:26.283Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r20","entryId":"1789400726283_000123","at":"2026-09-14T15:45:26.283Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r20","entryId":"1789400726282_000121","agentId":"worker-part1-27264","pid":27264,"stage":"part1","wallMs":5508,"turns":5,"reads":2,"tokens":6313,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"970b27e5fcbabf1e044f24caa94f2530800d36f8427366c1951ffbee5b099bc3","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1-r20\\merge.ts","verdict":null,"at":"2026-09-14T15:45:33.584Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r20","entryId":"1789400726282_000121","agentId":"judge-continuation-59628","pid":59628,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"970b27e5fcbabf1e044f24caa94f2530800d36f8427366c1951ffbee5b099bc3","continuation":"1789400735558_000124","at":"2026-09-14T15:45:35.558Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r20","entryId":"1789400735558_000124","agentId":"worker-part2-35576","pid":35576,"stage":"part2","wallMs":7289,"turns":4,"reads":1,"tokens":6244,"cases":7,"passed":7,"failed":[],"conclusion":"promote-candidate","digest":"1024dd9c8dffbce2dff8bbe0336577d8e39001805564aa509f7a9a30f2f3913f","continuedFrom":"970b27e5fcbabf1e044f24caa94f2530800d36f8427366c1951ffbee5b099bc3","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2-r20\\merge.ts","verdict":null,"at":"2026-09-14T15:45:44.703Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r20","entryId":"1789400735558_000124","agentId":"judge-continuation-69084","pid":69084,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"1024dd9c8dffbce2dff8bbe0336577d8e39001805564aa509f7a9a30f2f3913f","continuation":null,"at":"2026-09-14T15:45:46.415Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r21","entryId":"1789400748135_000125","at":"2026-09-14T15:45:48.138Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r21","entryId":"1789400748138_000126","at":"2026-09-14T15:45:48.139Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r21","entryId":"1789400748139_000127","at":"2026-09-14T15:45:48.139Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r21","entryId":"1789400748139_000128","at":"2026-09-14T15:45:48.139Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r21","entryId":"1789400748140_000129","at":"2026-09-14T15:45:48.140Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r21","entryId":"1789400748139_000127","agentId":"worker-part1-43188","pid":43188,"stage":"part1","wallMs":5031,"turns":4,"reads":1,"tokens":4723,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"970b27e5fcbabf1e044f24caa94f2530800d36f8427366c1951ffbee5b099bc3","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1-r21\\merge.ts","verdict":null,"at":"2026-09-14T15:45:54.901Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r21","entryId":"1789400748139_000127","agentId":"judge-continuation-58108","pid":58108,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"970b27e5fcbabf1e044f24caa94f2530800d36f8427366c1951ffbee5b099bc3","continuation":"1789400758443_000130","at":"2026-09-14T15:45:58.443Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r21","entryId":"1789400758443_000130","agentId":"worker-part2-38048","pid":38048,"stage":"part2","wallMs":7437,"turns":4,"reads":1,"tokens":6275,"cases":7,"passed":7,"failed":[],"conclusion":"promote-candidate","digest":"c32a7af3323a331853b241d4e7a23691442350de93d37941d605f22c0aa618da","continuedFrom":"970b27e5fcbabf1e044f24caa94f2530800d36f8427366c1951ffbee5b099bc3","artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2-r21\\merge.ts","verdict":null,"at":"2026-09-14T15:46:07.774Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r21","entryId":"1789400758443_000130","agentId":"judge-continuation-30604","pid":30604,"stage":"parent","verdict":"accepted","cases":7,"passed":7,"failed":[],"artifactDigest":"c32a7af3323a331853b241d4e7a23691442350de93d37941d605f22c0aa618da","continuation":null,"at":"2026-09-14T15:46:09.630Z"} +{"role":"plan","task":"chunk","channel":"ooo-continuation:chunk:r22","entryId":"1789400771384_000131","at":"2026-09-14T15:46:11.386Z"} +{"role":"plan","task":"duration","channel":"ooo-continuation:duration:r22","entryId":"1789400771386_000132","at":"2026-09-14T15:46:11.387Z"} +{"role":"plan","task":"merge","channel":"ooo-continuation:merge:r22","entryId":"1789400771387_000133","at":"2026-09-14T15:46:11.387Z"} +{"role":"plan","task":"average","channel":"ooo-continuation:average:r22","entryId":"1789400771387_000134","at":"2026-09-14T15:46:11.388Z"} +{"role":"plan","task":"paths","channel":"ooo-continuation:paths:r22","entryId":"1789400771388_000135","at":"2026-09-14T15:46:11.388Z"} +{"role":"part1","task":"merge","channel":"ooo-continuation:merge:r22","entryId":"1789400771387_000133","agentId":"worker-part1-52432","pid":52432,"stage":"part1","wallMs":5254,"turns":4,"reads":1,"tokens":4710,"cases":2,"passed":2,"failed":[],"conclusion":"promote-candidate","digest":"1ec30c9c8828b3a6c22092cc73d53aa62cd52b95c651f18a042441c0763c2249","continuedFrom":null,"artifact":"C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part1-r22\\merge.ts","verdict":null,"at":"2026-09-14T15:46:18.516Z"} +{"role":"judge","task":"merge","channel":"ooo-continuation:merge:r22","entryId":"1789400771387_000133","agentId":"judge-continuation-71816","pid":71816,"stage":"part1","verdict":"accepted","cases":2,"passed":2,"failed":[],"artifactDigest":"1ec30c9c8828b3a6c22092cc73d53aa62cd52b95c651f18a042441c0763c2249","continuation":"1789400780311_000136","at":"2026-09-14T15:46:20.312Z"} +{"role":"part2","task":"merge","channel":"ooo-continuation:merge:r22","pid":39872,"stage":"part2","failed":["invalid patch structure (conclusion=cannot-complete, submitted artifact kept at C:\\Users\\LEGION\\AppData\\Local\\Temp\\nmg-board-verbs\\.nmg\\ooo-continuation\\m6\\merge\\part2-r22\\artifact.failed.txt)"],"turns":3,"reads":1,"tokens":3454,"sessionId":"01a0a099-1520-7a76-b5ef-bfae558bfed7","wallMs":6293,"at":"2026-09-14T15:46:28.376Z"} diff --git a/docs/experiments/execution/ooo-retention-2026-09-13.md b/docs/experiments/execution/ooo-retention-2026-09-13.md index 1fe058b4..d5da8a82 100644 --- a/docs/experiments/execution/ooo-retention-2026-09-13.md +++ b/docs/experiments/execution/ooo-retention-2026-09-13.md @@ -1,9 +1,9 @@ # Retention, and the verdict the round no longer copies **Related:** [task-unit semantics design](../../design/task-unit-semantics.md) · -[proposed decision record](../../decisions/proposed/2026-09-13-task-unit-semantics.md) · +[proposed decision record](../../decisions/implemented/2026-09-13-task-unit-semantics.md) · [the single acceptance predicate](ooo-acceptance-predicate-2026-09-13.md) · -[board deliverable/verdict decision](../../decisions/proposed/2026-09-06-board-governance-addressing.md) +[board deliverable/verdict decision](../../decisions/implemented/2026-09-06-board-governance-addressing.md) Measured 2026-09-13. Zero model tokens. diff --git a/docs/experiments/execution/ooo-round-transition-2026-09-13.md b/docs/experiments/execution/ooo-round-transition-2026-09-13.md new file mode 100644 index 00000000..6281a8d1 --- /dev/null +++ b/docs/experiments/execution/ooo-round-transition-2026-09-13.md @@ -0,0 +1,47 @@ +# A round's transition and the board write inside it - 2026-09-13 + +The design's integration contract says the composed write entries reuse one implementation: standalone +they open the transaction boundary, and inside a transition they join it through the port the store +issued. This slice does the round's half. + +## What changed + +- `BoardAdmission.transaction()` no longer runs BEGIN/COMMIT/ROLLBACK of its own: it delegates to the + store's `writeTransaction()`, so the round's transitions _are_ the store's boundary. +- `publish()` and `publishReady()` take the optional port and hand it to `putTaskBoardEntry()`, so a + publication made inside a transition is part of that transition instead of a second BEGIN (which the + store refuses rather than nesting). + +No caller publishes inside a transition today — publishing happens after the outer commit, which is +what the design's static review says and what the green suites before this change confirmed. The point +of the change is that the rule no longer depends on that ordering: a publication reached inside a +transition now joins it, and a publication reached without a port is refused by name. + +## Evidence (re-runnable) + +- `npm test` - **1466 passed, 0 failed**. +- `tests/integration/ooo-transition-atomicity.test.ts` - **3 of 3**: a publication commits with the + transition; the round's own publication rolls back when the transition fails after it (the check a + second BEGIN cannot pass); a publication without a port inside a transition is refused and writes + nothing. +- `node --experimental-strip-types --test "evals/ooo-execution/*.test.ts"` - 88 passed, 0 failed. +- `npm run mutation:teeth` - **26 of 26 mutants caught by name across 6 targets, 6 of 6 restored + byte-identically, 0 inapplicable** (one added here: a round publication that opens its own + transaction). +- `npm run check`, `npm run docs:check`, `npm run complexity:gate` - clean. + +A commit-hygiene note, because it was caught by accident rather than by a gate: B2's adaptation of +`tests/integration/ooo-run-namespace.test.ts` (the raw reads that follow the new projection) was left +out of B2's commits, so the branch as pushed had a red suite in it. It is committed separately now, and +the failure mode is worth remembering: a refactor that moves a table can break a test in a file the +refactor never touched. + +## What this does not do + +- **Four store methods still begin their own transactions** (`removeMemoryFromChain`, + `upsertExternalNodeEmbeddings`, `upsertExternalLeafEmbeddings`, `upsertExternalEmbeddings`), so "the + store is the single BEGIN owner" is still not literally true. +- **The lifecycle half is not started**: no operations port carrying no `close()`, no borrowed view for + `status` (it must not migrate schema, publish handoffs or write), no daemon close order, and the + confirmed cancelled-round temporary-directory leak (`src/integration/ooo-cycle.ts:814` returns before + the `finally` at line 1037 that removes the default directory). diff --git a/docs/experiments/execution/ooo-run-namespace-2026-09-13.md b/docs/experiments/execution/ooo-run-namespace-2026-09-13.md new file mode 100644 index 00000000..21269314 --- /dev/null +++ b/docs/experiments/execution/ooo-run-namespace-2026-09-13.md @@ -0,0 +1,57 @@ +# Run-scoped round storage — 2026-09-13 + +The design's migration step 2 (`docs/design/task-unit-semantics.md`, "persistence"): a round's +storage is namespaced by run, so one store can hold several rounds, and every place that used to +be a singleton is keyed by `(run_id, …)`. + +## What this does not do + +- **The table is still one table.** `ooo_probe_tasks` holds the immutable manifest, the candidate + bytes and the derived state in the same row; splitting it is the next step (B2), not this one. +- **The round still owns its SQLite file.** The store is opened by the round rather than injected + from the daemon (B3). Namespacing makes the _later_ injection a change of owner, not a rewrite of + every statement — that is the reason for the order. +- **`checkId` is deliberately not run-scoped.** It is the identity a round log records, and a + replay runs under a new `runId` while reproducing the same attempts; uniqueness in the store is + the `(run_id, task_id)` key. Scoping it first _broke_ replay (`replay asks for exactly the +recorded attempts: 2 !== 3`), which is how the constraint was found rather than assumed. +- **Acceptance is not restored from a migrated store's old column.** A pre-namespace store recorded + the round's own verdict; that is the self-report the board's independent verdict replaced. The + migration keeps the artifact and leaves it unaccepted — visible, not silently accepted. + +## What changed + +- `BoardAdmissionOptions { runId? }`; `ooo_probe_runs (run_id PK, policy, cancel_reason, +cancelled_at, created_at)` replaces `ooo_probe_meta (id PK CHECK(id=1))`; `ooo_probe_tasks` and + `ooo_probe_checks` gain `run_id` with composite primary keys; the board channel is + `ooo-probe:` instead of the constant `ooo-process-probe`. +- Adoption: a named run opens it; a store with exactly one run is continued (so existing behaviour + is unchanged); a store with several runs **refuses** and asks to be named, because adopting the + newest and starting another are both silent answers to somebody's evidence. +- `migrateToRunScope()` rebuilds the legacy tables in a transaction (rename → create → copy → + drop), and throws rather than guessing if legacy rows exist without a `meta` row. A refused store + closes its handle: on Windows the caller is told to open a different database, and an open handle + would keep that file locked (found by the test's cleanup, then fixed). +- ~37 SQL sites now carry `run_id`; both prune paths and retention are per run. + +## Evidence + +`npm test` — **1452 passed, 0 failed** (product, includes the 4 new tests). +`tests/integration/ooo-run-namespace.test.ts` — **4 of 4**: two runs in one store do not collide, +do not see each other's rows, and cancel separately; a multi-run store refuses to guess; a one-run +store is still continued; a pre-namespace store is migrated in place keeping its run, its rows and +its terminal decision (which the migrated round then _enforces_, refusing a claim). +`npm run mutation:teeth` — **22 of 22 mutants caught by name across 6 targets, 6 of 6 restored +byte-identically, 0 inapplicable.** + +Two things this exercise found rather than assumed: + +1. The `claim-is-not-scoped-to-its-run` tooth **was not caught** by the first version of the test: + `next()` and `accepted()` do not observe a neighbouring run's row. The assertion that does is a + raw-row read of the other run's `owner` after this run claims the same task id. A tooth that + runs and passes is the difference between a test that describes the change and one that pins it. +2. Scoping `checkId` by run looked like the obvious completion of "one row per run" and was wrong: + replay is a _different_ run reproducing the _same_ attempts, and the round suites said so. + +Both are recorded here rather than in the commit message, because a reviewer can re-run these +commands and a commit message is not evidence. diff --git a/docs/experiments/execution/ooo-single-boundary-2026-09-13.md b/docs/experiments/execution/ooo-single-boundary-2026-09-13.md new file mode 100644 index 00000000..9f183932 --- /dev/null +++ b/docs/experiments/execution/ooo-single-boundary-2026-09-13.md @@ -0,0 +1,37 @@ +# One transaction boundary in the store, and a check that keeps it - 2026-09-13 + +The integration contract requires that the store is the only place that runs BEGIN/COMMIT/ROLLBACK. +After the port primitive (previous record) four store methods still opened their own: the memory-chain +removal and the three external-embedding upserts. This slice moves them onto the boundary and makes the +invariant mechanical. + +## What changed + +- `removeMemoryFromChain`, `upsertExternalNodeEmbeddings`, `upsertExternalLeafEmbeddings` and + `upsertExternalEmbeddings` call `writeTransaction()` instead of BEGIN/try/COMMIT/catch/ROLLBACK. + `src/core/store/base.ts` now contains exactly one BEGIN, one COMMIT and one ROLLBACK, and all three + are the boundary's own. +- The embedding upserts' cache refresh stays **outside** the transaction, where it already was: a batch + whose transaction did not commit must not warm the cache either. The conversion preserved that + instead of quietly pulling it inside. +- The invariant is a test, not a convention: `the store runs its transaction boundary in exactly one +place` counts the three statements in the source and checks that the BEGIN and COMMIT belong to + `writeTransaction` and that every `this.rollback()` call is one the boundary made. A behavioural + test cannot see a second boundary that happens to work. + +## Evidence (re-runnable) + +- `npm test` - **1467 passed, 0 failed**. +- `tests/core/store-transaction-port.test.ts` - **9 of 9** (eight refusals plus the one-boundary + invariant). +- `node --experimental-strip-types --test "evals/ooo-execution/*.test.ts"` - 88 passed, 0 failed. +- `npm run mutation:teeth` - **27 of 27 mutants caught by name across 6 targets, 6 of 6 restored + byte-identically, 0 inapplicable** (one added here: a method that opens its own transaction). +- `npm run check`, `npm run docs:check`, `npm run complexity:gate` - clean. + +## What this does not do + +The lifecycle half is still untouched: no operations port that does not carry `close()`, no borrowed +view for `status` (it must not migrate schema, publish handoffs or write), no daemon close order, and +the confirmed cancelled-round temporary-directory leak (`src/integration/ooo-cycle.ts:814` returns +before the `finally` at line 1037 that removes the default directory). diff --git a/docs/experiments/execution/ooo-task-tables-2026-09-13.md b/docs/experiments/execution/ooo-task-tables-2026-09-13.md new file mode 100644 index 00000000..8067f4dc --- /dev/null +++ b/docs/experiments/execution/ooo-task-tables-2026-09-13.md @@ -0,0 +1,66 @@ +# A task row's three kinds of fact, split into three tables — 2026-09-13 + +The design's persistence section (`docs/design/task-unit-semantics.md:200-208`) says a persisted +fact may be one of two kinds — the immutable run manifest, or an append-only run fact the board does +not have — and that **derived state has no authoritative storage**: `ready`/`blocked`/`accepted`, +dependency satisfaction, validity after cancellation and fusion candidates are computed. One row in +`ooo_probe_tasks` held all three at once, which is why the previous migration had to _decide_ not to +trust an old `accepted_entry_id` column, and why "the derived value can be rebuilt" was an assertion +rather than a test. + +## The shapes + +- `ooo_probe_manifest (run_id, id, revision, input, dependencies, position, effect, +source_revision, wait_event, operation, kind, patch_files, patch_editable)` — written once when the + plan is installed, never updated afterwards. +- `ooo_probe_facts (…, attempt, artifact, entry_id, external_ready, observed_revision, owner, +claim_time)` — what the round appended and the board does not carry. +- `ooo_probe_derived (…, input_digest)` — one column, because one thing is recomputable here. +- `ooo_probe_task_view` — a projection (manifest ⋈ facts ⋈ cache), not storage. Reads use it, so a + missing cache row degrades to the manifest's own answer instead of a lost fact. + +## What the recomputation check found + +The first version of the split put `owner` and `claim_time` in the cache, on the reasoning that the +board knows who holds a claim. Deleting the cache and rebuilding made that **wrong**: the round +resolves the entry when it accepts the artifact, and after that the board stops reporting +`claimedBy`, so a rebuild re-derived `null` where the cache had held the reviewer's name. Who claimed +is therefore a fact — it is what happened, and it is not still readable from the source — and it +moved to `ooo_probe_facts`. This is the kind of thing the design's check exists to catch: the split +looked right in the schema and was wrong in the sources. + +What remains in the cache is genuinely recomputable: `input_digest`, from the frozen manifest alone +(the ticket that mints it on a claim could be lost with the round's memory and rebuilt from the +manifest's bytes). The test deletes the whole cache, rebuilds it, compares the rebuilt values, and +then **delivers the next task in the round** — a rebuilt digest that were merely equal but unusable +would pass the comparison and fail the delivery. + +## Evidence (re-runnable) + +- `npm test` — **1455 passed, 0 failed** (product, includes the 3 new tests). +- `tests/integration/ooo-task-tables.test.ts` — **3 of 3**: a claim writes the facts and the cache but + touches no manifest byte; deleting the cache and rebuilding yields the same cache and a usable + round; corrupting the cache is repaired by the sources rather than trusted. +- `tests/integration/ooo-run-namespace.test.ts` — **4 of 4** unchanged (a pre-namespace store still + migrates, now straight into the three tables). +- `node --experimental-strip-types --test "evals/ooo-execution/*.test.ts"` — **88 passed, 0 failed** + (the round suites, including replay). +- `npm run mutation:teeth` — **23 of 23 mutants caught by name across 6 targets, 6 of 6 restored + byte-identically, 0 inapplicable** (the rebuild tooth was added here; the claim tooth moved with + the statement it pins). +- `npm run docs:check`, `npm run complexity:gate`, `npm run check` — clean. + +Two mechanical notes, because they cost real time and will recur: a raw SQL anchor that crossed a +method boundary silently deleted `transaction()` and `ensureColumns()` (found by `npm run check`, +not by reading the diff), and a mutation marker must be copied from the file's bytes — prettier +rewrapped the line this tooth targets, so a hand-typed marker matched nothing until it was retaken. + +## What this does not do + +- **The round still opens its own SQLite file.** Composing the daemon-owned store is the next step, + and the namespace plus this split are what make it a change of owner rather than a rewrite. +- **Facts are keyed by `(run_id, id)`, not append-only by sequence.** The attempt counter and the + latest candidate are facts; earlier attempts are not separately retained. The design's + `runFactsThroughSequence` view would want them, and the split is the place that would hold them. +- **The run-level manifest is still `ooo_probe_runs`** (policy, cancellation). It is immutable in + practice, but it is not yet expressed in the manifest's terms. diff --git a/docs/experiments/execution/ooo-transaction-port-2026-09-13.md b/docs/experiments/execution/ooo-transaction-port-2026-09-13.md new file mode 100644 index 00000000..a63c99c9 --- /dev/null +++ b/docs/experiments/execution/ooo-transaction-port-2026-09-13.md @@ -0,0 +1,60 @@ +# The store owns the transaction boundary, and a port is how work joins it - 2026-09-13 + +The design's "事务参与与连接生命周期" contract (docs/design/task-unit-semantics.md) requires that one +connection is not one transaction, that the store is the only place that runs BEGIN/COMMIT, and that +a caller already inside a transition joins it through a capability issued for that transition. This +slice implements that primitive and moves the one boundary the contract names by hand +(`putTaskBoardEntry`), with the refusals as tests. + +## What changed + +- `NmgStoreBase.writeTransaction(callback)` is the only BEGIN/COMMIT. It refuses to open a second + transaction while one is live, refuses to run at all on a quarantined connection, and refuses a + callback that returns a thenable - a port must not survive an await. +- `withPort(port, work)` joins the live transition. Validity is identity against the store's own + open transaction, so a port from another store, a port whose callback has returned, and a port that + was never issued are all refused by name. Nothing infers authority from a flag or a depth counter. +- A failure inside `withPort` marks the transaction rollback-only even when the caller catches it: + the work up to the failure already happened, and only the outermost decides whether anything + commits. +- A failed ROLLBACK keeps the original error and quarantines the connection, because its state is no + longer known. +- `putTaskBoardEntry(input, port?)` is now a thin entry over `insertTaskBoardEntry`: standalone it + opens the boundary, composed it joins the caller's, and reached inside a transition without a port + it is refused rather than nested. Its own BEGIN and catch/ROLLBACK are gone, so there is one + implementation for both paths. + +## Evidence (re-runnable) + +- `npm test` - **1463 passed, 0 failed** (1455 before this slice). +- `tests/core/store-transaction-port.test.ts` - **8 of 8**: the committed value is returned; a write + inside a transition without a port is refused and leaves nothing; a transition cannot open another; + another store's port is refused; a port used after its callback returned is refused; a thenable + callback is refused; a swallowed failure still forbids the commit; a failed rollback quarantines + the connection. +- `node --experimental-strip-types --test "evals/ooo-execution/*.test.ts"` - 88 passed, 0 failed. +- `npm run mutation:teeth` - **25 of 25 mutants caught by name across 6 targets, 6 of 6 restored + byte-identically, 0 inapplicable** (two added here: a nested write transaction becoming allowed, + and a swallowed failure still committing). +- `npm run check`, `npm run docs:check`, `npm run complexity:gate` - clean. + +Confirmed by reading the code, not restated from the design: the board methods used inside the +round's own transitions do not open a second BEGIN today (publishing happens after the outer commit), +which is why the whole product and research suites stayed green while the boundary moved. The design's +static finding about the cancelled-round cleanup is also correct as written: `runCycle` returns from +its early-cancel branch after `gate.close()` (src/integration/ooo-cycle.ts:814) while the default +temporary directory is only removed in the `finally` (line 1037), so a cancelled round leaves it +behind. + +## What this does not do + +- **The round still runs its own transaction.** `BoardAdmission.transaction()` is unchanged and does + not yet thread a port into its composed board writes (deliver, judge, resolve, retention). The + contract wants both boundaries changed, and doing it means every composed write inside a transition + takes the port explicitly. +- **Four store methods still begin their own transactions** (`removeMemoryFromChain`, + `upsertExternalNodeEmbeddings`, `upsertExternalLeafEmbeddings`, `upsertExternalEmbeddings`). They + are outside the board path, but "the store is the single BEGIN owner" is not yet literally true. +- **The lifecycle half is not started**: no operations port without `close()`, no borrowed view for + `status`, no daemon close order (stop new work, fence in-flight attempts, drain, checkpoint, close), + and the cancelled-round directory leak above is still there. diff --git a/docs/experiments/execution/task-semantics-design-cases-2026-09-13.md b/docs/experiments/execution/task-semantics-design-cases-2026-09-13.md index 7eb76f52..4bcf8928 100644 --- a/docs/experiments/execution/task-semantics-design-cases-2026-09-13.md +++ b/docs/experiments/execution/task-semantics-design-cases-2026-09-13.md @@ -1,7 +1,7 @@ # The design's enumerated cases, made executable **Related:** [task-unit semantics design](../../design/task-unit-semantics.md) · -[proposed decision record](../../decisions/proposed/2026-09-13-task-unit-semantics.md) · +[proposed decision record](../../decisions/implemented/2026-09-13-task-unit-semantics.md) · [slice 1: compiler and model](task-semantics-slice1-2026-09-13.md) · [one acceptance predicate](ooo-acceptance-predicate-2026-09-13.md) diff --git a/docs/experiments/execution/task-semantics-slice1-2026-09-13.md b/docs/experiments/execution/task-semantics-slice1-2026-09-13.md index 79e0c5fc..7101f8af 100644 --- a/docs/experiments/execution/task-semantics-slice1-2026-09-13.md +++ b/docs/experiments/execution/task-semantics-slice1-2026-09-13.md @@ -1,7 +1,7 @@ # The design's first slice: a shared pure-data compiler and a finite offline model **Related:** [task-unit semantics design](../../design/task-unit-semantics.md) · -[proposed decision record](../../decisions/proposed/2026-09-13-task-unit-semantics.md) · +[proposed decision record](../../decisions/implemented/2026-09-13-task-unit-semantics.md) · [OoO bootstrap design](../../design/ooo-execution-bootstrap.md) · [what the wait is worth](wait-value-2026-09-12.md) diff --git a/docs/experiments/execution/wait-value-2026-09-12.md b/docs/experiments/execution/wait-value-2026-09-12.md index 6c145882..1abc8d7c 100644 --- a/docs/experiments/execution/wait-value-2026-09-12.md +++ b/docs/experiments/execution/wait-value-2026-09-12.md @@ -2,7 +2,7 @@ **Related:** [design](../../design/ooo-execution-bootstrap.md) · [admission run](../ooo-admission-2026-09-08.md) · -[speculation decision](../../decisions/proposed/2026-09-11-ooo-speculation.md) +[speculation decision](../../decisions/implemented/2026-09-11-ooo-speculation.md) Measured 2026-09-12. Zero model tokens: no provider was contacted for any number below. diff --git a/docs/postmortem/0001-tools-outside-the-type-check.md b/docs/postmortem/0001-tools-outside-the-type-check.md new file mode 100644 index 00000000..97d28543 --- /dev/null +++ b/docs/postmortem/0001-tools-outside-the-type-check.md @@ -0,0 +1,87 @@ +# 0001 - The tool that checks the others is not itself checked + +[中文](0001-tools-outside-the-type-check.zh-CN.md) + +**Status:** open + +## Executive summary + +Two mutant anchors whose newlines I had written through a shell text path reached `tools/mutation-teeth.ts` +as real newlines inside a TypeScript string literal, so the tool died with `ERR_INVALID_TYPESCRIPT_SYNTAX` +instead of running - twice in one session, through the same mechanism. Nothing type-checks `tools/**`: +`npm run check` exits **0** with `const x: number = "not a number";` planted in a `tools/` file, because +`tsconfig.json` names exactly three `tools/` files and ESLint does not cover the directory. The type error +class this hides is measurable: adding `tools/**/*.ts` to the include list produces **3 errors in 2 files**. +The lesson: an allow-list of checked files makes "unchecked" the default for every new tool, and the missing +check is exactly what a mechanical edit path will find. + +## Summary + +While adding a mutation tooth I generate an anchor and a replacement as TypeScript string literals. Writing +them through a shell or Python path converts intended `\n` escapes into real line breaks, which is not a +type error but a syntax error in the module - so the file is broken at rest and only fails when Node loads +it. Both times the repair was a rewrite through an editor that treats the text literally. + +The systemic half is not the escaping mistake; it is that the repository had no way to notice. `tsconfig.json` +includes `src/**`, `.pi/extensions/**`, `claude-plugins/**`, and three named files under `tools/` +(`autodiff-benchmark.ts`, `repo-context.ts`, `agent-verify.ts`). `tools/mutation-teeth.ts` is not among +them, and ESLint's configured globs exclude the directory. A tool whose output other reviewers treat as +evidence - "27 of 27 mutants caught" - is therefore the least checked code in the change. + +## Impact + +No false evidence was produced: the tool refuses to start, so it never reported a mutant as caught, and no +number from it was published while it was broken. The cost was two debugging rounds spent on a failure the +type checker would have reported immediately. + +What was hidden is different and larger: type-level rot in any unchecked `tools/` script is invisible. On the +tree this was measured against, extending the include list surfaces `tools/fork-merge-demo.ts(89,39)` and +`(90,39)` (`TS18046`, `json.left`/`json.right` are `unknown`) and one `TS2322` in `tools/complexity-gate.ts`. +Those errors are pre-existing, not introduced here, and they are what the guardrail would have to clear first. + +## Timeline + +- B3b, adding the tooth `round-publication-opens-its-own-transaction`: the anchor was written through a shell + path, the escapes became newlines, and `npm run mutation:teeth` failed with `ERR_INVALID_TYPESCRIPT_SYNTAX`. + Repaired by re-writing the same literal through an editor that preserves `\n`. +- The single-boundary slice, adding the tooth `a-method-opens-its-own-transaction`: the same write path + produced the same defect in the same file. Both the write and the repair repeated. +- Probing the root cause: `const x: number = "not a number";` in a new `tools/` file, then `npm run check` - + **exit 0**. +- Measuring the guardrail's cost: a copy of `tsconfig.json` with `tools/**/*.ts` added - **3 errors, 2 files**. + +## Root cause + +The set of type-checked files is an allow-list, so a new tool joins the unchecked set by default and nothing +announces it. An unchecked file cannot fail a check, and a syntax error in a file that is never loaded by a +test is not observed at all - it is observed by the first person who runs it, in the middle of a run whose +partial output looks like progress. Any mechanical transformation that writes TypeScript source without +respecting its escaping (shell heredocs, Python string writes, `sed`) lands in that blind spot. + +This is why "I will be careful" is not the fix: the edit that broke it and the edit that repaired it were the +same keystrokes in a different tool, and only one of them was checkable. + +## Guardrails added + +Not yet. The candidate is a one-line change to `tsconfig.json`: + +```json +"tools/**/*.ts" +``` + +with the 3 measured errors cleared first. It is not landed here because the CI contract's owner is editing +the same files in the same worktree; landing it from this session would have made two in-flight changes +overlap. Until it lands, this class is still uncaught and the record stays `open`. + +The instance-level habit that survived is narrower and worth stating: mutant anchors are written with an +editor that preserves `\n`, never generated through a shell string. + +## Lessons + +- An allow-list of checked files is a promise about existing files; it says nothing about the next one. + Prefer a glob that includes the directory, and let the exceptions be explicit. +- "It is only a tool" is the wrong direction for scrutiny. The more a tool's output is quoted as evidence, + the more it needs the same checks as the code it judges - the mutation tool is the extreme case, because + its output is a count that nobody re-derives by hand. +- A failure that happens _before_ the tool starts is a lucky failure. The same blind spot would have hidden + a wrong result just as well. diff --git a/docs/postmortem/0001-tools-outside-the-type-check.zh-CN.md b/docs/postmortem/0001-tools-outside-the-type-check.zh-CN.md new file mode 100644 index 00000000..e47d8c6b --- /dev/null +++ b/docs/postmortem/0001-tools-outside-the-type-check.zh-CN.md @@ -0,0 +1,73 @@ +# 0001 - 检查别人的工具,自己没被检查 + +[English](0001-tools-outside-the-type-check.md) + +**Status:** open + +## Executive summary + +我两次把 mutant 锚点经 shell 文本路径写入 `tools/mutation-teeth.ts`,本意的 `\n` 变成字面换行,落在一个 +TypeScript 字符串字面量里;工具于是以 `ERR_INVALID_TYPESCRIPT_SYNTAX` 死掉而不是运行——同一会话、同一机制、 +两次。而 `tools/**` 根本不被类型检查:在一个 `tools/` 文件里放 `const x: number = "not a number";`, +`npm run check` 仍然 **exit 0**——因为 `tsconfig.json` 只逐名包含 `tools/` 的三个文件,ESLint 的配置也不覆盖该目录。 +它掩盖的错误类别可以量:把 `tools/**/*.ts` 加入 include,出现 **2 个文件 3 个错误**。 +结论:用白名单限定"被检查的文件",就等于让每个新工具默认不被检查——而缺的正是这个检查。 + +## Summary + +我在加 mutation tooth 时把锚点与替换写成 TS 字符串字面量。经 shell 或 Python 写文件会把本意的转义变成真换行; +这不是类型错误,而是模块的语法错误——所以文件在静态上就是坏的,只有 Node 加载它时才报错。两次的修复都是改用 +按字面处理文本的编辑器重写。 + +系统性的那一半不是转义失误本身,而是仓库当时没有办法发现它。`tsconfig.json` 包含 `src/**`、 +`.pi/extensions/**`、`claude-plugins/**` 以及 `tools/` 下**逐名列出**的三个文件 +(`autodiff-benchmark.ts`、`repo-context.ts`、`agent-verify.ts`),不含 `tools/mutation-teeth.ts`; +ESLint 的 glob 也不覆盖该目录。于是一个其输出被其他评审当作证据的工具("27/27 mutant 命中")反而成了 +本次变更里被检查得最少的代码。 + +## Impact + +没有产出假证据:工具拒绝启动,所以它从未把没能运行的 mutant 记成"命中",损坏期间也没有发布任何数字。 +代价是两轮调试,花在一个类型检查本可立刻报出的失败上。 + +被掩盖的部分不同、也更大:任何未被检查的 `tools/` 脚本里的类型腐坏都是不可见的。在本记录量测的那棵树上 +扩展 include,暴露出 `tools/fork-merge-demo.ts(89,39)`、`(90,39)`(`TS18046`,`json.left`/`json.right` 为 +`unknown`)以及 `tools/complexity-gate.ts` 的一个 `TS2322`。这些错误是既有的、不是本次引入的,也正是该 +guardrail 落地前必须先清掉的东西。 + +## Timeline + +- B3b 加牙 `round-publication-opens-its-own-transaction`:锚点经 shell 路径写入,转义变成换行, + `npm run mutation:teeth` 以 `ERR_INVALID_TYPESCRIPT_SYNTAX` 失败;用能保留 `\n` 的编辑器重写同一字面量修好。 +- 单一边界那一刀,加牙 `a-method-opens-its-own-transaction`:同一写入路径在同一文件里产生同一缺陷,写入与修复都重演。 +- 验证根因:在一个新的 `tools/` 文件里写 `const x: number = "not a number";`,`npm run check` —— **exit 0**。 +- 量 guardrail 成本:复制 `tsconfig.json` 并加入 `tools/**/*.ts` —— **2 个文件 3 个错误**。 + +## Root cause + +"被类型检查的文件"是一个白名单,于是新工具默认进入未被检查的集合,而且没有任何东西提示这件事。未被检查的 +文件无法让检查失败;一个不被任何测试加载的文件里的语法错误根本不会被观测到——它由第一个运行它的人观测到, +且出现在一次输出看起来像进展的运行中途。任何不尊重 TypeScript 转义而写出源码的机械变换(shell heredoc、 +Python 写字符串、`sed`)都会落进这个盲区。 + +所以"我会小心"不是解法:弄坏它的编辑与修好它的编辑,是同一批按键换了工具,而只有其中一个可被检查。 + +## Guardrails added + +尚未。候选是 `tsconfig.json` 里的一行: + +```json +"tools/**/*.ts" +``` + +前提是先清掉量到的 3 个错误。本记录没有落地它,是因为 CI 契约的拥有者正在同一工作树里改同样的文件, +从这里落地会让两处在飞中的变更重叠。在它落地之前,这个类别仍未被抓到,记录保持 `open`。 + +保留下来的实例级习惯更窄,但值得写明:mutant 锚点只用保留 `\n` 的编辑器写,绝不经 shell 字符串生成。 + +## Lessons + +- 被检查文件的白名单是对既有文件的承诺,对下一个文件什么也没说。宁可让 glob 覆盖整个目录,把例外写明确。 +- "这只是个工具"是错误的方向。一个工具的输出被引用来当证据越多,它就越需要和被它评判的代码同等的检查—— + mutation 工具是极端情形,因为它的输出是一个没人会手工复算的计数。 +- 在工具启动**之前**就失败,是幸运的失败。同一个盲区同样可以掩盖一个错误的结果。 diff --git a/docs/postmortem/0002-wrong-tree-verification.md b/docs/postmortem/0002-wrong-tree-verification.md new file mode 100644 index 00000000..3c33f694 --- /dev/null +++ b/docs/postmortem/0002-wrong-tree-verification.md @@ -0,0 +1,84 @@ +# 0002 - The tree I verified was not the tree I pushed + +[中文](0002-wrong-tree-verification.zh-CN.md) + +**Status:** open + +## Executive summary + +B2 split the round's task row into three tables and dropped `ooo_probe_tasks`. A test file the refactor never +touched still queried the dropped table by name; the adaptation existed in my working tree and never entered +the commit, so the pushed branch was red while every gate I had run was green. The gates were run against the +working tree, and a push publishes `HEAD` - the two differ by exactly the work that was never staged. The +lesson: "the branch is green" is a claim about a commit, and a claim about a directory is a different claim. + +## Summary + +The split shipped as four commits: the refactor, its acceptance tests, the retargeted mutation teeth, and the +record. The new table set is written by `src/core/store/base.ts`, and the suites read it through a projection. +One of those suites had been written one slice earlier, against the pre-split table, and read raw rows from it +by name. + +In my working tree that suite was adapted: raw reads go through the projection, and the retired table is named +`ooo_probe_tasks_pre_split`. In the commit it was not. The commit staged the files I had edited by hand, and +that test file was not one of them - so the version that reached the remote still queried `ooo_probe_tasks`, +which that same commit had dropped. The suite was red on the branch as pushed. + +It was found by accident, while stashing the working tree before an unrelated run: the stashed (committed) +state failed with `no such table: ooo_probe_tasks` at `tests/integration/ooo-run-namespace.test.ts:93`. The +repair is commit `13317c82`, whose message states the gap in its own body. + +## Impact + +The branch under review carried a red suite until the repair landed. `main` was never affected; no published +number depended on it. The failure could not be mistaken for a pass - it fails loudly and immediately. + +Whether the branch's CI reported it during that window is **not reconstructed** here. The observation that +matters is narrower and does not depend on that answer: a reviewer who trusted my summary of "product +1467/0" would have been trusting a number measured on a tree that was never pushed. + +## Timeline + +- The four B2 commits are prepared and the product suite is run - green - against the working tree. +- The commits are staged selectively, by the files I had edited. `tests/integration/ooo-run-namespace.test.ts` + is not among them, although B2 is what invalidated it. +- The branch is pushed and the pull request updated. +- A later run stashes the working tree to test something else, and the committed state fails by name + (`no such table: ooo_probe_tasks`). +- `13317c82` fixes the suite and states the gap in the commit body. + +## Root cause + +Two mechanisms, and only their combination produces this. + +First, the object I verified and the object I published were different: `npm test` runs against a directory, +a push publishes a commit tree. Nothing in the workflow compares the two, and the difference is precisely the +unstaged work. This is the general form - a verification result is only as good as the identity of the tree +it was measured on, and that identity was in my head rather than in the command. + +Second, a rename or split is a cross-file event, but only one of its two halves is visible. The TypeScript +compiler sees the code side and stays silent here; the other half lives inside SQL strings, which no compiler +reads. "Stage only what I edited" is the correct habit for preserving a shared worktree's unrelated changes, +and at the same time it silently drops fixes the change made necessary in files nobody edited. + +## Guardrails added + +None for the class; the instance is repaired by `13317c82`. A mutation target lists that suite, so a _code_ +regression in it is caught - but "the adaptation never entered the commit" is invisible to every check, and +staging discipline is a session behaviour that no check can see. + +Candidates, named and not landed: + +- Verify the committed tree before pushing it (`git stash -u && npm test`, or `git archive HEAD` into a + temporary worktree) whenever the working tree and the commit differ. +- After a table rename or split, search the whole suite for the retired name - not only the files being edited. +- Make the staging step itself state what it left out, so the difference is a reviewed line rather than an + accident. + +## Lessons + +- "Green" without its object is not a statement. A verification is a triple: result, tree, command. +- A rename is a cross-file event whose second half is in strings. The compiler covers the half that is code. +- Selective staging protects other people's work and hides my own debt in the same motion; those two effects + need separate handling rather than one habit doing both. +- The discovery was lucky and the luck is the finding: a stash is not a verification procedure. diff --git a/docs/postmortem/0002-wrong-tree-verification.zh-CN.md b/docs/postmortem/0002-wrong-tree-verification.zh-CN.md new file mode 100644 index 00000000..103144af --- /dev/null +++ b/docs/postmortem/0002-wrong-tree-verification.zh-CN.md @@ -0,0 +1,71 @@ +# 0002 - 我验证的树,不是我推送的树 + +[English](0002-wrong-tree-verification.md) + +**Status:** open + +## Executive summary + +B2 把 round 的任务行拆成三张表并删掉了 `ooo_probe_tasks`。一个重构根本没碰过的测试文件仍按名字查询那张被删的表; +适配确实存在于我的工作树里,但从未进入提交,于是**我推送出去的分支是红的,而我跑过的每一道门都是绿的**—— +门跑的是工作树,推送发布的是 `HEAD`,两者之差恰好就是从未被暂存的那部分工作。 +结论:"这条分支是绿的"是关于**一个 commit** 的断言;关于**一个目录**的断言是另一回事。 + +## Summary + +这次拆分以四个 commit 落地:重构、它的验收测试、被重新指向的 mutation 牙、以及记录。新的表集合由 +`src/core/store/base.ts` 写入,测试套件通过一个投影读取它。其中一套测试是更早一刀写的,写的是拆分前的表, +并按名字直接读原始行。 + +在我的工作树里那套测试是适配过的:原始读走投影,退役的表改名为 `ooo_probe_tasks_pre_split`。在提交里不是。 +那次提交只暂存了我手工编辑过的文件,而那个测试文件不在其中——于是到远端的版本仍在查询 `ooo_probe_tasks`, +而同一个 commit 刚刚把它删掉。按推送出去的样子,这套测试是红的。 + +它是**偶然**发现的:在跑另一件事之前 stash 工作树,被提交的状态以 `no such table: ooo_probe_tasks` 失败, +位置是 `tests/integration/ooo-run-namespace.test.ts:93`。修复是 commit `13317c82`,它把这段缺口写在自己的正文里。 + +## Impact + +在修复落地前,处于评审中的分支带着一套红的测试。`main` 从未受影响;没有任何已发布的数字依赖它。这个失败 +不可能被误当成通过——它响亮且立刻失败。 + +那个窗口内这条分支的 CI 是否报出了它,**此处不做重建**。真正要紧的观察更窄,也不依赖那个答案:一个信任我 +"product 1467/0" 这句总结的评审者,信任的是一个在**从未被推送的树**上测出来的数字。 + +## Timeline + +- 准备 B2 的四个 commit,对工作树跑产品套件——绿。 +- 按"我编辑过的文件"选择性暂存提交。`tests/integration/ooo-run-namespace.test.ts` 不在其中,尽管正是 B2 让它失效。 +- 推送分支并更新 pull request。 +- 之后为跑别的东西 stash 工作树,被提交的状态按名字失败(`no such table: ooo_probe_tasks`)。 +- `13317c82` 修好该套测试,并在提交正文里写明这段缺口。 + +## Root cause + +两个机制,且只有它们的组合才产生这个结果。 + +第一,我验证的对象与我发布的对象不同:`npm test` 跑的是一个目录,推送发布的是一个 commit 树。工作流里没有 +任何东西比较这两者,而两者之差恰好就是未暂存的工作。这是一般形式——**验证结果的成色取决于"它是在哪棵树上量的" +这个身份**,而这个身份此前在我脑子里,不在命令里。 + +第二,改名或拆分是跨文件事件,但只有一半是可见的。TypeScript 编译器看得见代码那一半,在这里它保持沉默; +另一半活在 SQL 字符串里,没有编译器读它。"只暂存我编辑过的文件"是共享工作树里保护他人改动的正确习惯, +同时它也悄悄丢掉了那些"变更使其必要、但没人编辑过"的文件里的修复。 + +## Guardrails added + +类别层面没有;实例由 `13317c82` 修复。某个 mutation 目标把该套测试列入其 suite 列表,所以其中**代码**层面的回归 +会被抓到——但"适配从未进入提交"对每一道检查都是不可见的,而暂存纪律是检查看不到的会话行为。 + +候选(已命名、未落地): + +- 当工作树与提交不一致时,推送前验证**被提交的树**(`git stash -u && npm test`,或 `git archive HEAD` 到临时工作树)。 +- 表改名或拆分之后,在整个套件里搜索退役的名字,而不只在正在编辑的文件里。 +- 让暂存这一步自己说明"它漏掉了什么",使那处差异成为被评审的一行,而不是一次意外。 + +## Lessons + +- "绿"若不说明它的对象,就不是一个断言。一次验证是一个三元组:结果、树、命令。 +- 改名是跨文件事件,它的第二半在字符串里;编译器只覆盖代码那一半。 +- 选择性暂存在同一个动作里既保护了他人的工作,也藏起了我自己的欠债;这两种效果需要分别处理,而不是靠同一个习惯兼顾。 +- 发现靠的是运气,而运气本身就是发现:stash 不是一种验证程序。 diff --git a/docs/postmortem/0003-checks-read-a-live-mutant.md b/docs/postmortem/0003-checks-read-a-live-mutant.md new file mode 100644 index 00000000..42c6872a --- /dev/null +++ b/docs/postmortem/0003-checks-read-a-live-mutant.md @@ -0,0 +1,174 @@ +# 0003 - The checks I read were reading a mutant + +[中文](0003-checks-read-a-live-mutant.zh-CN.md) + +**Status:** resolved + +## Executive summary + +I ran `npm run lint` and `npm run complexity:gate` while a detached `mutation:teeth` run was holding a +live mutant in `evals/ooo-execution/plan-driver.ts` and `src/integration/ooo-execution.ts`. The failures +they reported described the *mutant*, not my code: a `no-constant-binary-expression` at the exact line a +generated `if (false && …)` had been substituted into, an unused-variable warning that was the +parent-check mutant, and two function-complexity numbers measured against mutated source. I noticed only +because the lint line was recognisably one of my own mutants. The hazard was already written down as +"never edit or stage a file a mutation run is rewriting" - a rule about *writes*. A check only *reads*, +which is why the rule did not fire, and the harness left nothing in the tree that a check could see. + +## Summary + +The slice under work adds fusion legality to `src/integration/ooo-execution.ts`, an accounting mode to +`evals/ooo-execution/cost-model.ts` and a session policy to `evals/ooo-execution/plan-driver.ts`, with new +named mutants in `tools/mutation-teeth.ts`. Because the mutation sweep is one of the three expensive +lanes, it was launched detached, and it was still running when I ran the cheap gates in the foreground. + +The same class then escaped a **second** time, in the opposite direction, and that is what turned a +nuisance into a post-mortem. The detached sweep had been launched in a way that did not survive its +shell, so it was killed mid-run - leaving its mutant in `evals/ooo-execution/plan-driver.ts`, where the +bound check had been substituted with `if (false) return undefined;`. Nothing said so: a killed harness +writes no summary and removes no lock at the time. The next sweep inherited that file as its *baseline*, +its own clean run failed (`clean run failed, the harness proves nothing` - the tool was right and I was +not), and one of its mutants could not be located at all, because the anchor it wanted had already been +replaced by the leftover. Two of the driver's tests failed for a day's worth of the opposite reason to +the first escape: not a check reading a mutant, but a **mutant that outlived its harness**, and the +existing rule - `git diff` the target of a died run - was the one step I skipped. + +`mutation:teeth` works by substituting a named wrong version into its target file, running the suites +that are supposed to catch it, and restoring the file byte-identically. While it is between the +substitution and the restore, **the target file on disk is the mutant**. Two of its registered mutants +substitute `if (false && )` into `sharedSessionLegal`, and the lint output named exactly that +construct as a constant-truthiness error at `:225`; a third mutant replaces the parent check's verdict, +which is why a warning appeared for a variable that is used in my version. The complexity gate reported +`sharedSessionLegal` at 18 and `runOneUnit` at 17 - numbers computed from whatever text was on disk at +that moment, which is a different question from the one I asked. + +Re-measured on a quiet tree, the real numbers were 18 and 16, and both did need the helper extraction the +repository's rule requires. That nearness is what makes the class dangerous rather than merely annoying: +the same mechanism that produced a false failure would have produced a false **pass**, and a green lane +read during a mutant window is evidence about a tree that never existed. + +Building the guardrail then exposed two more faces of the class, both inside the guardrail itself. First, the +lock's `live` field was never written on substitution - a multi-hunk edit had failed as a whole and only the +restore half was reapplied - so a running sweep reported `live: false`, and the field lied in the direction that +makes a wedge look impossible. Second, a sweep started from inside a `node --test` process inherited +`NODE_TEST_CONTEXT`, and the nested runner then exited 0 **having run no test at all**: the harness read that as +"the suite passed" and turned every mutant of that target into a false "not caught". Both were found by +exercising the guardrail instead of trusting it, which is the only reason they are in this record rather than in +a later reader's debugging session. + +## Impact + +No product behaviour was affected and no verification result was published: the contaminated readings +were recognised before they were acted on, and both gates were re-run on a quiet tree (lint 0 errors, +complexity `ok`, 20 methods above 15 unchanged from baseline). The cost is time and risk - I ordered +`lint` output as a real failure, started reasoning about a refactor to satisfy mutant complexity numbers, +and only the accidental match between the lint construct and a mutant I had written seconds earlier +stopped it from becoming a change. Had the mutants been less recognisable, the same reading would have +entered this session's record as a fact about my change. + +The second escape cost more than the first: two named mutants ran against a tree that already held a +mutant, so their verdicts described a baseline that never existed, and one mutant could not be located at +all. Its cost was paid twice - once by the checks that misread, once by the tests that failed on a file +nobody had touched. + +The refusal has one deliberate exemption: `agent:verify --dry-run`. A dry run reads the route config and the +change list, not the mutated file, so it cannot report on a mutant - while refusing it made "what would you +run?" unanswerable exactly when a session needs it, and made two of the verifier's own tests fail while a sweep +held the tree. A run that records a verdict is the one that must not. + +What it could **not** do: mutate the commit. Nothing was staged while the sweep ran, and the sweep +restores each target byte-identically before moving on (`17 of 17` and `1 of 1` restorations on the runs +in this session). + +## Timeline + +- The fusion legality, cost-model and driver changes are written, with their named mutants registered. +- `npm run mutation:teeth --targets evals/ooo-execution/plan-driver.ts` is launched detached; its header + says so, and the foreground is free. +- `npm run lint` is run in the foreground, after the sweep has already substituted a mutant into + `plan-driver.ts`. It reports 2 errors and a warning. +- `npm run complexity:gate` is run in the same window. It reports two functions above the limit. +- The lint message is read closely, and its line (`if (false && …)`) is one of my own mutants: the + reading is contaminated, and everything measured in that window is discarded. +- The sweep finishes and reports `10 of 11` caught with one mutant whose anchor no longer matched, and + one that the suite could not distinguish. +- Re-launched and re-checked: the run that came back `10 of 10` had measured a **tree that still held the + killed run's mutant** in the bound check. The suite's two fusion cases fail on the restored file only + until that leftover is removed - `if (session.units.length >= bound) return undefined;` came back, and + the suite passed 15 of 15 again. The bound mutant had been unlocatable for exactly that reason. +- With the tree quiet: the redundant check and its mutant are deleted, the helpers are extracted, and + lint and complexity are re-run green. + +## Root cause + +Two mechanisms, and the second is why the first kept happening. + +First, a mutation sweep is a **write window over the tree**, and every check is a **reader** of that same +tree. The repository's rule named the write side only - "do not edit or stage a file a mutation run is +rewriting" - which is the side the sweep's own process controls. A reader has no way to tell a mutant +from a change, and a check that reports on the wrong text looks exactly like a check that reports on the +right one. + +Second, the fact "a mutant is live" existed **only inside the harness's process**. Nothing in the tree +said so: no lock, no marker, no file a check could read. So the rule had to be carried by the session's +memory across a long, decomposed task - and a rule that depends on remembering, in the middle of a task +whose whole point is many parallel lanes, is a rule that will be dropped. + +Third, and this is what made the class escape twice: a harness that is **killed** leaves the tree in the +state its process was in, and the state was "a mutant is substituted". The rule that should have caught it +was there - `git diff` the target of a run that died - but it was a step in a document, addressed to the +reader's memory, at the moment when the reader has the *least* reason to be careful, since a killed run +looks like nothing happened at all. + +## Guardrails added + +The mechanism is now visible in the tree, and the checks read it: + +- `tools/mutation-teeth.ts` writes `.temp/mutation-lock.json` naming the target file for as long as a + mutant is live (and removes it on restore, on refusal and on exit), and refuses to start while another + sweep's lock exists - two sweeps in one worktree is the same hazard with a different reader. +- `tools/mutation-lock.ts` is the one home for that lock: the writers and the readers of the tree agree on + one file and one meaning instead of each tool inventing its own signal. A lock whose owner is **dead** is + no longer ignored as harmless: it is reported as `a previous sweep died holding `, the target is + named, `git diff` is named, and a new sweep **refuses to start** rather than taking it over silently - the + exact step that was skipped here. +- [`tools/agent-verify.ts`](../../tools/agent-verify.ts) **refuses to verify** while a lock is present + (running or stale), naming the file the sweep is holding, so the lane that is supposed to summarise this + work cannot summarise a mutant. +- [`tools/repo-context.ts`](../../tools/repo-context.ts) (`npm run agent:context`) prints the live lock in + its reconciliation section, because that is the command a session runs first. +- [`skills/repo-development/SKILL.md`](../../skills/repo-development/SKILL.md) states the widened rule + beside the mutation lane: a running sweep makes the tree unreadable for checks, not only unwritable for + edits. +- A case in `tests/tools/agent-verify.test.ts` fails if the refusal stops firing while a lock is present + (and passes again once the lock is gone). +- A fourth reading is pinned by running a real (cheap) sweep and watching it: a substituted mutant must be + reported as `live: true`, because a field written only on the restore path is a field that lies. The lock root + also reads `MUTATION_LOCK_ROOT` in *every* helper rather than in some of them, so a test can point a sweep at + its own directory instead of writing into the tree under test. +- The integration case that watches a sweep waits for it to finish instead of killing it, and then asserts + that the run reported `restoredByteIdentically` and left no hazard: its first version killed the child in a + `finally`, and that killed sweep left its mutant in `src/core/store/clock.ts` - in the very tree the test was + running in. A guardrail that creates the failure it guards against is worse than none, because it arrives with + a green check. +- The harness strips `NODE_TEST_CONTEXT` before running a suite, so a sweep started inside a `node --test` + process runs the suites it claims to run. +- `tests/tools/mutation-lock.test.ts` pins the three readings of a lock: a live owner is a running sweep, a + dead owner is a stale one, and **a stale one refuses a new sweep** by name. + +## Lessons + +- A verification is a triple - result, command, **object** - and "object" includes *the tree is + quiescent*. 0002 was this same lesson with a different mechanism: there the object was a working tree + instead of the commit; here it is a tree mid-substitution instead of the source. +- A guardrail is code, so it carries the same failure modes as the thing it guards: this one lied in a field, ran + no tests when nested, and refused too much. Each was found by exercising it, not by reading it. +- A process that mutates a shared resource must be recoverable by inspection after it dies. "If it died, + check the file" is the right instinct and the wrong mechanism: the check is the lock, and it belongs in + the tree, where the next process finds it without remembering anything. +- The dangerous half of a contaminated reading is the false **pass**, not the false failure. A false + failure gets investigated; a false pass gets recorded. +- A rule about a lane has to name every role that touches the resource. "Do not write while X runs" and + "do not read while X runs" are different rules, and the second one has more ways to be broken. +- If a fact is only in a process's memory, a rule cannot be built on it. The lock file is not + bureaucracy: it is the difference between a rule and a hope. diff --git a/docs/postmortem/0003-checks-read-a-live-mutant.zh-CN.md b/docs/postmortem/0003-checks-read-a-live-mutant.zh-CN.md new file mode 100644 index 00000000..1ee3b933 --- /dev/null +++ b/docs/postmortem/0003-checks-read-a-live-mutant.zh-CN.md @@ -0,0 +1,121 @@ +# 0003 - 我读到的检查,读的是 mutant + +[English](0003-checks-read-a-live-mutant.md) + +**Status:** resolved + +## Executive summary + +在一条 detach 的 `mutation:teeth` 正在把 mutant 留在树里时,我跑了 `npm run lint` 与 +`npm run complexity:gate`。它们报出的失败描述的是 **mutant**,不是我的代码:`no-constant-binary-expression` +正好落在被替换成 `if (false && …)` 的那一行,一个未使用变量告警就是父检查那个 mutant,两个复杂度数字 +是照着被改写的源码量出来的。我之所以察觉,只因为那行 lint 认出是我自己刚写的 mutant。风险其实早已写下 +——"mutant 在跑时不要编辑、不要 `git add` 它的目标文件"——但那是一条关于**写**的规则。检查只是**读**, +所以规则没有触发,而 harness 也没有在树里留下任何检查能看见的痕迹。 + +## Summary + +本次改动为 `src/integration/ooo-execution.ts` 增加融合合法性、为 `evals/ooo-execution/cost-model.ts` 增加记账模式、为 `evals/ooo-execution/plan-driver.ts` 增加会话策略, +并在 `tools/mutation-teeth.ts` 登记新的具名 mutant。mutation 扫描属于三条昂贵车道之一,因此被 detach 启动;当我在前台 +跑那些廉价 gate 时,它仍在运行。 + +`mutation:teeth` 的工作方式是把一个具名的错误版本替换进目标文件、跑本应抓住它的套件、再把文件按字节还原。 +在替换与还原之间,**磁盘上的目标文件就是 mutant**。它的两个已登记 mutant 会把 `if (false && <条件>)` 替换进 +`sharedSessionLegal`,而 lint 输出正是把该结构在 `:225` 报为常量真值错误;第三个 mutant 替换父检查的 +裁决,这解释了为什么会冒出一个在我版本里其实被使用的变量的告警。复杂度 gate 报出 `sharedSessionLegal` 18、`runOneUnit` 17—— +这两个数字是按当时磁盘上的任意文本量出来的,与我要问的问题并不是同一个。 + +在安静的树上重测,真实数字是 18 与 16,二者确实需要仓库规则所要求的辅助函数抽取。这种“接近”正是该失败 +类别危险而非仅仅恼人的原因:同一个机制既能造出假失败,也能造出假**通过**,而在 mutant 窗口里读到的一条 +绿灯,是关于一棵从未存在过的树的证据。 + +同一个失败类别随后**第二次**逃脱,方向相反——这正是让一次烦扰变成事故记录的原因。那条被 detach 的扫描是以 +无法在父 shell 退出后存活的方式启动的,于是在运行途中被杀,把它的 mutant 留在了 `evals/ooo-execution/plan-driver.ts`: +bound 检查被替换成了 `if (false) return undefined;`。没有任何东西说明这一点——被杀掉的 harness 不写总结,当时也不移除锁。下一条扫描把这个文件继承为**基线**, +它自己的 clean run 失败(`clean run failed, the harness proves nothing`——工具说得对,是我不对),并且有一个 mutant 完全无法定位,因为它要的锚点已被遗留的 mutant 替换。 + +建立防护之后,同一个类别又在**防护自身**里露出两张面孔。第一,锁的 `live` 字段在替换时从未被写入——一次多 hunk 的 +编辑整体失败、只补回了 restore 那半——于是正在运行的扫描报 `live: false`,而字段的说谎方向恰好让“卡住”看起来 +不可能发生。第二,在 `node --test` 进程里启动的扫描继承了 `NODE_TEST_CONTEXT`,嵌套的 runner 于是**一个测试都没跑就 +退出 0**:harness 把它读成“套件通过”,并把该目标的每个 mutant 都变成假的“未被抓住”。两者都是靠**使用**防护、 +而不是相信它才发现的。 + +## Impact + +产品行为零影响,也没有发布任何验证结论:污染读数在被采信之前就被识别,两个 gate 都在安静的树上重跑 +(lint 0 error,复杂度 `ok`,20 个超阈值方法与基线相同)。代价是时间与风险——我把 lint 输出当成真实失败, +开始为满足 mutant 的复杂度数字构思重构,只因为那个被 lint 指出的结构与我几秒前写的 mutant 恰好同名, +才没有变成一次改动。如果 mutant 没那么好认,同样的读数会作为"关于我这次改动的事实"进入本次记录。 + +它**不可能**影响提交:扫描运行期间没有暂存任何东西,而扫描在每个目标之间会把文件按字节还原 +(本次会话的运行分别报 `17 of 17` 与 `1 of 1` 还原)。 + +第二次逃脱的代价更大:两个具名 mutant 是在一棵**已经持有 mutant** 的树上运行的,它们的结论描述了一个从未存在过的 +基线,另有一个 mutant 完全无法定位。代价付了两次——一次是误读的检查,一次是那个没人改过的文件上失败的测试。 + +## Timeline + +- 融合合法性 / 成本模型 / 驱动策略三处改动写完,具名 mutant 已登记。 +- detach 启动 `npm run mutation:teeth --targets evals/ooo-execution/plan-driver.ts`,前台空闲。 +- 前台跑 `npm run lint`——此时扫描已把 mutant 替换进 `plan-driver.ts`,报 2 error + 1 warning。 +- 同一窗口内跑 `npm run complexity:gate`,报两个函数超阈值。 +- 细看 lint 行(`if (false && …)`)认出是自己写的 mutant:读数污染,该窗口内所有测量作废。 +- 扫描结束:`10 of 11`,一个 mutant 锚点失配、一个套件无法区分。 +- 树安静后:删掉冗余检查与其 mutant、抽取辅助函数,lint 与复杂度重跑通过。 + +## Root cause + +两条机制,第二条正是第一条反复出现的原因。 + +第一,一次 mutation 扫描是**对树的写窗口**,而每个检查都是同一棵树的**读者**。仓库的规则只写了写的一侧 +——"扫描在改写文件时不要编辑、不要暂存"——那恰是扫描进程自己能控制的一侧。读者无法分辨 mutant 与改动, +而"报错了别的文本"的检查看起来和"报对了"的检查一模一样。 + +第二,"有一个 mutant 活着"这件事**只存在于 harness 进程内**:树里没有锁、没有标记、没有任何检查能读的 +文件。于是规则只能靠会话记忆跨过一整个被拆解的长任务来携带——而在一个"多车道并行"本身就是重点的任务里, +依赖记住的规则,注定会被丢掉。 + +第三,也是这个类别能逃脱两次的原因:一个**被杀掉**的 harness 会把树留在它进程当时的状态,而那个状态是 +"mutant 已替换"。本该抓住它的规则是存在的——对死掉的运行先 `git diff` 它的目标——但那只是文档里的一步, +写给读者的记忆,而且落在读者**最不可能谨慎**的时刻:因为被杀掉的运行看起来什么都没发生。 + +## Guardrails added + +机制现在写在树里,检查也读它: + +- `tools/mutation-teeth.ts` 在 mutant 存活期间写 `.temp/mutation-lock.json`(记录目标文件),在还原、拒绝 + 与退出时清除;若已存在另一条扫描的锁则拒绝启动——同一 worktree 里两条扫描是同一个风险换了读者。 +- `tools/mutation-lock.ts` 是这把锁的唯一归属:树的写者与读者共用一个文件、一个含义,而不是每个工具各自 + 发明信号。 + 所有者**已死**的锁不再被当作无害而忽略:它被报成 `a previous sweep died holding `,点名目标、点名 `git diff`, + 并且新的扫描**拒绝启动**而不是静默接手——正是这里被跳过的那一步。 +- [`tools/agent-verify.ts`](../../tools/agent-verify.ts) 在锁存在时**拒绝验证**,并点名扫描正在持有的文件, + 让本该汇总本次工作的那条车道无法汇总一个 mutant。 +- [`tools/repo-context.ts`](../../tools/repo-context.ts)(`npm run agent:context`)在 Reconciliation 段打印 + 活锁——因为那是会话最先跑的指令。 +- [`skills/repo-development/SKILL.md`](../../skills/repo-development/SKILL.md) 在 mutation 车道旁写下扩宽后的 + 规则:扫描运行时,树对检查是**不可读**的,不止是不可写。 +- `tests/tools/agent-verify.test.ts` 新增用例:锁存在时拒绝必须触发(去掉锁后必须重新通过)。 + +- `tests/tools/mutation-lock.test.ts` 钉住一把锁的三种读法:活着的所有者是正在运行的扫描,死掉的所有者是陈旧的, + 而**陈旧的锁会按名字拒绝新的扫描**。 + +- 再加一种读法被钉住:真跑一次(便宜的)扫描并观察——被替换的 mutant 必须报 `live: true`,因为只在 restore 路径 + 写入的字段就是会说谎的字段。锁根也在**每个** helper 里读 `MUTATION_LOCK_ROOT`,而不是只读一部分,这样测试可以把 + 扫描指向自己的目录,而不写进被测的那棵树。 +- harness 在跑套件前会剥掉 `NODE_TEST_CONTEXT`,因此在 `node --test` 里启动的扫描会真的跑它所声称的套件。 +- 观察扫描的那个集成用例**等它跑完**而不是杀掉它,然后断言该运行报了 `restoredByteIdentically` 且没有留下危害: + 它的第一版在 `finally` 里杀掉了子进程,那次被杀掉的扫描把 mutant 留在了 `src/core/store/clock.ts`——正是测试自己 + 所在的那棵树。制造自己所防范之失败的防护,比没有防护更糟,因为它还带着一个绿色勾。 +## Lessons + +- 防护本身是代码,因此拥有它所防护之物的同一批失效模式:这把锁在一个字段上说谎、被嵌套时一个测试都不跑、并且拒绝得 + 过多。三者都是靠**使用**它才发现的,而不是靠读它。 +- 改写共享资源的进程,必须能在它死掉之后**被检查而恢复**。"它死了就去看那个文件"直觉对、机制错:检查应当就是 + 那把锁,而锁属于树——下一个进程不必记住任何事情就能找到它。 +- 一次验证是三元组——结果、命令、**对象**——而"对象"包含"树是静止的"。0002 是同一条教训换了机制:那里的 + 对象是工作树而不是提交;这里是替换到一半的树而不是源码。 +- 污染读数里**危险的是假通过**,不是假失败。假失败会被调查,假通过会被记录。 +- 关于某条车道的规则必须点名所有接触该资源的角色。"X 运行时不要写"和"X 运行时不要读"是两条规则, + 而后者有更多破坏方式。 +- 如果事实只存在于某个进程的记忆里,规则就不能建立在它之上。锁文件不是官僚程序,它是"规则"与"指望"的区别。 diff --git a/docs/postmortem/0004-flaky-was-a-clock-boundary.md b/docs/postmortem/0004-flaky-was-a-clock-boundary.md new file mode 100644 index 00000000..ba890773 --- /dev/null +++ b/docs/postmortem/0004-flaky-was-a-clock-boundary.md @@ -0,0 +1,103 @@ +# 0004 - "Flaky" was a clock boundary + +[中文](0004-flaky-was-a-clock-boundary.zh-CN.md) + +**Status:** resolved + +## Executive summary + +`npm run test:product` failed once, under load, in `demoteMemory: demotes LTG memory to STG`. A previous +session recorded it in the ledger as *flaky, not fixed* and moved on; the entry name was the whole +diagnosis, and it closed the question for a session. Run under a loop instead of once, it reproduced in +about 1 write per 1500: a memory was written with `valid_from = …38.468Z` and read back while SQLite's +`now` said `…38.467Z`, so the read path's `valid_from <= now` was false, the row was invisible, and the +caller got `memory is not active` for a memory it had just written. The defect is real and lives in +the product's current-value window: two clock readers (JavaScript's `Date` and SQLite's `strftime('now')`) +disagree at millisecond granularity. + +## Summary + +The current-value predicate compares a stored timestamp against SQLite's own clock, four times in +`src/core/store/base.ts` and `src/core/store/retrieval.ts`. Writes stamp from JavaScript, so the two +sources race: whenever the row's stamp lands within ~1-2 ms *after* the reading connection's `now`, a +predicate that means "this value is in force" answers "no" for a row that was written microseconds ago. +Under a loaded machine the skew widens; the loop that found it needed 3000 iterations to see 2-4 hits. + +The fix keeps the comparison but widens the window by a named grace, in one new home, +`src/core/store/clock.ts`: `CLOCK_GRACE_MS = 50`, with `clockNow("later" | "earlier")` feeding the four +predicates. The grace only ever *widens* what counts as current - it never narrows - so the change cannot +hide an expiry, and the two directions are separate so the asymmetry is visible at each site. Measured +after the change: 3000 iterations, 0 failures (was 2-4). + +SQLite has no `milliseconds` date modifier: `'+50 milliseconds'` makes `strftime` return `NULL`, and the +comparison then excludes **every** row silently (`ok: null`), which is why the grace is expressed as +fractional seconds (`'+0.050 seconds'`) and why a mutant now pins that choice. + +## Impact + +A memory written within about a millisecond of being read could be invisible to one read: the write +succeeded, the row was stored, and the *next* read saw it. No durable loss and no corruption - the failure +mode is a single false "not active" answer, which surfaces as an error from `requireActiveMemory` or as a +missing row in maintenance, demotion, dedup and search paths. It reached a product test only because that +test reads a just-written memory in a tight loop; under load a user-visible `not active` error for a +just-saved memory was possible. + +The second, larger impact is the one this record exists for: the defect was **labelled** instead of +diagnosed. "Flaky, not fixed" is a statement about a test, and it was written down as a fact about the +product - and a labelled failure is a failure nobody has to look at again. + +## Timeline + +- `npm run test:product` reports 1 failure of 1447 under load: `demoteMemory: demotes LTG memory to STG`. +- A previous session re-runs the suite alone, sees it pass, and records the row as "flaky, not fixed". +- In this session the user refuses that label: a failure appearing more than once has to be treated as a + case. +- A loop harness (`.temp/flake-hunt.ts`) writes and immediately reads a memory: 2 failures in 3000 + iterations, both `memory … is not active` for a row that exists. +- Dumping the row and SQLite's `now` side by side shows the 1 ms order: stamp `…38.468Z`, `now` `…38.467Z`. +- Fix: the window's grace, in `src/core/store/clock.ts`, wired into the four predicates. Loop: 0 of 3000. +- A deterministic test (`tests/core/store/current-value-window.test.ts`, 6 cases) pins both boundaries, and + 4 named mutants (2 on the boundaries, 1 on the grace, 1 on the SQLite time unit) make the pin checkable: + 4 of 4 caught. +- The ledger row is corrected from "flaky, not fixed" to the fixed defect with its reproduction rate. + +## Root cause + +Two mechanisms again, and the second one is the reason the first survived. + +The technical one: the "is this value in force" predicate compares timestamps written by two different +clock readers. JavaScript stamps `valid_from`; SQLite supplies `now` at read time. Two readers of the same +wall clock do not return the same instant, and the comparison is strict - so a difference of one +millisecond in the wrong direction makes a fresh row look like a future one. Nothing in the code said +which clock had authority, because both were "the clock". + +The process one: an intermittent failure that passes on re-run is easy to file as flakiness, and the +ledger made that filing *look* like a result. "Flaky, not fixed" has no reproduction attempt attached, no +rate, and no hypothesis - it is a label wearing a diagnosis's clothes. The failure reappeared later in the +session, which is the evidence that the label was wrong; without the user's push it would have been +labelled a second time. + +## Guardrails added + +- `src/core/store/clock.ts` is the single home for the current-value window's grace, so a future predicate + has one place to read it from instead of hand-writing another comparison. +- `tests/core/store/current-value-window.test.ts` (6 cases) makes the boundary deterministic: a value + stamped half a grace in the future is current; a minute in the future is not; the same on the expiry + side; 400 write-then-read rounds never fail; and the window widens on both boundaries and never narrows. +- 4 named mutants in `tools/mutation-teeth.ts` (own target for `src/core/store/clock.ts`) make the test's + teeth checkable, including the SQLite time unit that silently excluded every row. +- The ledger's `test:product` row no longer says "flaky": an intermittent failure is recorded with its + reproduction attempt and rate, or it is recorded as open. The rule's home is + [`skills/repo-development/SKILL.md`](../../skills/repo-development/SKILL.md). + +## Lessons + +- An intermittent failure is evidence of a timing dependency until a reproduction says otherwise. "Flaky" + is a claim about a test; it is not a diagnosis, and it must not be recorded as one. +- When two readers of the same quantity disagree, the code has to say which one is authoritative - or, as + here, deliberately widen the comparison so that neither is asked for an impossible precision. +- A wrong answer from a *silently* NULL expression is worse than an exception: `strftime` with an unknown + modifier excluded every row and reported `ok: null`. A mutant is what makes that reachable-in-theory + mistake permanent. +- Labelling is cheap, and that is exactly why it is dangerous: the cost of a wrong label shows up in a + later session, under someone else's deadline. diff --git a/docs/postmortem/0004-flaky-was-a-clock-boundary.zh-CN.md b/docs/postmortem/0004-flaky-was-a-clock-boundary.zh-CN.md new file mode 100644 index 00000000..ec4f30b8 --- /dev/null +++ b/docs/postmortem/0004-flaky-was-a-clock-boundary.zh-CN.md @@ -0,0 +1,85 @@ +# 0004 - "flaky" 其实是一个时钟边界 + +[English](0004-flaky-was-a-clock-boundary.md) + +**Status:** resolved + +## Executive summary + +`npm run test:product` 在负载下失败过一次,用例是 `demoteMemory: demotes LTG memory to STG`。上一个会话 +在 ledger 里把它记成 *flaky, not fixed* 就走开了:那个标签本身就是全部诊断,并且让这个问题沉寂了一整个 +会话。改用循环而不是跑一次之后,它在约 1/1500 次写入中复现:一条记忆以 `valid_from = …38.468Z` 写入, +读回时 SQLite 的 `now` 是 `…38.467Z`,于是读路径的 `valid_from <= now` 为假,行不可见,调用方为一条**刚写 +入**的记忆收到 `memory is not active`。这是真实缺陷,位于产品的 current-value 窗口:两个时钟读者 +(JavaScript 的 `Date` 与 SQLite 的 `strftime('now')`)在毫秒这一级不一致。 + +## Summary + +current-value 谓词把存储时间戳与 SQLite 自己的时钟比较,在 `src/core/store/base.ts` 与 +`src/core/store/retrieval.ts` 中共四处。写入由 JavaScript 打时间戳,于是两个来源会竞争:只要行的时间戳落在 +读数连接 `now` 之后 ~1–2 ms 内,一个含义为"该值正在生效"的谓词就会对一条刚刚写入的行回答"否"。机器有 +负载时偏差更大;发现它的循环需要 3000 次迭代才看到 2–4 次命中。 + +修复保留比较,但把窗口按一个有名字的宽限放宽,唯一的归属是新的 `src/core/store/clock.ts`: +`CLOCK_GRACE_MS = 50`,由 `clockNow("later" | "earlier")` 供给四个谓词。宽限只**放宽**"算作生效"的范围, +从不收紧,因此不可能掩盖过期;两个方向分开表达,使这种不对称在每个站点都可见。改后实测:3000 次迭代, +0 失败(此前 2–4)。 + +SQLite 没有 `milliseconds` 日期修饰符:`'+50 milliseconds'` 会让 `strftime` 返回 `NULL`,比较随后**静默** +排除**每一行**(`ok: null`)——正因如此宽限写成秒的小数(`'+0.050 seconds'`),也正因如此现在有一个 mutant +钉住这个选择。 + +## Impact + +一条在被读之前约一毫秒内写入的记忆,可能在**一次**读中不可见:写入成功、行已存储,而**下一次**读能看见。 +没有持久丢失、没有损坏——失效模式是一次假的"not active"回答,它表现为 `requireActiveMemory` 抛错,或在维护、 +降级、去重与搜索路径中少一行。它能进到产品测试,只因为那个测试在紧循环里读一条刚写的记忆;在有负载时, +用户对一条刚保存的记忆看到 `not active` 报错是可能的。 + +第二个、也是本记录存在理由的影响是:这个缺陷被**贴了标签**而不是被诊断。"flaky, not fixed"是关于测试的 +陈述,却被当作关于产品的事实写下——而被贴标签的失败,是没人需要再看一眼的失败。 + +## Timeline + +- 负载下 `npm run test:product` 报 1447 中 1 失败:`demoteMemory: demotes LTG memory to STG`。 +- 上一个会话单独重跑该套件、看到通过,就在 ledger 里记成 "flaky, not fixed"。 +- 本次会话用户拒绝这个标签:出现不止一次的失败必须当成一个案子处理。 +- 循环脚本(`.temp/flake-hunt.ts`)写一条记忆后立刻读:3000 次中 2 次失败,都是对一条存在的行报 + `memory … is not active`。 +- 把行与 SQLite 的 `now` 并列打印,看到那 1 ms 的先后:时间戳 `…38.468Z`,`now` `…38.467Z`。 +- 修复:窗口宽限,落在 `src/core/store/clock.ts`,接入四个谓词。循环:3000 次 0 失败。 +- 一个确定性测试(`tests/core/store/current-value-window.test.ts`,6 个用例)钉住两个边界;4 个具名 mutant + (2 个钉边界、1 个钉宽限、1 个钉 SQLite 时间单位)让这个钉子可被检查:4/4 被抓住。 +- ledger 中那一行从 "flaky, not fixed" 更正为已修复的缺陷及其复现率。 + +## Root cause + +又是两条机制,第二条正是第一条能存活下来的原因。 + +技术上的:那个"该值是否生效"的谓词比较由两个不同时钟读者写下的时间戳。JavaScript 写 `valid_from`, +SQLite 在读时提供 `now`。同一个墙上时钟的两个读者不会返回同一瞬间,而比较是严格的——于是往错误方向差 +一毫秒,就足以让一条新行看起来属于未来。代码里没有任何地方说明哪个时钟有权威性,因为两者都被当作"那个时钟"。 + +流程上的:一个重跑就通过的间歇失败,很容易被归档为 flaky,而 ledger 让这种归档**看起来**像一个结论。 +"flaky, not fixed"没有复现尝试、没有比率、没有假设——它是穿着诊断外衣的标签。这个失败在会话稍后再次出现, +那就是标签错了的证据;若没有用户的坚持,它会被贴第二次标签。 + +## Guardrails added + +- `src/core/store/clock.ts` 是 current-value 窗口宽限的唯一归属,未来的谓词有一处可读,而不必再手写一个比较。 +- `tests/core/store/current-value-window.test.ts`(6 个用例)让边界确定化:时间戳在未来半个宽限内的算生效; + 未来一分钟的算不生效;过期侧同理;400 轮"写后立刻读"永不失败;窗口在两个边界都放宽、从不收紧。 +- `tools/mutation-teeth.ts` 中 4 个具名 mutant(`src/core/store/clock.ts` 拥有独立 target)让这个测试的"牙齿" + 可被检查,其中包括那个会静默排除每一行的 SQLite 时间单位。 +- ledger 的 `test:product` 行不再写 "flaky":间歇失败要么连同复现尝试与比率一起记录,要么记为 open。 + 这条规则的归属是 [`skills/repo-development/SKILL.md`](../../skills/repo-development/SKILL.md)。 + +## Lessons + +- 间歇失败在复现给出结论之前,都是"存在时序依赖"的证据。"flaky"是关于测试的说法,不是诊断,更不能当作 + 诊断记录。 +- 当同一个量的两个读者不一致时,代码必须说明谁有权威性;或者像这里一样,刻意放宽比较,使任何一方都不必被 + 要求不可能达到的精度。 +- 来自**静默** NULL 表达式的错误答案比异常更糟:`strftime` 遇到未知修饰符会排除每一行并报 `ok: null`。 + 一个 mutant 才能让这种"理论上可能"的失误变成永久可见。 +- 贴标签很便宜,这正是它危险的原因:错误标签的代价会在之后的会话里、在别人的截止日期下出现。 diff --git a/docs/postmortem/README.md b/docs/postmortem/README.md index a01670fb..b171abc6 100644 --- a/docs/postmortem/README.md +++ b/docs/postmortem/README.md @@ -137,6 +137,10 @@ nothing catches yet, which is the point of writing it down. | Class | Canonical case | Rule it produced | | ----------------------- | -------------- | ---------------- | +| unchecked tooling | 0001 | | +| wrong-tree verification | 0002 | | +| mutant-in-tree reading | 0003 | the sweep's lock: `agent:verify` refuses to report, and a new sweep refuses to start, while one holds the tree (`tools/mutation-lock.ts`) | +| mislabelled flake | 0004 | an intermittent failure is recorded with its reproduction attempt and rate, or left open - never as "flaky" (`skills/repo-development/SKILL.md`) | | ----- | -------------- | ---------------- | A class name is a short noun phrase, not a sentence, and it is not invented before @@ -146,5 +150,9 @@ one place to read the whole taxonomy. ## Index -| # | Record | Failure class | -| ---- | ------------------------------------------------------------------------- | ----------------------- | +| # | Record | Failure class | +| ---- | --------------------------------------------------------------------------------------------- | ----------------------- | +| 0001 | [The tool that checks the others is not itself checked](0001-tools-outside-the-type-check.md) | unchecked tooling | +| 0002 | [The tree I verified was not the tree I pushed](0002-wrong-tree-verification.md) | wrong-tree verification | +| 0003 | [The checks I read were reading a mutant](0003-checks-read-a-live-mutant.md) | mutant-in-tree reading | +| 0004 | ["Flaky" was a clock boundary](0004-flaky-was-a-clock-boundary.md) | mislabelled flake | diff --git a/docs/postmortem/README.zh-CN.md b/docs/postmortem/README.zh-CN.md index 28dbfdcd..f036f4cb 100644 --- a/docs/postmortem/README.zh-CN.md +++ b/docs/postmortem/README.zh-CN.md @@ -88,14 +88,22 @@ 索引把每篇记录的类别写成自由文本。本表是分类体系:一类失败有一个典型案例——读者应先读的那篇记录——和它产生的规则时,就占一行。规则列空着,就意味着这一类目前没有任何东西拦住它,而把它写下来正是意义所在。 -| 类别 | 典型案例 | 它产生的规则 | -| ----------------------- | -------------------------------------------- | ------------ | -| ---- | -------- | ------------ | +| 类别 | 典型案例 | 它产生的规则 | +| ----------------------- | -------- | ------------ | +| unchecked tooling | 0001 | | +| wrong-tree verification | 0002 | | +| mutant-in-tree reading | 0003 | the sweep's lock: `agent:verify` refuses, and a new sweep refuses to start, while a sweep holds the tree (`tools/mutation-lock.ts`) | +| mislabelled flake | 0004 | an intermittent failure is recorded with its reproduction attempt and rate, or left open - never as "flaky" (`skills/repo-development/SKILL.md`) | +| ----- | -------- | ------------ | 类别名是简短名词短语,不是句子;在没有案例可指之前不凭空发明:第一篇记录命名它,第二篇确认它。记录的类别只写在索引里,不写在记录里,所以整个分类体系只有一处可读。 ## 索引 -| # | 记录 | Failure class | -| ---- | ------------------------------------------------------------------------- | ----------------------- | -| --- | ---- | ------------- | +| # | 记录 | Failure class | +| ---- | -------------------------------------------------------------------------- | ----------------------- | +| 0001 | [检查别人的工具,自己没被检查](0001-tools-outside-the-type-check.zh-CN.md) | unchecked tooling | +| 0002 | [我验证的树,不是我推送的树](0002-wrong-tree-verification.zh-CN.md) | wrong-tree verification | +| 0003 | [我读到的检查,读的是 mutant](0003-checks-read-a-live-mutant.zh-CN.md) | mutant-in-tree reading | +| 0004 | ["flaky" 其实是一个时钟边界](0004-flaky-was-a-clock-boundary.zh-CN.md) | mislabelled flake | +| --- | ---- | ------------- | diff --git a/evals/ooo-execution/board-deliver.ts b/evals/ooo-execution/board-deliver.ts index 03b05f6b..1180ba61 100644 --- a/evals/ooo-execution/board-deliver.ts +++ b/evals/ooo-execution/board-deliver.ts @@ -1,20 +1,22 @@ /** * Evidence driver: deliver an artifact to an entry through the board protocol, from the * process that holds the claim. Refuses if the claim is not this agent's, and - * asserts that the store recorded exactly the digest this process computed. + * asserts that the daemon recorded exactly the digest this process computed. + * + * It reaches the board through the daemon that serves the round's store (`--daemon `), + * as a client: the drivers do not open a database of their own. * * Usage: * node --experimental-strip-types evals/ooo-execution/board-deliver.ts \ - * --channel --entry --agent --digest \ - * [--ref ] [--summary ] [--store ] + * --daemon --channel --entry --agent \ + * --digest [--ref ] [--summary ] */ import { createHash } from "node:crypto"; import { readFileSync, statSync } from "node:fs"; -import { join, resolve } from "node:path"; -import { homedir } from "node:os"; +import { resolve } from "node:path"; import { parseArgs } from "node:util"; -const { NmgStoreBase } = await import("../../src/core/store/base.ts"); +import { boardCall, roundDaemon } from "./round-client.ts"; // node:util owns flag parsing; an unknown flag or a repeated one is an error rather // than something this script silently ignores. @@ -22,7 +24,7 @@ const { values } = parseArgs({ options: { channel: { type: "string" }, entry: { type: "string" }, - store: { type: "string" }, + daemon: { type: "string" }, agent: { type: "string" }, ref: { type: "string" }, digest: { type: "string" }, @@ -37,15 +39,15 @@ const ref = values.ref; for (const [name, value] of Object.entries({ channel, entry: entryId, agent: agentId })) { if (!value) throw new Error(`--${name} is required`); } +if (!values.daemon) { + throw new Error("--daemon is required: the store path whose daemon serves this round's board"); +} -const storePath = resolve( - values.store ?? join(process.env.NMG_DATA_DIR ?? join(homedir(), ".nmg"), "nmg.sqlite"), -); -const store = new NmgStoreBase(storePath); -try { - const entry = store - .readTaskBoard({ taskId: channel!, limit: 200 }) - .entries.find((candidate) => candidate.id === entryId); +const state = roundDaemon(resolve(values.daemon)); +{ + const read = await boardCall(state, { action: "read", taskId: channel!, agentId, limit: 200 }); + if (read.action !== "read") throw new Error("the board did not answer a read with entries"); + const entry = read.entries.find((candidate) => candidate.id === entryId); if (!entry) throw new Error(`no entry ${entryId} in ${channel}`); if (entry.status !== "open") throw new Error(`entry ${entryId} is ${entry.status}`); if (entry.claimedBy !== agentId) { @@ -64,7 +66,8 @@ try { throw new Error(`--digest does not match the bytes at ${ref} (${digest})`); } - const delivered = store.deliverTaskBoardEntry({ + const result = await boardCall(state, { + action: "deliver", taskId: channel!, entryId: entryId!, agentId: agentId!, @@ -72,8 +75,10 @@ try { ref, summary: values.summary, }); + if (result.action !== "deliver") throw new Error("the board did not answer a delivery"); + const delivered = result.entry; if (delivered.deliverableDigest !== digest) { - throw new Error(`store recorded ${delivered.deliverableDigest}, computed ${digest}`); + throw new Error(`the daemon recorded ${delivered.deliverableDigest}, computed ${digest}`); } console.log( JSON.stringify({ @@ -87,6 +92,4 @@ try { delivererPid: process.pid, }), ); -} finally { - store.close(); } diff --git a/evals/ooo-execution/board-judge.ts b/evals/ooo-execution/board-judge.ts index 4b9eea5d..c24b032c 100644 --- a/evals/ooo-execution/board-judge.ts +++ b/evals/ooo-execution/board-judge.ts @@ -3,18 +3,20 @@ * evidence rather than on the claim. It recomputes the artifact digest from the bytes at * `ref` and refuses to accept when it does not match what was delivered. * + * It reaches the board through the daemon that serves the round's store (`--daemon `), + * as a client: the drivers do not open a database of their own. + * * Usage: * node --experimental-strip-types evals/ooo-execution/board-judge.ts \ - * --channel ooo-process-probe --entry --agent coordinator \ - * --verdict accepted|rejected|undecidable --reason "..." [--store ] + * --daemon --channel ooo-probe: --entry --agent coordinator \ + * --verdict accepted|rejected|undecidable --reason "..." */ import { createHash } from "node:crypto"; import { readFileSync, statSync } from "node:fs"; -import { join, resolve } from "node:path"; -import { homedir } from "node:os"; +import { resolve } from "node:path"; import { parseArgs } from "node:util"; -const { NmgStoreBase } = await import("../../src/core/store/base.ts"); +import { boardCall, roundDaemon } from "./round-client.ts"; // node:util owns flag parsing; an unknown flag or a repeated one is an error rather // than something this script silently ignores. @@ -22,7 +24,7 @@ const { values } = parseArgs({ options: { channel: { type: "string" }, entry: { type: "string" }, - store: { type: "string" }, + daemon: { type: "string" }, agent: { type: "string" }, verdict: { type: "string" }, reason: { type: "string" }, @@ -46,15 +48,15 @@ for (const [name, value] of Object.entries({ if (!["accepted", "rejected", "undecidable"].includes(verdict!)) { throw new Error(`--verdict must be accepted | rejected | undecidable, got ${verdict}`); } +if (!values.daemon) { + throw new Error("--daemon is required: the store path whose daemon serves this round's board"); +} -const storePath = resolve( - values.store ?? join(process.env.NMG_DATA_DIR ?? join(homedir(), ".nmg"), "nmg.sqlite"), -); -const store = new NmgStoreBase(storePath); -try { - const entry = store - .readTaskBoard({ taskId: channel!, limit: 200 }) - .entries.find((candidate) => candidate.id === entryId); +const state = roundDaemon(resolve(values.daemon)); +{ + const read = await boardCall(state, { action: "read", taskId: channel!, agentId, limit: 200 }); + if (read.action !== "read") throw new Error("the board did not answer a read with entries"); + const entry = read.entries.find((candidate) => candidate.id === entryId); if (!entry) throw new Error(`no entry ${entryId} in ${channel}`); if (!entry.deliverableDigest) throw new Error(`entry ${entryId} carries no deliverable`); if (entry.deliveredBy === agentId) { @@ -75,13 +77,16 @@ try { ); } - const judged = store.judgeTaskBoardEntry({ + const result = await boardCall(state, { + action: "judge", taskId: channel!, entryId: entryId!, agentId: agentId!, verdict: verdict!, reason: reason!, }); + if (result.action !== "judge") throw new Error("the board did not answer a judgement"); + const judged = result.entry; console.log( JSON.stringify({ entryId: judged.id, @@ -91,6 +96,4 @@ try { reason: judged.verdictReason, }), ); -} finally { - store.close(); } diff --git a/evals/ooo-execution/board-slots.test.ts b/evals/ooo-execution/board-slots.test.ts new file mode 100644 index 00000000..1df5db85 --- /dev/null +++ b/evals/ooo-execution/board-slots.test.ts @@ -0,0 +1,172 @@ +import assert from "node:assert/strict"; +import test, { type TestContext } from "node:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + BoardAdmission, + type PatchTaskSpec, + type ProbePlan, +} from "../../src/integration/ooo-board.ts"; +import { expectedRenameOf, RENAME_TARGET, renameSource } from "./rename-probe.ts"; + +const TARGET = RENAME_TARGET; +const source = renameSource(); +const expected = expectedRenameOf(source); + +/** Two independent patch tasks plus one that depends on the first, so "more than one claim at once" + * is real work and a dependency is still a dependency. */ +const plan: ProbePlan = [ + ["P", "", [], "isolated-artifact", null, null], + ["Q", "", [], "isolated-artifact", null, null], + ["R", "", ["P"], "isolated-artifact", null, null], +]; + +async function verifyRename() { + return "accept" as const; +} + +const spec = (id: string): PatchTaskSpec => ({ + instruction: `rename for ${id}`, + files: { [TARGET]: source }, + editable: [TARGET], + verify: verifyRename, +}); + +function fixture( + t: TestContext, + options: ConstructorParameters[3], +): BoardAdmission { + const dir = mkdtempSync(join(tmpdir(), "ooo-slots-")); + const gate = new BoardAdmission( + join(dir, "store.sqlite"), + plan, + { P: spec("P"), Q: spec("Q"), R: spec("R") }, + options, + ); + t.after(() => { + gate.close(); + rmSync(dir, { recursive: true, force: true }); + }); + return gate; +} + +/** The host submission path: the result entry carries the ticket and the artifact, and the store + * decides. Accepting a claimed task is what frees its dependents. */ +async function acceptUnit( + gate: BoardAdmission, + ticket: { owner: string; patch?: { digest: string } }, +) { + const entry = gate.putTaskBoardEntry({ + taskId: gate.channel, + agentId: ticket.owner, + kind: "result", + content: JSON.stringify({ + ticket, + artifact: JSON.stringify({ + digest: ticket.patch!.digest, + files: [{ path: TARGET, content: expected }], + }), + }), + expiresAt: new Date(gate.now + 60_000).toISOString(), + }); + return gate.submit(entry.id); +} + +const handoffs = (gate: BoardAdmission) => + gate + .readTaskBoard({ taskId: gate.channel }) + .entries.filter((entry) => entry.kind === "handoff") + .sort((left, right) => String(left.to).localeCompare(String(right.to))); + +test("a declared budget holds two claims at once, and the store is why each handoff is directed", (t) => { + const dir = mkdtempSync(join(tmpdir(), "ooo-slots-unrefused-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + // A budget above one without a target is refused, and by name: the store keeps one outstanding + // un-directed actionable per channel and queues the next as `pending`, so the second slot's + // handoff could be published but its claim would be refused - a fallback, not a second slot. + assert.throws( + () => + new BoardAdmission( + join(dir, "store.sqlite"), + plan, + { P: spec("P"), Q: spec("Q"), R: spec("R") }, + { slots: 2 }, + ), + /must name each handoff's target/, + ); + + const gate = fixture(t, { slots: 2, handoffTarget: (taskId) => `worker-${taskId}` }); + assert.deepEqual(gate.candidates(), ["P", "Q"], "R waits on P, at any budget"); + assert.deepEqual(gate.startable(), ["P", "Q"], "two slots may start both legal tasks"); + // The publication is the load-bearing part: both handoffs exist before either is claimed, and + // neither is queued behind the other, because a directed entry is not serialised. + const published = handoffs(gate); + assert.equal(published.length, 2, "one handoff per startable task, published up front"); + assert.deepEqual( + published.map((entry) => entry.to), + ["worker-P", "worker-Q"], + ); + assert.deepEqual( + published.map((entry) => entry.serialState), + [null, null], + "a directed entry is not serialised, which is what lets two claims coexist", + ); + // A republish must not withdraw the offer it just made, and it must not churn it either: a reader + // that saw the offer by id would otherwise chase an entry that was resolved and replaced. With one + // slot the non-head handoff is retired as unselected, which is right there and wrong here. + const idsBefore = published.map((entry) => entry.id); + gate.refresh(); + assert.deepEqual( + handoffs(gate).map((entry) => entry.id), + idsBefore, + "a republish keeps every startable handoff, by identity", + ); + assert.throws(() => gate.claim("R", "worker-R"), /unfulfilled dependencies/); + // The non-head is claimed first, deliberately: with two slots both legal tasks are startable, and + // a licence that named only the head would refuse this one. + assert.equal(gate.claim("Q", "worker-Q").owner, "worker-Q", "the second legal task is startable"); + assert.equal(gate.claim("P", "worker-P").owner, "worker-P", "the second claim is held at once"); + // Both slots are spent: the rule's own answer is an empty set, for both readings. + assert.deepEqual(gate.candidates(), []); + assert.deepEqual(gate.startable(), []); + assert.throws(() => gate.claim("Q", "worker-Q"), /task already claimed/); +}); + +test("at the default budget the licence is still the head of the ordered set", (t) => { + const gate = fixture(t, {}); + assert.deepEqual( + gate.candidates(), + ["P", "Q"], + "the legal set is what a caller may report or rank, whatever the budget", + ); + assert.deepEqual(gate.startable(), ["P"], "the licence is the budget's part of that set"); + const published = handoffs(gate); + assert.equal(published.length, 1, "one slot publishes one handoff"); + assert.equal(published[0]!.to, null, "the default budget publishes the broadcast handoff"); + assert.equal( + published[0]!.serialState, + "outstanding", + "an un-directed actionable takes the board's serial slot, which is the D14 boundary", + ); + // Q has no published handoff, because a run with one slot never offers it one - the refusal + // names that, and it is the same refusal the layer gave before a budget existed. + assert.throws(() => gate.claim("Q", "worker-Q"), /no published handoff for this task/); + const ticket = gate.claim("P", "worker-P"); + assert.equal(ticket.owner, "worker-P"); + assert.deepEqual(gate.candidates(), [], "a spent budget selects nothing, as it always did"); +}); + +test("accepting a claimed unit frees its dependent while the other slot is still held", async (t) => { + const gate = fixture(t, { slots: 2, handoffTarget: (taskId) => `worker-${taskId}` }); + const first = gate.claim("P", "worker-P"); + gate.claim("Q", "worker-Q"); + assert.deepEqual(gate.candidates(), [], "both slots are spent, and R is not free yet"); + assert.equal(await acceptUnit(gate, first), "accepted"); + assert.deepEqual( + gate.candidates(), + ["R"], + "P's acceptance frees R, and Q's claim still holds the other slot", + ); + assert.deepEqual(gate.startable(), ["R"], "one slot left is one startable task"); +}); diff --git a/evals/ooo-execution/board-worker.ts b/evals/ooo-execution/board-worker.ts index 5f89a56d..93873d8f 100644 --- a/evals/ooo-execution/board-worker.ts +++ b/evals/ooo-execution/board-worker.ts @@ -4,23 +4,27 @@ * * It deliberately does not judge its own delivery: judging is a different agent's act. * + * It reaches the board through the daemon that serves the round's store (`--daemon `), + * as a client: the drivers do not open a database of their own, because a second writer is what the + * run's coordinated transition exists to prevent. + * * Usage: * node --experimental-strip-types evals/ooo-execution/board-worker.ts \ - * --channel ooo-process-probe --out .nmg/board/worker-output.txt [--entry ] - * [--store ] [--lease 1800] [--agent worker-ooo-] [--suites a.test.ts,b.test.ts] + * --daemon --channel ooo-probe: \ + * --out .nmg/board/worker-output.txt [--entry ] + * [--lease 1800] [--agent worker-ooo-] [--suites a.test.ts,b.test.ts] * - * Refuses to run without a channel and an output path. Asserts that it holds the claim, - * that the artifact file exists and is non-empty, that the digest recomputes, and that - * the store recorded exactly the digest it reported. + * Refuses to run without a channel, an output path and a daemon that serves the store. Asserts that + * it holds the claim, that the artifact file exists and is non-empty, that the digest recomputes, + * and that the daemon recorded exactly the digest it reported. */ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +import { dirname, resolve } from "node:path"; import { spawnSync } from "node:child_process"; -import { homedir } from "node:os"; import { parseArgs } from "node:util"; -const { NmgStoreBase } = await import("../../src/core/store/base.ts"); +import { boardCall, roundDaemon } from "./round-client.ts"; const DEFAULT_SUITES = [ "tests/core/task-board-deliverable.test.ts", @@ -36,7 +40,7 @@ const { values } = parseArgs({ channel: { type: "string" }, entry: { type: "string" }, out: { type: "string" }, - store: { type: "string" }, + daemon: { type: "string" }, agent: { type: "string" }, lease: { type: "string" }, suites: { type: "string" }, @@ -47,36 +51,42 @@ const channel = values.channel; const out = values.out; if (!channel) throw new Error("--channel is required: a taskId is the board's only boundary"); if (!out) throw new Error("--out is required: an artifact the deliverer can point at"); +if (!values.daemon) { + throw new Error("--daemon is required: the store path whose daemon serves this round's board"); +} -const storePath = resolve( - values.store ?? join(process.env.NMG_DATA_DIR ?? join(homedir(), ".nmg"), "nmg.sqlite"), -); +const state = roundDaemon(resolve(values.daemon)); const agentId = values.agent ?? `worker-ooo-${process.pid}`; const leaseSeconds = Number(values.lease ?? 1800); const suites = (values.suites ?? DEFAULT_SUITES.join(",")).split(",").filter(Boolean); -const store = new NmgStoreBase(storePath); -try { +{ // 1. Find the ready handoff: open, actionable, and not already held by someone alive. - const open = () => store.readTaskBoard({ taskId: channel!, limit: 200 }).entries; + const open = async () => { + const read = await boardCall(state, { action: "read", taskId: channel, agentId, limit: 200 }); + if (read.action !== "read") throw new Error("the board did not answer a read with entries"); + return read.entries; + }; + const entries = await open(); const candidate = values.entry - ? open().find((entry) => entry.id === values.entry) - : open() - .filter((entry) => entry.kind === "handoff" && entry.status === "open") - .at(-1); - if (!candidate) throw new Error(`no open handoff in ${channel} (store: ${storePath})`); + ? entries.find((entry) => entry.id === values.entry) + : entries.filter((entry) => entry.kind === "handoff" && entry.status === "open").at(-1); + if (!candidate) throw new Error(`no open handoff in ${channel} (daemon: ${state.port})`); if (candidate.status !== "open") { throw new Error(`entry ${candidate.id} is ${candidate.status}; nothing to claim`); } // 2. Claim it. The store refuses a pending serial entry, a finalised one, and a claim - // held by a live peer — so a throw here means the work is genuinely mine to do. - const claimed = store.claimTaskBoardEntry({ + // held by a live peer - so a throw here means the work is genuinely mine to do. + const claimResult = await boardCall(state, { + action: "claim", taskId: channel, entryId: candidate.id, agentId, leaseSeconds, }); + if (claimResult.action !== "claim") throw new Error("the board did not answer a claim"); + const claimed = claimResult.entry; if (claimed.claimedBy !== agentId) { throw new Error(`claim did not land: holder is ${claimed.claimedBy ?? "none"}`); } @@ -118,7 +128,12 @@ try { const fail = count("fail"); if (![tests, pass, fail].every(Number.isFinite) || tests === 0) { try { - store.releaseTaskBoardEntry({ taskId: channel, entryId: candidate.id, agentId }); + await boardCall(state, { + action: "release", + taskId: channel, + entryId: candidate.id, + agentId, + }); } catch { // Releasing is courtesy; the lease expiring is the guarantee. } @@ -135,7 +150,8 @@ try { (failures.length ? ` failures=[${failures.join("; ")}]` : ""); // 5. Deliver. Only the live claim holder can do this; the store re-checks under CAS. - const delivered = store.deliverTaskBoardEntry({ + const deliverResult = await boardCall(state, { + action: "deliver", taskId: channel, entryId: candidate.id, agentId, @@ -143,8 +159,10 @@ try { ref: out, summary, }); + if (deliverResult.action !== "deliver") throw new Error("the board did not answer a delivery"); + const delivered = deliverResult.entry; if (delivered.deliverableDigest !== digest) { - throw new Error(`store recorded ${delivered.deliverableDigest}, I computed ${digest}`); + throw new Error(`the daemon recorded ${delivered.deliverableDigest}, I computed ${digest}`); } console.log( JSON.stringify({ @@ -158,6 +176,4 @@ try { verdict: delivered.verdict ?? null, }), ); -} finally { - store.close(); } diff --git a/evals/ooo-execution/cancellation.test.ts b/evals/ooo-execution/cancellation.test.ts deleted file mode 100644 index cafb3224..00000000 --- a/evals/ooo-execution/cancellation.test.ts +++ /dev/null @@ -1,196 +0,0 @@ -// S2 exit: cancelling a running round leaves no orphan check process and no completion after -// the decision. This runs real child processes and real git worktrees. -// -// Platform note, measured rather than assumed: on Windows a plain `child.kill()` already -// reaps the check's own children, because libuv puts non-detached children in a job object -// (verified: this test passes with the tree kill replaced by `child.kill("SIGKILL")`). The -// explicit `taskkill /T` in `candidate.ts` is therefore belt-and-braces there, while the -// POSIX branch — a detached process group plus `process.kill(-pid)` — is what makes it hold -// where no such job object exists. What this test pins is the property, not the mechanism: -// after cancellation the check is gone, its worktree is gone, and nothing is accepted. -import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import test from "node:test"; -import type { ServerState } from "../../src/cli/lifecycle.ts"; -import type { TaskBoardEntry } from "../../src/core/types.ts"; -import type { BoardTicket } from "../../src/integration/ooo-board.ts"; -import { Actor } from "./process-driver.ts"; -import { verifyCandidate } from "../../src/integration/ooo-candidate.ts"; -import { runCycle } from "../../src/integration/ooo-cycle.ts"; - -/** A check that reports its own pid and a grandchild's, then outlives the round. */ -function longCheck(marker: string): string { - return ` -const { spawn } = require("node:child_process"); -const { writeFileSync } = require("node:fs"); -const grandchild = spawn(process.execPath, ["-e", "setTimeout(() => {}, 120000)"], { - stdio: "ignore", -}); -writeFileSync(${JSON.stringify(marker)}, JSON.stringify({ check: process.pid, grandchild: grandchild.pid })); -setTimeout(() => {}, 120000); -`; -} - -const alive = (pid: number) => { - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } -}; - -async function waitFor(condition: () => boolean, timeoutMs: number): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - if (condition()) return true; - await new Promise((resolve) => setTimeout(resolve, 25)); - } - return condition(); -} - -test( - "cancelling a round kills its running check and leaves no accepted work", - { timeout: 120_000 }, - async (t) => { - const root = mkdtempSync(join(tmpdir(), "nmg-ooo-cancel-")); - // The candidate worktree is created under the OS temp dir, and other test files create - // their own concurrently. Pointing temp at a directory this test owns makes the leak check - // about *this* round instead of about whatever else happens to be running. - const directory = join(root, "work"); - mkdirSync(directory, { recursive: true }); - for (const key of ["TMPDIR", "TMP", "TEMP"]) process.env[key] = directory; - const marker = join(root, "check.json"); - const controller = new AbortController(); - t.after(() => { - for (const key of ["TMPDIR", "TMP", "TEMP"]) delete process.env[key]; - rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); - }); - - const round = runCycle({ - repository: process.cwd(), - revision: "HEAD", - baseline: { - "src/probe.ts": "export const value = 1;\n", - "src/probe.test.ts": "test('probe identity', () => {});\n", - }, - checks: [{ label: "slow-check", command: process.execPath, args: ["-e", longCheck(marker)] }], - runChecks: verifyCandidate, - signal: controller.signal, - worker: async (_task, frozen) => - JSON.stringify({ - digest: frozen.digest, - conclusion: "no-change-needed", - summary: "the check has not reported, so there is nothing to repair yet", - evidence: "the round was cancelled before the check terminal event", - citations: [], - }), - aInstruction: "repair what the check exposes", - bInstruction: "add the missing regressions", - aEditable: ["src/probe.ts"], - bEditable: ["src/probe.test.ts"], - budget: { perFile: 8_000, output: 8_000 }, - limits: { turns: 2, reads: 2, timeoutMs: 60_000 }, - }); - - assert.ok(await waitFor(() => existsSync(marker), 60_000), "the check never started"); - const pids = JSON.parse(readFileSync(marker, "utf8")) as { check: number; grandchild: number }; - assert.ok(alive(pids.check) && alive(pids.grandchild), "the check should be running"); - controller.abort(); - - const result = await round; - // The round's decision is explicit and durable, not an implied timeout. - assert.equal(result.cancelled, "operator cancelled the round"); - assert.deepEqual(result.accepted, {}); - assert.equal(result.composed.verdict, "undecidable"); - assert.equal( - result.log.findLast((event) => event.kind === "terminal")!.cancelled, - "operator cancelled the round", - ); - // Nothing the check was doing keeps running afterwards. - assert.ok( - await waitFor(() => !alive(pids.check) && !alive(pids.grandchild), 20_000), - "a check process survived the cancellation", - ); - // The cancelled round does not leak its candidate worktree either. - const worktrees = execFileSync("git", ["worktree", "list"], { - cwd: process.cwd(), - encoding: "utf8", - }); - const owned = worktrees - .split("\n") - .filter((line) => line.replaceAll("\\", "/").includes(directory.replaceAll("\\", "/"))); - assert.deepEqual(owned, [], `the candidate worktree leaked: ${owned.join(", ")}`); - }, -); - -test( - "cancelling with a live worker in another process leaves no ghost completion", - { timeout: 90_000 }, - async () => { - const directory = mkdtempSync(join(tmpdir(), "nmg-ooo-cancel-mp-")); - const database = join(directory, "nmg.sqlite"); - const actors: Actor[] = []; - const start = (role: string) => { - const actor = new Actor(role, database); - actors.push(actor); - return actor; - }; - try { - let daemon = start("daemon"); - let endpoint = (await daemon.ready) as ServerState; - const worker = start("worker"); - await worker.ready; - assert.equal(new Set([process.pid, daemon.child.pid, worker.child.pid]).size, 3); - await worker.call("connect", { endpoint, agent: "worker-1" }); - const ticket = await worker.call("take", { task: "B" }); - assert.equal(ticket.attempt, 1); - - // The worker is mid-attempt: its delivery has been verified and is held before the - // final transaction, which is the exact window a cancellation has to fence. - await daemon.call("pauseNext"); - const verified = daemon.event("verified"); - const pending = worker.call("deliver", { task: "B" }); - await verified; - // The operator cancels from a *different* process than the one holding the claim. - assert.equal( - await daemon.call("cancel", { reason: "operator stopped the round" }), - "operator stopped the round", - ); - await daemon.call("resume"); - - assert.equal(await pending, "stale", "a cancelled round must not complete the attempt"); - assert.deepEqual(await daemon.call("accepted"), {}); - assert.equal(await daemon.call("next"), null, "nothing is selectable after cancellation"); - const board = await worker.call("board"); - assert.equal( - board.filter( - (entry: TaskBoardEntry) => - entry.kind === "handoff" && JSON.parse(entry.content).id === "C", - ).length, - 0, - "a cancelled round must not unlock its dependent", - ); - - // The decision is durable: a restart keeps the reason and still refuses the worker. - await daemon.kill(); - daemon = start("daemon"); - endpoint = (await daemon.ready) as ServerState; - await worker.call("connect", { endpoint, agent: "worker-1" }); - assert.equal(await daemon.call("cancelled"), "operator stopped the round"); - assert.equal(await worker.call("deliver", { task: "B" }), "stale"); - assert.deepEqual(await daemon.call("accepted"), {}); - assert.equal(await daemon.call("next"), null); - // Two refusals, from two different layers: the published handoff is gone, and the - // coordinator itself refuses a new claim in a cancelled round. - await assert.rejects(worker.call("take", { task: "B" }), /no ready handoff/); - await assert.rejects(worker.call("requestClaim", { task: "B" }), /cancelled/); - } finally { - await Promise.all(actors.map((actor) => actor.kill())); - rmSync(directory, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); - } - }, -); diff --git a/evals/ooo-execution/check-events.test.ts b/evals/ooo-execution/check-events.test.ts index a8ed1532..249965f9 100644 --- a/evals/ooo-execution/check-events.test.ts +++ b/evals/ooo-execution/check-events.test.ts @@ -1,25 +1,25 @@ import assert from "node:assert/strict"; import test, { type TestContext } from "node:test"; -import { mkdtempSync, rmSync, readFileSync } from "node:fs"; +import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BoardAdmission } from "../../src/integration/ooo-board.ts"; -import { expectedRename, verifyRenameCandidate } from "../../src/integration/ooo-verifier.ts"; +import { verifyRenameCandidate } from "../../src/integration/check-runner.ts"; +import { expectedRenameOf, renameSource } from "./rename-probe.ts"; import { checkResultValid, sameCheck, type CheckResult, type CheckTicket, -} from "../../src/integration/ooo-check.ts"; +} from "../../src/integration/check-ticket.ts"; test("contract: real syntax-check terminal identity is admitted by the board coordinator", async (t) => { const { gate } = fixture(t); const ticket = gate.issueCheck("A", "syntax-host"); - const source = readFileSync( - new URL("../../src/integration/ooo-execution.ts", import.meta.url), - "utf8", - ); - const check = await verifyRenameCandidate(source, expectedRename(source), ticket.checkId); + // The probe's frozen target, not the live file: the same reason the patch probe gives (the + // candidate is the whole file, and the shared work contract bounds what a dependency may carry). + const source = renameSource(); + const check = await verifyRenameCandidate(source, expectedRenameOf(source), ticket.checkId); assert.equal(check.checkId, ticket.checkId); assert.equal(check.verdict, "accept"); gate.now = Date.now(); @@ -232,7 +232,7 @@ test("safety: an artifact submitted under a ticket from another run is stale, no // B is the selected task while A waits on its external event, so it is claimable. const ticket = gate.claim("B", "worker-1"); const entry = gate.putTaskBoardEntry({ - taskId: "ooo-process-probe", + taskId: gate.channel, agentId: ticket.owner, kind: "result", content: JSON.stringify({ ticket: { ...ticket, runId: "some-other-run" }, artifact: "6" }), diff --git a/evals/ooo-execution/cost-model.test.ts b/evals/ooo-execution/cost-model.test.ts new file mode 100644 index 00000000..84e6fd1c --- /dev/null +++ b/evals/ooo-execution/cost-model.test.ts @@ -0,0 +1,153 @@ +// The advisory cost model's own properties. These are the checks that separate "measured" from +// "ran": the design's ordering depends on this model being an instrument, so a violated property is +// a broken instrument rather than a finding, and each one throws here. +// +// The last case is the one worth keeping: the model may not grow a quality term. A simulated pass +// rate would be exactly the thing the design forbids ("模拟的 token 或命中率不冒充实测"), and it +// would be tempting to add. +import assert from "node:assert/strict"; +import test from "node:test"; +import { + assertModelProperties, + type CostParams, + fusionAccounting, + fusionVerdict, + type PlanShape, + planEdges, + simulatePlan, +} from "./cost-model.ts"; + +const base: CostParams = { + workMs: 8_000, + rederiveMs: 6_000, + hitRate: 0.5, + verifyMs: 1_500, + contextMsPerUnit: 1_200, + coarseContextSaving: 0.5, + sessionStartMs: 1_500, + sessionStartMeasured: false, + unitsPerSession: 2, + slots: 4, +}; + +test("the model's own properties hold, and it throws instead of warning", () => { + assertModelProperties(base); + assertModelProperties({ ...base, slots: 1 }); + assertModelProperties({ ...base, hitRate: 0, rederiveMs: 0 }); +}); + +test("one execution slot buys nothing, and splitting without slots is a loss", () => { + const independent: PlanShape = { units: 4, density: 0, seed: 7 }; + const serial = simulatePlan(independent, { ...base, slots: 1 }); + assert.equal(serial.savedMs, 0, "one slot cannot overlap anything"); + assert.ok( + serial.makespanMs > serial.coarseMs, + "a split on one slot pays four session boundaries and buys no overlap", + ); +}); + +test("independent work on enough slots finishes in one unit plus one check per unit", () => { + const independent: PlanShape = { units: 4, density: 0, seed: 7 }; + const wide = simulatePlan(independent, base); + const perUnit = + base.workMs + (1 - base.hitRate) * base.rederiveMs + base.contextMsPerUnit + base.verifyMs; + assert.equal(wide.makespanMs, perUnit + 3 * base.verifyMs); + assert.equal(wide.hostMs, 4 * base.verifyMs, "the host checks four candidates, one at a time"); + assert.ok(wide.makespanMs >= wide.criticalPathMs, "no schedule finishes below its critical path"); + assert.ok(wide.slotUtilisation > 0 && wide.slotUtilisation <= 1); +}); + +test("a chain gains nothing from splitting however many slots it has", () => { + const chain: PlanShape = { units: 4, density: 1, seed: 7 }; + const chained = simulatePlan(chain, base); + assert.equal(chained.savedMs, 0); + assert.ok(chained.makespanMs > chained.coarseMs, "the coarse arm keeps its one session"); +}); + +test("the turning point: independence pays, and a single dependency edge does not", () => { + const gain = (shape: PlanShape) => + simulatePlan(shape, base).coarseMs - simulatePlan(shape, base).makespanMs; + assert.ok(gain({ units: 4, density: 0, seed: 7 }) > 0, "independent units pay off"); + assert.ok(gain({ units: 2, density: 1, seed: 7 }) < 0, "one dependency edge is a chain"); + assert.ok( + gain({ units: 8, density: 0, seed: 7 }) > gain({ units: 4, density: 0, seed: 7 }), + "more independent units pay off more", + ); +}); + +test("the graph is derived from the seed, so a gain cannot come from a hand-picked shape", () => { + const edges = planEdges({ units: 6, density: 0.5, seed: 11 }); + assert.deepEqual(edges, planEdges({ units: 6, density: 0.5, seed: 11 })); + assert.equal(planEdges({ units: 6, density: 0, seed: 11 }).length, 0); + assert.equal( + planEdges({ units: 4, density: 1, seed: 11 }).length, + 6, + "every forward edge exists", + ); + assert.ok( + edges.every(([from, to]) => from < to), + "edges point forward, so the plan is acyclic", + ); +}); + +test("impossible input is refused rather than defaulted", () => { + const shape: PlanShape = { units: 4, density: 0, seed: 7 }; + assert.throws(() => simulatePlan(shape, { ...base, slots: 0 }), /slots/); + assert.throws(() => simulatePlan(shape, { ...base, hitRate: 1.5 }), /hit-rate/); + assert.throws(() => simulatePlan(shape, { ...base, verifyMs: -1 }), /verify-ms/); + assert.throws(() => planEdges({ units: 0, density: 0, seed: 1 }), /units/); + assert.throws(() => planEdges({ units: 4, density: 2, seed: 1 }), /density/); + assert.throws(() => simulatePlan(shape, { ...base, unitsPerSession: 0 }), /unitsPerSession/); + assert.throws(() => simulatePlan(shape, { ...base, unitsPerSession: 1.5 }), /unitsPerSession/); +}); + +test("fusion books the shared startup once per session, not once per unit", () => { + const shape: PlanShape = { units: 4, density: 0, seed: 7 }; + const fused = simulatePlan(shape, { ...base, unitsPerSession: 4 }); + const plain = simulatePlan(shape, base); + assert.equal(fused.sharedStartupMs, base.sessionStartMs, "one session pays its startup once"); + assert.equal( + fused.fusionSavedMs, + 3 * base.contextMsPerUnit, + "four units in one session drop three boundaries", + ); + assert.equal(fused.fusedMs, plain.makespanMs - 3 * base.contextMsPerUnit + base.sessionStartMs); + // The same plan with a bound of two: two sessions, each paying its own startup and dropping one + // boundary, and the two lines never collapse into one number. + const pairs = simulatePlan(shape, { ...base, unitsPerSession: 2 }); + assert.equal(pairs.sharedStartupMs, 2 * base.sessionStartMs); + assert.equal(pairs.fusionSavedMs, 2 * base.contextMsPerUnit); +}); + +test("a fusion bound of one unit removes no boundary and still pays the startup", () => { + const shape: PlanShape = { units: 4, density: 0, seed: 7 }; + const none = simulatePlan(shape, { ...base, unitsPerSession: 1 }); + assert.equal(none.fusionSavedMs, 0, "a unit per session has no boundary to remove"); + assert.equal(none.fusedMs, none.makespanMs + 4 * base.sessionStartMs); + assert.equal( + fusionVerdict(fusionAccounting(shape, { ...base, unitsPerSession: 1 }), base), + "none", + ); +}); + +test("an assumed session startup never reads as a gain", () => { + const shape: PlanShape = { units: 4, density: 0, seed: 7 }; + const fusion = fusionAccounting(shape, { ...base, unitsPerSession: 4 }); + assert.equal(fusionVerdict(fusion, base), "unmeasured", "no run has priced the startup"); + assert.equal(fusionVerdict(fusion, { ...base, sessionStartMeasured: true }), "gain"); + const costly = fusionAccounting(shape, { ...base, unitsPerSession: 2, sessionStartMs: 9_000 }); + assert.equal(fusionVerdict(costly, { ...base, sessionStartMeasured: true }), "cost"); +}); + +test("the model has no quality term: a simulated pass rate is not available to report", () => { + const result = simulatePlan({ units: 4, density: 0.25, seed: 7 }, base) as unknown as Record< + string, + unknown + >; + for (const key of Object.keys(result)) + assert.doesNotMatch( + key, + /quality|pass|accept|correct|success/i, + `the cost model grew a ${key} term; cost only, or the design's caveat is false`, + ); +}); diff --git a/evals/ooo-execution/cost-model.ts b/evals/ooo-execution/cost-model.ts new file mode 100644 index 00000000..67da0dbe --- /dev/null +++ b/evals/ooo-execution/cost-model.ts @@ -0,0 +1,534 @@ +// Advisory offline cost model for the granularity / concurrency arms of the main verification +// (`docs/design/task-unit-semantics.md`, "主验证:粒度、并发与融合"). +// +// The design orders this before any paid call: "离线模型先覆盖不同粒度、依赖密度、共享上下文、事实 +// 命中率及验证成本;它只能发现逻辑错误和成本转折点,不能预测真实模型质量". Two consequences are +// built in rather than promised: +// +// - **It simulates cost only.** There is no quality term, because nothing here may stand in for +// the parent check passing: a simulated pass rate is not a measured one. +// - **Every term is a declared parameter, not a fitted constant.** The terms that stand for a real +// quantity name it; the ones that are assumptions say so and are meant to be swept. +// +// What it computes for one plan shape and one cost vector: +// +// - the dependency graph, derived from (units, density, seed) rather than hand-written, so a +// "gain" cannot come from a graph picked to produce one; +// - the makespan under `slots` execution slots and a *single* host check queue (the host runs one +// candidate check at a time, which is the term the design says dominates every round); +// - the coarse arm's own makespan on the same vector, so the two arms differ only in shape; +// - the time concurrency actually bought, and the slot time nobody could use. +// +// Self-checks (`assertModelProperties`, all throwing): one slot reproduces the serial order and +// buys nothing; independent work on enough slots finishes in one unit plus as many checks as there +// are units; a chain buys nothing however many slots it has, and still pays the extra boundaries; +// and more dependencies never buy more overlap. The first version of this file failed the second +// check, because it forgot that the host queue serialises the checks. +// +// Usage: +// node --experimental-strip-types evals/ooo-execution/cost-model.ts --units 4 --density 0.2 \ +// --work-ms 8000 --rederive-ms 6000 --hit-rate 0.5 --verify-ms 1500 --context-ms 1200 \ +// --coarse-context-saving 0.5 --slots 3 --out +// node --experimental-strip-types evals/ooo-execution/cost-model.ts --sweep --out + +import { writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; + +/** The plan's shape. `density` is the fraction of the possible forward edges that exist, so 0 is + * fully independent work and 1 is a chain. */ +export interface PlanShape { + units: number; + density: number; + seed: number; +} + +/** The cost vector. Bracketed names say which real quantity a term stands for; `assumed` marks a + * term with no measurement behind it, which the sweep varies instead of fixing. */ +export interface CostParams { + /** Model work for one unit that has nothing to re-derive. [per-call latency] */ + workMs: number; + /** The work a fact hit removes: re-deriving an artifact that is already accepted. + * [fact hit rate] */ + rederiveMs: number; + hitRate: number; + /** One candidate check on the host, run one at a time. [host cost] */ + verifyMs: number; + /** A legal session boundary: re-establishing context for the next unit. [handoff cost] */ + contextMsPerUnit: number; + /** How much the parent's single session saves by not paying that boundary per unit. + * `assumed`: the coarse arm is not free either, and this is the term an operator must justify. */ + coarseContextSaving: number; + /** The cost one session pays once, however many units it runs: process and frozen-prompt startup. + * `assumed`: no run has measured it yet. */ + sessionStartMs: number; + /** Declared provenance of `sessionStartMs`, not a fitted constant. Until a run has measured it, + * fusion's difference is reported as unmeasured rather than read as a gain (`fusionVerdict`). */ + sessionStartMeasured: boolean; + /** How many units may share one session. Declared, not derived: the bound is the runtime's policy + * (short ready chains, not a greedy swallow of the DAG), and this model only prices the bound. */ + unitsPerSession: number; + slots: number; +} + +export interface Simulated { + units: number; + edges: number; + /** Wall time of the fine plan: model work plus host checks on one host queue. */ + makespanMs: number; + /** The same plan on one slot: the no-concurrency bound. */ + serialMs: number; + /** Wall time of the coarse arm (one session doing the same work) on the same vector. */ + coarseMs: number; + /** The fine plan's makespan with execution fusion: the same units and the same checks, less the + * boundaries fusion removes, plus the shared startup each session pays. */ + fusedMs: number; + /** What the removed session boundaries are worth, on the measured boundary term. Its own line. */ + fusionSavedMs: number; + /** The shared session startup, booked once per session and never amortised into a unit. Its own + * line, so the two can never be collapsed into one number that hides which term answered. */ + sharedStartupMs: number; + /** Host time spent checking candidates. Serial by construction. */ + hostMs: number; + /** What concurrency bought: `serialMs - makespanMs`, and it is zero for a chain. */ + savedMs: number; + /** The longest dependency chain, which is the floor no number of slots goes below. The design's + * question for the C arm is how much of this the schedule actually needs. */ + criticalPathMs: number; + /** Slot time not spent on model work, including slots deliberately left unused. */ + unusedSlotMs: number; + /** `busySlotMs / (makespanMs * slots)`: how much of the capacity bought was used. */ + slotUtilisation: number; +} + +/** Deterministic forward edges from (units, density, seed): a split's shape is swept, not chosen. */ +export function planEdges(shape: PlanShape): [number, number][] { + if (!Number.isInteger(shape.units) || shape.units < 1) + throw new Error(`units must be a positive integer, got ${shape.units}`); + if (!(shape.density >= 0 && shape.density <= 1)) + throw new Error(`density must be within [0,1], got ${shape.density}`); + const edges: [number, number][] = []; + let state = shape.seed >>> 0; + const random = () => { + // xorshift32: reproducible from the seed, and the seed is recorded in the output. + state ^= state << 13; + state >>>= 0; + state ^= state >> 17; + state ^= state << 5; + state >>>= 0; + return state / 0x1_0000_0000; + }; + for (let from = 0; from < shape.units; from += 1) + for (let to = from + 1; to < shape.units; to += 1) + if (random() < shape.density) edges.push([from, to]); + return edges; +} + +function unitCost(params: CostParams, withContext: boolean): number { + return ( + params.workMs + + (1 - params.hitRate) * params.rederiveMs + + (withContext ? params.contextMsPerUnit : 0) + ); +} + +/** + * Greedy list scheduling over `slots` for model work, followed by one host queue for the checks. + * A dependency is satisfied by an *accepted* artifact, so a unit waits for its predecessor's check, + * not for its model call — which is the difference an out-of-order round exists to exploit. + */ +function schedule( + shape: PlanShape, + params: CostParams, + edges: readonly [number, number][], +): { + makespanMs: number; + hostMs: number; + unusedSlotMs: number; + slotUtilisation: number; +} { + const perUnit = unitCost(params, true); + const prerequisites = new Map(); + for (const [from, to] of edges) prerequisites.set(to, [...(prerequisites.get(to) ?? []), from]); + + const slotFree = Array.from({ length: params.slots }, () => 0); + const startedAt = new Map(); + const modelEnd = new Map(); + const doneAt = new Map(); + let hostFree = 0; + let hostMs = 0; + + const units = Array.from({ length: shape.units }, (_, unit) => unit); + for (let guard = 0; doneAt.size < shape.units; guard += 1) { + if (guard > shape.units + 2) throw new Error("the schedule made no progress"); + // Start every unit whose predecessors are accepted, as early as a slot allows. Choosing the + // slot with the smallest `max(slot free, ready time)` is optimal for this greedy order. + const startable = units.filter( + (unit) => + !startedAt.has(unit) && (prerequisites.get(unit) ?? []).every((from) => doneAt.has(from)), + ); + for (const unit of startable) { + const readyAt = Math.max( + 0, + ...(prerequisites.get(unit) ?? []).map((from) => doneAt.get(from)!), + ); + let best = 0; + let bestAt = Number.POSITIVE_INFINITY; + for (let slot = 0; slot < slotFree.length; slot += 1) { + const at = Math.max(slotFree[slot]!, readyAt); + if (at < bestAt) { + bestAt = at; + best = slot; + } + } + startedAt.set(unit, bestAt); + modelEnd.set(unit, bestAt + perUnit); + slotFree[best] = bestAt + perUnit; + } + // The host checks finished work in the order it finished, one at a time. + const finishedWork = units + .filter((unit) => modelEnd.has(unit) && !doneAt.has(unit)) + .sort((left, right) => modelEnd.get(left)! - modelEnd.get(right)!); + for (const unit of finishedWork) { + hostFree = Math.max(hostFree, modelEnd.get(unit)!) + params.verifyMs; + hostMs += params.verifyMs; + doneAt.set(unit, hostFree); + } + } + const busyMs = shape.units * perUnit; + const makespanMs = Math.max(...doneAt.values(), ...slotFree); + const capacityMs = makespanMs * params.slots; + return { + makespanMs, + hostMs, + unusedSlotMs: Math.max(0, capacityMs - busyMs), + slotUtilisation: capacityMs === 0 ? 0 : busyMs / capacityMs, + }; +} + +function checkParams(params: CostParams): void { + if (!Number.isInteger(params.slots) || params.slots < 1) + throw new Error(`slots must be a positive integer, got ${params.slots}`); + for (const [name, value] of [ + ["hit-rate", params.hitRate], + ["coarse-context-saving", params.coarseContextSaving], + ] as const) + if (!(value >= 0 && value <= 1)) throw new Error(`${name} must be within [0,1], got ${value}`); + for (const [name, value] of [ + ["work-ms", params.workMs], + ["rederive-ms", params.rederiveMs], + ["verify-ms", params.verifyMs], + ["context-ms", params.contextMsPerUnit], + ["session-start-ms", params.sessionStartMs], + ] as const) + if (!(value >= 0) || !Number.isFinite(value)) + throw new Error(`${name} must be a finite non-negative number, got ${value}`); + checkedFusionUnits(params.unitsPerSession); +} + +/** The longest chain through the plan, by unit index (every edge points forward). This is the floor + * the makespan approaches as slots grow, so a report can say how much of the wait is structural. */ +function criticalPathMs( + shape: PlanShape, + params: CostParams, + edges: readonly [number, number][], +): number { + // Each unit on the chain costs its model work plus its own check: checks are serial and a + // dependent waits for an accepted artifact. Adding every check here would count them twice. + const perUnit = unitCost(params, true) + params.verifyMs; + const longest = Array.from({ length: shape.units }, () => perUnit); + for (const [from, to] of edges) longest[to] = Math.max(longest[to]!, longest[from]! + perUnit); + return Math.max(...longest); +} + +/** Fusion's two lines, kept apart by construction: what the removed boundaries are worth, and what + * the shared session startup costs instead. One net number would hide which of the two the answer + * came from, and booking the startup per unit is the double booking the design forbids. */ +export interface Fusion { + unitsPerSession: number; + sessions: number; + boundarySavedMs: number; + sharedStartupMs: number; +} + +/** The fusion bound is a policy, and a policy of half a unit is not a smaller bound: it is an unusable + * one, and rounding it silently would hide the caller's mistake. */ +function checkedFusionUnits(unitsPerSession: number): number { + if (!Number.isSafeInteger(unitsPerSession) || unitsPerSession < 1) + throw new Error(`unitsPerSession must be a positive integer, got ${unitsPerSession}`); + return unitsPerSession; +} + +export function fusionAccounting(shape: PlanShape, params: CostParams): Fusion { + const per = checkedFusionUnits(params.unitsPerSession); + const sessions = Math.ceil(shape.units / per); + return { + unitsPerSession: per, + sessions, + boundarySavedMs: (shape.units - sessions) * params.contextMsPerUnit, + sharedStartupMs: sessions * params.sessionStartMs, + }; +} + +/** A net gain is a finding only out of a term with a measurement behind it. Until a run has measured + * the session startup, fusion's difference is `unmeasured` - the two lines are reported, and no + * threshold is read out of a guess - and a bound of one unit fuses nothing at all. */ +export function fusionVerdict( + fusion: Fusion, + params: CostParams, +): "none" | "unmeasured" | "gain" | "cost" { + if (fusion.unitsPerSession === 1) return "none"; + if (!params.sessionStartMeasured) return "unmeasured"; + return fusion.sharedStartupMs < fusion.boundarySavedMs ? "gain" : "cost"; +} + +export function simulatePlan(shape: PlanShape, params: CostParams): Simulated { + checkParams(params); + const edges = planEdges(shape); + const fine = schedule(shape, params, edges); + const serial = schedule(shape, { ...params, slots: 1 }, edges); + // The coarse arm: one session doing the same work, and the same one check per unit of work, less + // whatever the single session saved by not paying a session boundary per unit. + const coarseMs = + shape.units * + (params.workMs + + (1 - params.hitRate) * params.rederiveMs - + params.contextMsPerUnit * params.coarseContextSaving) + + shape.units * params.verifyMs; + const fusion = fusionAccounting(shape, params); + return { + units: shape.units, + edges: edges.length, + makespanMs: fine.makespanMs, + serialMs: serial.makespanMs, + coarseMs, + fusedMs: fine.makespanMs - fusion.boundarySavedMs + fusion.sharedStartupMs, + fusionSavedMs: fusion.boundarySavedMs, + sharedStartupMs: fusion.sharedStartupMs, + hostMs: fine.hostMs, + savedMs: serial.makespanMs - fine.makespanMs, + criticalPathMs: criticalPathMs(shape, params, edges), + unusedSlotMs: fine.unusedSlotMs, + slotUtilisation: fine.slotUtilisation, + }; +} + +/** Properties the design would be wrong to violate, so a violation is a broken instrument rather + * than a finding. Throws; never warns. */ +export function assertModelProperties(params: CostParams): void { + const independent: PlanShape = { units: 4, density: 0, seed: 7 }; + const chain: PlanShape = { units: 4, density: 1, seed: 7 }; + const perUnit = unitCost(params, true); + + const one = simulatePlan(independent, { ...params, slots: 1 }); + const wide = simulatePlan(independent, { ...params, slots: 4 }); + if (wide.makespanMs > one.makespanMs) + throw new Error("more slots made an independent plan slower; the scheduler is wrong"); + if (one.savedMs !== 0) throw new Error("one slot reported a saving; nothing can overlap there"); + // One unit of work plus four serial checks. Forgetting the host queue is how this model first + // produced a wrong number, so the check names it. + const expected = perUnit + 4 * params.verifyMs; + if (Math.abs(wide.makespanMs - expected) > 1e-6) + throw new Error( + `four independent units on four slots should finish in ${expected}, got ${wide.makespanMs}`, + ); + + const chained = simulatePlan(chain, { ...params, slots: 4 }); + if (chained.savedMs !== 0) + throw new Error( + `splitting a chain reported ${chained.savedMs}ms saved; a chain has nothing to overlap`, + ); + if (chained.makespanMs < chained.coarseMs) + throw new Error( + "the coarse arm lost to a chain: the same work with fewer session boundaries cannot lose", + ); + + // More dependencies cannot create more overlap, and the floor never falls below the longest path. + let previous = Number.POSITIVE_INFINITY; + for (const density of [0, 0.25, 0.5, 1]) { + const plan = simulatePlan({ units: 8, density, seed: 7 }, { ...params, slots: 4 }); + if (plan.savedMs > previous + 1e-6) + throw new Error(`density ${density} saved more than a sparser plan; overlap is not monotone`); + if (plan.makespanMs < plan.criticalPathMs - 1e-6) + throw new Error(`a schedule finished below its own critical path at density ${density}`); + if (plan.slotUtilisation > 1) + throw new Error(`slot utilisation above 1 at density ${density}: ${plan.slotUtilisation}`); + previous = plan.savedMs; + } + + // Fusion's accounting has two ways to be wrong that a report would not show: booking the shared + // startup per unit (the double booking the design forbids), and pretending a bound that fuses + // nothing is free. The bound is set here rather than taken from `params`, so the property is about + // the accounting and not about whichever bound the caller declared. + const perSession = fusionAccounting( + { units: 4, density: 0, seed: 7 }, + { ...params, unitsPerSession: 4 }, + ); + if (perSession.sessions !== 1 || perSession.sharedStartupMs !== params.sessionStartMs) + throw new Error( + `one session of four units pays its startup once, got ${perSession.sharedStartupMs} for ${perSession.sessions} session(s)`, + ); + if (perSession.boundarySavedMs !== 3 * params.contextMsPerUnit) + throw new Error( + `one session of four units removes three boundaries, got ${perSession.boundarySavedMs}`, + ); + const nothingFused = simulatePlan(independent, { ...params, unitsPerSession: 1 }); + if (nothingFused.fusionSavedMs !== 0) + throw new Error( + `a bound of one unit removed ${nothingFused.fusionSavedMs}ms of boundaries; there are none to remove`, + ); + if (nothingFused.fusedMs < nothingFused.makespanMs) + throw new Error( + "a bound that fuses nothing reported a fused plan cheaper than the unfused one: it still pays the startup", + ); +} + +const USAGE = `usage: + cost-model.ts --units --density <0..1> --work-ms --rederive-ms --hit-rate <0..1> + --verify-ms --context-ms --coarse-context-saving <0..1> --slots + [--seed ] [--out ] + cost-model.ts --sweep [--out ]`; + +function number(values: Record, name: string): number { + const raw = values[name]; + if (typeof raw !== "string") throw new Error(`--${name} is required and must be a number`); + const parsed = Number(raw); + if (!Number.isFinite(parsed)) throw new Error(`--${name} is not a number: ${raw}`); + return parsed; +} + +/** The sweep's base vector: the terms with a real counterpart are set to values a live round has + * actually shown (a model call of seconds, a host check of ~1.5s); the two assumptions are varied + * in the report rather than trusted at one value. */ +const SWEEP_BASE: CostParams = { + workMs: 8_000, + rederiveMs: 6_000, + hitRate: 0.5, + verifyMs: 1_500, + contextMsPerUnit: 1_200, + coarseContextSaving: 0.5, + sessionStartMs: 1_500, + sessionStartMeasured: false, + unitsPerSession: 2, + slots: 4, +}; + +function run(): void { + const { values } = parseArgs({ + options: { + units: { type: "string" }, + density: { type: "string" }, + seed: { type: "string" }, + "work-ms": { type: "string" }, + "rederive-ms": { type: "string" }, + "hit-rate": { type: "string" }, + "verify-ms": { type: "string" }, + "context-ms": { type: "string" }, + "session-start-ms": { type: "string" }, + "session-start-measured": { type: "boolean" }, + "units-per-session": { type: "string" }, + "coarse-context-saving": { type: "string" }, + slots: { type: "string" }, + sweep: { type: "boolean" }, + out: { type: "string" }, + }, + allowPositionals: false, + }); + const output: Record = { + measuredAt: new Date().toISOString(), + model: "advisory-cost-only", + }; + + if (values.sweep) { + assertModelProperties(SWEEP_BASE); + const rows: Record[] = []; + for (const density of [0, 0.25, 0.5, 1]) { + for (const units of [1, 2, 4, 8]) { + const shape: PlanShape = { units, density, seed: 7 }; + const fine = simulatePlan(shape, SWEEP_BASE); + const serial = simulatePlan(shape, { ...SWEEP_BASE, slots: 1 }); + rows.push({ + units, + density, + edges: fine.edges, + coarseMs: Math.round(fine.coarseMs), + fineOneSlotMs: Math.round(serial.makespanMs), + fineMultiSlotMs: Math.round(fine.makespanMs), + hostMs: Math.round(fine.hostMs), + savedMs: Math.round(fine.savedMs), + criticalPathMs: Math.round(fine.criticalPathMs), + unusedSlotMs: Math.round(fine.unusedSlotMs), + slotUtilisation: Number(fine.slotUtilisation.toFixed(3)), + // Positive means the finer split with several slots beat the coarse arm on this vector. + gainVsCoarseMs: Math.round(fine.coarseMs - fine.makespanMs), + }); + } + } + output.params = SWEEP_BASE; + output.rows = rows; + output.paysFrom = rows + .filter((row) => Number(row.gainVsCoarseMs) > 0) + .map((row) => `units=${row.units} density=${row.density}`); + // Fusion's turning point, swept on both sides because neither term is measured: the boundary a + // session removes is priced by the pilot, the startup it pays is still an assumption. Every row + // therefore says `unmeasured`, and the thresholds below are what an experiment would test - not a + // verdict this model is entitled to give. + const fusionRows: Record[] = []; + for (const sessionStartMs of [0, 600, 1_500]) { + for (const unitsPerSession of [1, 2, 4]) { + const swept: CostParams = { ...SWEEP_BASE, sessionStartMs, unitsPerSession }; + const fusion = fusionAccounting({ units: 8, density: 0.25, seed: 7 }, swept); + fusionRows.push({ + units: 8, + unitsPerSession, + sessions: fusion.sessions, + sessionStartMs, + boundarySavedMs: Math.round(fusion.boundarySavedMs), + sharedStartupMs: Math.round(fusion.sharedStartupMs), + netMs: Math.round(fusion.sharedStartupMs - fusion.boundarySavedMs), + verdict: fusionVerdict(fusion, swept), + }); + } + } + output.fusionRows = fusionRows; + output.fusionThresholds = fusionRows + .filter((row) => Number(row.netMs) < 0) + .map((row) => `unitsPerSession=${row.unitsPerSession} sessionStartMs=${row.sessionStartMs}`); + } else { + const params: CostParams = { + workMs: number(values, "work-ms"), + rederiveMs: number(values, "rederive-ms"), + hitRate: number(values, "hit-rate"), + verifyMs: number(values, "verify-ms"), + contextMsPerUnit: number(values, "context-ms"), + coarseContextSaving: number(values, "coarse-context-saving"), + sessionStartMs: number(values, "session-start-ms"), + sessionStartMeasured: values["session-start-measured"] === true, + unitsPerSession: number(values, "units-per-session"), + slots: number(values, "slots"), + }; + const shape: PlanShape = { + units: number(values, "units"), + density: number(values, "density"), + seed: values.seed === undefined ? 7 : number(values, "seed"), + }; + assertModelProperties(params); + const fusion = fusionAccounting(shape, params); + output.params = params; + output.shape = shape; + output.result = simulatePlan(shape, params); + output.fusion = { ...fusion, verdict: fusionVerdict(fusion, params) }; + } + output.caveat = + "cost only: this model has no quality term, so it can show a cost turning point and cannot " + + "show that a finer split keeps parent quality."; + const text = JSON.stringify(output, null, 2); + if (values.out) writeFileSync(values.out, `${text}\n`); + console.log(text); +} + +const invokedPath = process.argv[1] ? resolve(process.argv[1]) : ""; +if (invokedPath === fileURLToPath(import.meta.url)) run(); + +export { USAGE }; diff --git a/evals/ooo-execution/cycle.test.ts b/evals/ooo-execution/cycle.test.ts deleted file mode 100644 index e44d9291..00000000 --- a/evals/ooo-execution/cycle.test.ts +++ /dev/null @@ -1,738 +0,0 @@ -import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; -import test from "node:test"; -import { promisify } from "node:util"; -import { runCycle, type CheckRunner } from "../../src/integration/ooo-cycle.ts"; -import { preparePatchWork, type FrozenPatchWork } from "../../src/integration/ooo-patch.ts"; - -const run = promisify(execFile); - -const IMPL = "src/check.ts"; -const TESTS = "src/check.test.ts"; -const baseline = { [IMPL]: "export const a = 1;\n", [TESTS]: "test('a', () => {});\n" }; -const checks = [{ label: "fixed", command: process.execPath, args: ["-e", "process.exit(0)"] }]; - -/** Stub host check: rejects a candidate that keeps marker "BROKEN". */ -const runChecks: CheckRunner = async ({ files }) => ({ - verdict: Object.values(files).some((text) => text.includes("BROKEN")) ? "reject" : "accept", - outcomes: [{ label: "fixed", status: "passed" }], -}); - -/** Stub mutants: the declaration is only usable if the frozen suite misses it, and - * a candidate closes it only by making the mutated implementation fail the check. */ -const mutant = (id: string, to: string) => ({ - id, - path: IMPL, - from: "export const a = 1;\n", - to, -}); -const mutationRunner: CheckRunner = async ({ files }) => { - const impl = files[IMPL] ?? ""; - const tests = files[TESTS] ?? ""; - const failed = - Object.values(files).some((text) => text.includes("BROKEN")) || - (impl.includes("MUTANT") && tests.includes("detects-mutant")); - return { - verdict: failed ? "reject" : "accept", - outcomes: [{ label: "fixed", status: failed ? "failed" : "passed" }], - }; -}; - -const options = ( - worker: Parameters[0]["worker"], - runner: CheckRunner = runChecks, -) => ({ - repository: process.cwd(), - revision: "0".repeat(40), - baseline, - checks, - runChecks: runner, - worker, - aInstruction: "A works.", - bInstruction: "B works.", - aEditable: [IMPL], - bEditable: [TESTS], - budget: { perFile: 20_000, output: 40_000 }, - limits: { turns: 4, reads: 3, timeoutMs: 60_000 }, -}); - -test("contract: B runs while A waits on the real check, then A, then C promotes", async () => { - const order: string[] = []; - const result = await runCycle( - options(async (taskId, frozen, dependencies) => { - order.push(taskId); - if (taskId === "A") { - // A's frozen envelope must carry the real check outcome, and its digest must - // bind that text, not the pre-check version. - assert.match(frozen.work.instruction, /Check [0-9a-f-]+ finished with accept/); - return JSON.stringify({ - digest: frozen.digest, - files: [{ path: IMPL, content: "export const a = 2;\n" }], - }); - } - if (taskId === "B") { - assert.deepEqual(dependencies, {}); - assert.match(frozen.work.instruction, /Host-proven faults/); - return JSON.stringify({ - digest: frozen.digest, - files: [{ path: TESTS, content: "test('a', () => {}); test('b', () => {});\n" }], - }); - } - assert.deepEqual(Object.keys(dependencies).sort(), ["A", "B"]); - return JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "promote-candidate", - summary: "composed check passed", - evidence: "candidate-check accept", - citations: [], - }); - }), - ); - assert.deepEqual(order, ["B", "A", "C"]); - assert.deepEqual(result.verdicts, { B: "accepted", A: "accepted", C: "accepted" }); - assert.equal(result.composed.verdict, "accept"); - const steps = result.timeline.map((entry) => entry.step); - assert.ok(steps.indexOf("check-issued") < steps.indexOf("claim:B")); - assert.ok(steps.indexOf("check-terminal") < steps.indexOf("claim:A")); - // A's input digest is the post-check envelope, so the check evidence is bound. - assert.match(result.timeline.find((entry) => entry.step === "claim:A")!.detail!, /digest=/); -}); - -test("safety: a worker that fails or truncates is a recorded failed attempt, not a crash", async () => { - for (const message of [ - "Pi snapshot task did not finish within its bounded contract: stopReason=length, turns=2", - "pi turn error: This operation was aborted", - ]) { - const result = await runCycle( - options(async () => { - throw new Error(message); - }), - ); - assert.equal(result.verdicts.B, "rejected"); - assert.equal(result.verdicts.A, undefined); - assert.equal(result.verdicts.C, undefined); - assert.deepEqual(result.accepted, {}); - assert.equal(result.rejections[0].artifact, message); - assert.ok(result.timeline.some((entry) => entry.step === "worker-failed:B")); - } -}); - -test("contract: an artifact the shared contract refuses records why, and still submits", async () => { - const result = await runCycle( - options(async (_task, frozen) => - // An extra top-level key is invisible in the verdict without this record. - JSON.stringify({ digest: frozen.digest, files: [], verdict: "accept" }), - ), - ); - assert.equal(result.verdicts.B, "rejected"); - const recorded = result.timeline.find((entry) => entry.step === "contract-error:B"); - assert.ok(recorded, "the shared contract's own reason is recorded"); - assert.match(recorded.detail!, /invalid patch files|invalid patch structure/); - assert.ok(result.timeline.some((entry) => entry.step === "submit:B")); -}); - -test("safety: rejection stops without synthetic lease expiry or automatic model retry", async () => { - let runs = 0; - const result = await runCycle( - options(async (taskId, frozen) => { - runs += 1; - const path = taskId === "A" ? IMPL : TESTS; - return JSON.stringify({ digest: frozen.digest, files: [{ path, content: "BROKEN\n" }] }); - }), - ); - assert.equal(result.verdicts.B, "rejected"); - assert.equal(result.verdicts.A, undefined); - assert.equal(result.verdicts.C, undefined); - assert.deepEqual(result.accepted, {}); - assert.equal(runs, 1); - assert.equal(result.rejections.length, 1); - assert.ok(!result.timeline.some((entry) => entry.step === "retry:B")); - assert.ok(result.timeline.some((entry) => entry.step === "skip:A")); - assert.ok(result.timeline.some((entry) => entry.step === "skip:C")); -}); - -test("safety: nonempty worker conclusions do not establish coverage", async () => { - for (const conclusion of ["no-change-needed", "cannot-complete", "promote-candidate"]) { - const result = await runCycle( - options(async (_task, frozen) => - JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion, - summary: "Everything is covered", - evidence: "the tests passed", - citations: [], - }), - ), - ); - assert.equal(result.verdicts.B, "rejected"); - assert.deepEqual(result.accepted, {}); - } -}); - -test("contract: a no-change conclusion is accepted only when every frozen case resolves", async () => { - const cases = [ - { name: "stale", token: "stale" }, - { name: "duplicate", token: "duplicate" }, - { name: "cancel-late", token: "cancel" }, - ]; - const run = (citations: { case: string; test: string }[]) => - runCycle({ - ...options(async (_task, frozen) => - JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "no-change-needed", - summary: "cases already covered", - evidence: "each cited title exists in the frozen test source", - citations, - }), - ), - noChangeCases: { B: cases }, - }); - const fully = [ - { case: "stale", test: "a" }, - { case: "duplicate", test: "a" }, - { case: "cancel-late", test: "a" }, - ]; - assert.equal((await run(fully)).verdicts.B, "accepted"); - // A missing case, a misquoted title, and an invented case all fail closed. - for (const citations of [ - [{ case: "stale", test: "a" }], - [ - { case: "stale", test: "a" }, - { case: "duplicate", test: "a" }, - { case: "cancel-late", test: "not a real test title" }, - ], - [...fully, { case: "invented", test: "a" }], - ]) { - const result = await run(citations); - assert.equal(result.verdicts.B, "rejected"); - assert.ok(result.timeline.some((entry) => entry.step === "citation-unresolved")); - } -}); - -test("safety: a cannot-complete report stops the round instead of unlocking dependents", async () => { - const result = await runCycle( - options(async (_task, frozen) => - JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "cannot-complete", - summary: "the required check cannot run here", - evidence: "host check missing", - citations: [], - }), - ), - ); - assert.equal(result.verdicts.B, "rejected"); - assert.equal(result.verdicts.A, undefined); - assert.equal(result.verdicts.C, undefined); - assert.ok(result.timeline.some((entry) => entry.step === "blocked")); -}); - -test("contract: only resolvable citations plus complete passing checks admit no-change", async () => { - const worker = async (_task: string, frozen: FrozenPatchWork) => - JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "no-change-needed", - summary: "bounded claim", - evidence: "host reviewed fixed assertion", - citations: [{ case: "stale", test: "a" }], - }); - for (const status of ["passed", "failed", "skipped"]) { - const result = await runCycle({ - ...options(worker, async () => ({ - verdict: "accept", - outcomes: [{ label: "fixed", status }], - })), - noChangeCases: { B: [{ name: "stale", token: "stale" }] }, - }); - assert.equal(result.verdicts.B, status === "passed" ? "accepted" : "rejected"); - } - const noCases = await runCycle(options(worker)); - assert.equal(noCases.verdicts.B, "rejected", "without host-frozen cases nothing is claimable"); - const empty = await runCycle({ - ...options(worker), - checks: [], - noChangeCases: { B: [{ name: "stale", token: "stale" }] }, - }); - assert.equal(empty.verdicts.B, "rejected", "an empty check set cannot admit a claim"); -}); - -test("contract: a patch proves a case only by naming its frozen title token", async () => { - const patch = (content: string) => async (_task: string, frozen: FrozenPatchWork) => - JSON.stringify({ digest: frozen.digest, files: [{ path: TESTS, content }] }); - const rules = { B: [{ name: "late-result-after-cancellation", token: "cancellation" }] }; - const named = await runCycle({ - ...options( - patch("test('a', () => {}); test('rejects a result after cancellation', () => {});\n"), - ), - noChangeCases: rules, - }); - assert.equal(named.verdicts.B, "accepted"); - const unnamed = await runCycle({ - ...options(patch("test('a', () => {}); test('tests a late result', () => {});\n")), - noChangeCases: rules, - }); - assert.equal(unnamed.verdicts.B, "rejected"); - assert.ok(unnamed.timeline.some((entry) => entry.step === "case-unresolved")); - // An unstated token is never guessed: the same patch fails without one. - const noToken = await runCycle({ - ...options( - patch("test('a', () => {}); test('rejects a result after cancellation', () => {});\n"), - ), - noChangeCases: { B: [{ name: "late-result-after-cancellation", token: "" }] }, - }); - assert.equal(noToken.verdicts.B, "rejected"); -}); - -test("contract: a declared mutant the baseline already detects invalidates the premise", async () => { - const dispatched: string[] = []; - const optionsWith = () => ({ - ...options(async (task, frozen) => { - dispatched.push(task); - return JSON.stringify({ - digest: frozen.digest, - files: [{ path: TESTS, content: "changed" }], - }); - }), - runChecks: async () => ({ - verdict: "reject" as const, - outcomes: [{ label: "fixed", status: "failed" }], - }), - mutations: { B: [mutant("m1", "export const a = 2;\n")] }, - }); - // The check always rejects, so the declared mutant is "already detected"; the - // premise is false and the round fails closed instead of crashing or accepting. - const result = await runCycle(optionsWith()); - assert.equal(result.verdicts.B, "rejected"); - assert.deepEqual(result.accepted, {}); - assert.ok(result.timeline.some((entry) => entry.step === "precondition-failed")); - assert.ok(result.timeline.some((entry) => entry.step === "premise-invalid")); - // The premise gates the *dispatch*, not only the acceptance: a false premise must not - // spend a model call on work the host will refuse on its own evidence. - assert.ok(!dispatched.includes("B"), `B was dispatched anyway: ${JSON.stringify(dispatched)}`); - assert.ok(result.rejections.some((item) => item.task === "B" && item.attempt === 0)); -}); - -test("contract: a patch is accepted only when it passes intact and kills a declared mutant", async () => { - const patch = (content: string) => async (_task: string, frozen: FrozenPatchWork) => - JSON.stringify({ digest: frozen.digest, files: [{ path: TESTS, content }] }); - const withMutant = (content: string) => ({ - ...options(patch(content), mutationRunner), - mutations: { B: [mutant("m1", "export const a = 1; // MUTANT\n")] }, - }); - // Passes intact, fails on the mutant: the gap is closed, so this is accepted. - const closing = await runCycle(withMutant("test('a', () => {}); // detects-mutant\n")); - assert.equal(closing.verdicts.B, "accepted"); - assert.deepEqual(closing.killed.B, ["m1"]); - assert.deepEqual(closing.survived.B ?? [], []); - // Passes both: the test does not detect the fault, so it proves nothing. - const useless = await runCycle(withMutant("test('a', () => {}); // no detection\n")); - assert.equal(useless.verdicts.B, "rejected"); - assert.ok(useless.timeline.some((entry) => entry.step === "gap-not-closed")); - assert.deepEqual(useless.survived.B, ["m1"]); - // A candidate that does not pass intact is rejected before any mutant runs. - const broken = await runCycle(withMutant("BROKEN // detects-mutant\n")); - assert.equal(broken.verdicts.B, "rejected"); - assert.equal(broken.killed.B, undefined); -}); - -test("contract: proven-gap mode replaces the title-token rule instead of stacking with it", async () => { - const result = await runCycle({ - ...options( - async (_task, frozen) => - JSON.stringify({ - digest: frozen.digest, - files: [{ path: TESTS, content: "test('a', () => {}); // detects-mutant\n" }], - }), - mutationRunner, - ), - // A title token the candidate cannot satisfy, plus a mutant it does satisfy. - noChangeCases: { B: [{ name: "artifact-escapes-scope", token: "escap" }] }, - mutations: { B: [mutant("m1", "export const a = 1; // MUTANT\n")] }, - }); - assert.equal(result.verdicts.B, "accepted"); - assert.ok(!result.timeline.some((entry) => entry.step === "case-unresolved")); - assert.deepEqual(result.killed.B, ["m1"]); -}); - -test("contract: a declared mutant is stated in the frozen instruction, so the worker can target it", async () => { - let instruction = ""; - const result = await runCycle({ - ...options(async (taskId, frozen) => { - if (taskId === "B") instruction = frozen.work.instruction; - return JSON.stringify({ - digest: frozen.digest, - files: [{ path: TESTS, content: "test('a', () => {}); // detects-mutant\n" }], - }); - }, mutationRunner), - mutations: { B: [mutant("m1", "export const a = 1; // MUTANT\n")] }, - }); - assert.equal(result.verdicts.B, "accepted"); - assert.match(instruction, /"id":"m1"/); - assert.match(instruction, /export const a = 1; \/\/ MUTANT/); -}); - -test("safety: a proven surviving mutant makes a no-change conclusion false", async () => { - const conclusion = async (_task: string, frozen: FrozenPatchWork) => - JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "no-change-needed", - summary: "already covered", - evidence: "cited title exists", - citations: [{ case: "stale", test: "a" }], - }); - const resolved = await runCycle({ - ...options(conclusion, mutationRunner), - noChangeCases: { B: [{ name: "stale", token: "stale" }] }, - }); - assert.equal(resolved.verdicts.B, "accepted", "a resolved citation is enough without mutants"); - const proven = await runCycle({ - ...options(conclusion, mutationRunner), - noChangeCases: { B: [{ name: "stale", token: "stale" }] }, - mutations: { B: [mutant("m1", "export const a = 1; // MUTANT\n")] }, - }); - assert.equal(proven.verdicts.B, "rejected"); - assert.ok(proven.timeline.some((entry) => entry.step === "gap-proven")); -}); - -test("contract: a declared precondition unmet by the accepted artifact reopens its producer", async () => { - let aRuns = 0; - const result = await runCycle({ - ...options(async (taskId, frozen) => { - if (taskId === "C") - return JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "promote-candidate", - summary: "composed", - evidence: "host check", - citations: [], - }); - if (taskId === "A") { - aRuns += 1; - // The first attempt produces an artifact that does not satisfy the declared - // requirement; only the reopened attempt adds the required test title. - const content = aRuns === 1 ? "plain\n" : "test('guards the protocol', () => {});\n"; - return JSON.stringify({ digest: frozen.digest, files: [{ path: IMPL, content }] }); - } - return JSON.stringify({ - digest: frozen.digest, - files: [{ path: TESTS, content: `test('b', () => {});\n` }], - }); - }), - requires: { C: [{ kind: "test-title", task: "A", token: "guards the protocol" }] }, - }); - assert.equal(aRuns, 2, "A ran once, was reopened, and ran again"); - assert.equal(result.verdicts.A, "accepted"); - assert.equal(result.verdicts.C, "accepted"); - assert.equal(result.measurements.reopens.length, 1); - const steps = result.timeline.map((entry) => entry.step); - assert.ok(steps.includes("dependency-rejected")); - assert.ok(steps.includes("reopen")); - // The reopened attempt gets fresh check evidence bound to the new input. - assert.equal(steps.filter((step) => step === "check-issued").length, 2); - assert.equal(result.composed.verdict, "accept"); -}); - -test("safety: exhausting the reopen budget blocks the dependent instead of looping", async () => { - const result = await runCycle({ - ...options(async (taskId, frozen) => - taskId === "C" - ? JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "promote-candidate", - summary: "composed", - evidence: "host check", - citations: [], - }) - : JSON.stringify({ - digest: frozen.digest, - files: [ - { - path: taskId === "A" ? IMPL : TESTS, - content: "still missing the requirement\n", - }, - ], - }), - ), - requires: { C: [{ kind: "test-title", task: "A", token: "never-appears" }] }, - maxReopens: 1, - }); - assert.equal(result.measurements.reopens.length, 1); - assert.equal(result.verdicts.C, "blocked"); - assert.ok(result.timeline.some((entry) => entry.step === "reopen-exhausted")); -}); - -test("contract: the round reports what the out-of-order task hid and what the host spent", async () => { - const result = await runCycle( - options(async (_task, frozen) => ({ - artifact: JSON.stringify({ - digest: frozen.digest, - files: [{ path: TESTS, content: "test('a', () => {}); test('c', () => {});\n" }], - }), - metrics: { tokens: 1_234, turns: 3, checks: 1 }, - })), - ); - assert.deepEqual(result.measurements.workers.B, { - tokens: 1_234, - turns: 3, - checks: 1, - ms: result.measurements.workers.B.ms, - }); - assert.ok(result.measurements.workers.B.ms >= 0); - assert.ok(result.measurements.hiddenWaitMs >= 0); - assert.ok(result.measurements.hostChecks > 0); - assert.ok(result.measurements.hostMs >= 0); - assert.deepEqual(result.measurements.reopens, []); -}); - -test("contract: a mid-attempt pushback ends the dependent and reopens the named dependency", async () => { - const cRuns: number[] = []; - let aRuns = 0; - const requirement = "A adds a test titled *plain* that really asserts the protocol"; - const result = await runCycle({ - ...options(async (taskId, frozen) => { - if (taskId === "C") { - cRuns.push(aRuns); - // The host can see the required title, so it lets C start; only C can discover - // that the placeholder test does not assert what the requirement means. - if (cRuns.length === 1) - return { - artifact: "", - pushback: { - dependency: "A", - requirement, - evidence: "the only matching test is a placeholder with no assertions", - }, - }; - return JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "promote-candidate", - summary: "the reopened artifact satisfies the requirement", - evidence: "the new title asserts the protocol", - citations: [], - }); - } - if (taskId === "A") { - aRuns += 1; - return JSON.stringify({ - digest: frozen.digest, - files: [ - { - path: IMPL, - content: - aRuns === 1 - ? "test('plain placeholder', () => {});\n" - : "test('plain', () => { assert.equal(1, 1); });\n", - }, - ], - }); - } - return JSON.stringify({ - digest: frozen.digest, - files: [{ path: TESTS, content: "test('b', () => {});\n" }], - }); - }), - requires: { C: [{ kind: "test-title", task: "A", token: "plain" }] }, - }); - assert.equal(cRuns.length, 2, "C ran, pushed back, and ran again"); - assert.equal(aRuns, 2, "the reopen made A run again with the consumer's evidence"); - assert.equal(result.measurements.reopens.length, 1); - assert.ok(result.timeline.some((entry) => entry.step === "pushback:C")); - assert.ok(result.timeline.some((entry) => entry.step === "reopen")); - assert.equal(result.verdicts.A, "accepted"); - assert.equal(result.verdicts.C, "accepted"); -}); - -const cleanCandidate = async (taskId: string, frozen: FrozenPatchWork) => { - if (taskId === "C") - return JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "no-change-needed", - summary: "Nothing to promote", - evidence: "no accepted patch in this round", - citations: [], - }); - return JSON.stringify({ - digest: frozen.digest, - files: [{ path: taskId === "A" ? IMPL : TESTS, content: `changed by ${taskId}` }], - }); -}; - -test("safety: stub workers cannot self-approve, and C cannot promote a rejected composition", async () => { - const selfApproved = await runCycle( - options(async (taskId, frozen) => { - const path = taskId === "A" ? IMPL : TESTS; - const extra = taskId === "A" ? { passed: true, verdict: "accept" } : {}; - return JSON.stringify({ - digest: frozen.digest, - files: [{ path, content: "changed" }], - ...extra, - }); - }), - ); - assert.equal(selfApproved.verdicts.B, "accepted"); - assert.equal(selfApproved.verdicts.A, "rejected"); - assert.equal(selfApproved.verdicts.C, undefined); - assert.deepEqual(Object.keys(selfApproved.accepted), ["B"]); - - const wrongConclusion = await runCycle(options(cleanCandidate)); - assert.equal(wrongConclusion.verdicts.A, "accepted"); - assert.equal(wrongConclusion.verdicts.C, "rejected"); -}); - -test("safety: an artifact carrying the pre-check digest is rejected, not accepted", async () => { - const stale = preparePatchWork({ - taskId: "A", - attempt: 1, - instruction: "pre-check instruction", - files: baseline, - editable: [IMPL], - budget: { perFile: 20_000, output: 40_000 }, - limits: { turns: 4, reads: 3, timeoutMs: 60_000 }, - }); - const result = await runCycle( - options(async (taskId, frozen) => { - const digest = taskId === "A" ? stale.digest : frozen.digest; - return JSON.stringify({ - digest, - files: [{ path: taskId === "A" ? IMPL : TESTS, content: "changed" }], - }); - }), - ); - assert.equal(result.verdicts.B, "accepted"); - assert.equal(result.verdicts.A, "rejected"); - assert.equal(result.verdicts.C, undefined); - assert.deepEqual(Object.keys(result.accepted), ["B"]); -}); - -test("safety: a premise that could not be measured is refused as unmeasured, not as detected", async () => { - const dispatched: string[] = []; - let calls = 0; - const result = await runCycle({ - ...options(async (task, frozen) => { - dispatched.push(task); - return JSON.stringify({ - digest: frozen.digest, - files: [{ path: TESTS, content: "changed" }], - }); - }), - // The mutant measurement cannot run at all: no result is a missing premise, never - // evidence that the fault is already covered. - runChecks: async ({ files }) => { - calls += 1; - return Object.values(files).some((text) => text.includes("MUTANT")) - ? { - verdict: "undecidable" as const, - outcomes: [{ label: "fixed", status: "undecidable" as const }], - } - : { verdict: "accept" as const, outcomes: [{ label: "fixed", status: "passed" as const }] }; - }, - mutations: { B: [mutant("m1", "export const a = 1; // MUTANT\n")] }, - }); - assert.ok(calls > 0, "the premise proof must actually run"); - assert.equal(result.verdicts.B, "rejected"); - assert.ok(result.timeline.some((entry) => entry.step === "premise-unmeasured")); - assert.ok(!result.timeline.some((entry) => entry.step === "precondition-failed")); - assert.ok(!dispatched.includes("B"), "an unproven premise must not spend a model call"); - assert.ok( - result.log.some((event) => event.kind === "mutant" && event.outcome === "unmeasured"), - JSON.stringify(result.log.filter((event) => event.kind === "mutant")), - ); -}); - -/** Real CPU work for a chosen duration, in a real child process: not a sleep, so the overlap - * measured here is real work overlapping real work. */ -const busy = (ms: number): string => - `const end=Date.now()+${ms};let a=1;while(Date.now() (ms > 0 ? run(process.execPath, ["-e", busy(ms)]) : Promise.resolve()); - -test("the hidden wait is the independent task's own work, not its later verification", async () => { - // The check is longer than the task's own work, while the task's verification (the candidate - // check the host runs afterwards) is longer still. Counting claim-to-submission therefore - // reports the whole check as hidden; counting claim-to-return reports what the task really - // covered. Only the second answer is the wait that out-of-order execution hid. - const checkMs = 1_500; - const taskMs = 200; - const slow: CheckRunner = async () => { - await work(checkMs); - return { verdict: "accept", outcomes: [{ label: "fixed", status: "passed" }] }; - }; - const result = await runCycle( - options(async (taskId, frozen) => { - if (taskId === "B") await work(taskMs); - if (taskId === "A") - return JSON.stringify({ - digest: frozen.digest, - files: [{ path: IMPL, content: "export const a = 2;\n" }], - }); - if (taskId === "B") - return JSON.stringify({ - digest: frozen.digest, - files: [{ path: TESTS, content: "test('a', () => {}); test('b', () => {});\n" }], - }); - return JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "promote-candidate", - summary: "composed check passed", - evidence: "candidate-check accept", - citations: [], - }); - }, slow), - ); - assert.deepEqual(result.verdicts, { B: "accepted", A: "accepted", C: "accepted" }); - const { hiddenWaitMs } = result.measurements; - assert.ok( - hiddenWaitMs < taskMs + 400, - `hidden wait ${hiddenWaitMs} ms must be bounded by B's own work (${taskMs} ms), not by the ${checkMs} ms check or by B's verification`, - ); - assert.ok( - hiddenWaitMs > taskMs / 2, - `a real overlap should still be reported, got ${hiddenWaitMs}`, - ); -}); - -test("with nothing independent to overlap, nothing is reported as hidden", async () => { - // The old definition reported the check's own window here (a few hundred milliseconds of - // dispatch and verification) even though no work was overlapped at all. - const result = await runCycle( - options( - async (taskId, frozen) => { - if (taskId === "A") - return JSON.stringify({ - digest: frozen.digest, - files: [{ path: IMPL, content: "export const a = 2;\n" }], - }); - if (taskId === "B") - return JSON.stringify({ - digest: frozen.digest, - files: [{ path: TESTS, content: "test('a', () => {}); test('b', () => {});\n" }], - }); - return JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "promote-candidate", - summary: "composed check passed", - evidence: "candidate-check accept", - citations: [], - }); - }, - async () => ({ verdict: "accept", outcomes: [{ label: "fixed", status: "passed" }] }), - ), - ); - assert.ok( - result.measurements.hiddenWaitMs < 100, - `no independent work ran, so hidden wait must be ~0, got ${result.measurements.hiddenWaitMs}`, - ); -}); diff --git a/evals/ooo-execution/families.test.ts b/evals/ooo-execution/families.test.ts new file mode 100644 index 00000000..42a9b7b7 --- /dev/null +++ b/evals/ooo-execution/families.test.ts @@ -0,0 +1,170 @@ +// The two task families the granularity arms run, checked offline and with no model: the instrument's +// own answers have to be accepted by both plans - one unit over the whole task, or four units where +// the last one waits for the three builders - and a wrong answer has to be rejected by the frozen +// checks. +// +// This is the F2c/F3 pair. The report family is the instrument's own; the pipeline family is held out +// of it, so a plan that only works on the family it was built against cannot pass here. The two specs +// of each family differ only in granularity, so a difference between the arms is the plan's and not +// the task's. +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import test from "node:test"; +import { cannedWorker, runPlan, specFrom, type SpecFile } from "./plan-driver.ts"; + +/** Each family: its directory, the one unit the coarse plan declares, and the four of the fine one + * in plan order - the last of which is the summary that waits for the other three. */ +const FAMILIES = [ + { + name: "report", + dir: "evals/ooo-execution/fixtures/report", + coarseUnit: "report", + fineUnits: ["alpha", "beta", "gamma", "summary"], + summary: "summary", + }, + { + name: "pipeline", + dir: "evals/ooo-execution/fixtures/pipeline", + coarseUnit: "pipeline", + fineUnits: ["normalize", "scale", "total", "summarize"], + summary: "summarize", + }, +] as const; + +const read = (dir: string, name: string): SpecFile => + JSON.parse(readFileSync(resolve(dir, name), "utf8")) as SpecFile; + +for (const family of FAMILIES) { + const coarse = read(family.dir, "coarse.spec.json"); + const fine = read(family.dir, "fine.spec.json"); + + test(`${family.name}: one plan is the other's work at another granularity`, () => { + assert.deepEqual( + Object.keys(coarse.units), + [family.coarseUnit], + "the coarse plan is a single unit over the whole task", + ); + assert.deepEqual(Object.keys(fine.units), family.fineUnits); + assert.deepEqual( + fine.plan.map((row) => row.id), + family.fineUnits, + ); + assert.deepEqual( + fine.plan.find((row) => row.id === family.summary)?.dependencies, + family.fineUnits.slice(0, 3), + "the summary depends on all three builders, which is what makes the granularity real", + ); + assert.deepEqual( + Object.values(fine.units) + .flatMap((unit) => unit.editable) + .sort(), + coarse.units[family.coarseUnit]! + .editable.slice() + .sort(), + "the fine plan's units cover exactly the files the coarse unit may edit", + ); + }); + + test(`${family.name}: both plans accept the instrument's answers, and the same composed ones`, async () => { + const arms = [ + { name: "coarse", file: coarse, slots: 1, units: 1 }, + { name: "fine", file: fine, slots: 1, units: 4 }, + { name: "fine", file: fine, slots: 2, units: 4 }, + ]; + const seen: { name: string; slots: number; units: number; host: number }[] = []; + for (const arm of arms) { + const run = await runPlan(specFrom(arm.file, cannedWorker(arm.file), arm.slots)); + assert.deepEqual(run.incomplete, [], `${arm.name}/${arm.slots}: ${run.incomplete.join("; ")}`); + assert.deepEqual( + run.units.map((unit) => unit.verdict), + run.units.map(() => "accepted"), + `${arm.name}/${arm.slots}: a correct answer must be accepted`, + ); + assert.equal( + run.parent?.verdict, + "accept", + `${arm.name}/${arm.slots}: the composed check sees every unit's work, not the last unit's ` + + "frozen copies of its siblings", + ); + assert.equal(run.units.length, arm.units); + seen.push({ name: arm.name, slots: arm.slots, units: run.units.length, host: run.hostChecks }); + } + // The composed acceptance is the same one in both plans - the same frozen checks over the same + // frozen files - so the granularity is the only thing the arms differ in. + assert.deepEqual( + coarse.parentChecks, + fine.parentChecks, + "both plans are accepted by the same composed check", + ); + assert.deepEqual( + seen.map((arm) => [arm.units, arm.host]), + [ + [1, 1], + [4, 4], + [4, 4], + ], + `the plans kept their own number of units: ${JSON.stringify(seen)}`, + ); + }); + + test(`${family.name}: a wrong answer fails the unit's own check, and the composition with it`, async () => { + const first = family.fineUnits[0]; + const unit = fine.units[first]!; + const editable = unit.editable[0]!; + const wrong: SpecFile = { + ...fine, + units: { + ...fine.units, + [first]: { + ...unit, + canned: { ...unit.canned, [editable]: `${family.dir}/${first}-wrong.canned.ts` }, + }, + }, + }; + const run = await runPlan(specFrom(wrong, cannedWorker(wrong), 1)); + assert.equal( + run.units.find((unit) => unit.taskId === first)?.verdict, + "rejected", + "the frozen check rejects the wrong answer, so acceptance is the check's judgement", + ); + assert.deepEqual( + run.order, + [first], + "a rejected attempt keeps its claim, so the plan does not move on to the next unit", + ); + assert.ok( + run.incomplete.some((entry) => entry.startsWith(first)), + `incomplete: ${JSON.stringify(run.incomplete)}`, + ); + assert.equal( + run.parent?.verdict, + "reject", + "the composition cannot accept without the builder its summary needs", + ); + }); + + test(`${family.name}: a unit nothing checks is refused rather than accepted on nothing`, () => { + const first = family.fineUnits[0]; + const unit = fine.units[first]!; + const unchecked: SpecFile = { + ...fine, + units: { + ...fine.units, + // The same unit without its own checks: a unit is checked by what it declares, or by the + // spec's own list, and having neither is a refusal rather than a unit accepted on nothing. + [first]: { + instruction: unit.instruction, + editable: unit.editable, + ...(unit.visible ? { visible: unit.visible } : {}), + ...(unit.canned ? { canned: unit.canned } : {}), + }, + }, + }; + assert.throws( + () => specFrom(unchecked, cannedWorker(unchecked), 1), + new RegExp(`${first}: no checks`), + "a unit is checked by what it declares, or by the spec's own list", + ); + }); +} diff --git a/evals/ooo-execution/fixtures/pipeline/coarse.spec.json b/evals/ooo-execution/fixtures/pipeline/coarse.spec.json new file mode 100644 index 00000000..cfc3265c --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/coarse.spec.json @@ -0,0 +1,95 @@ +{ + "baseline": [ + "evals/ooo-execution/fixtures/pipeline/frozen.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.ts", + "evals/ooo-execution/fixtures/pipeline/scale.ts", + "evals/ooo-execution/fixtures/pipeline/total.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ], + "plan": [{ "id": "pipeline", "effect": "isolated-artifact" }], + "units": { + "pipeline": { + "instruction": "Implement the four builders in this directory so that all five frozen test files pass. frozen.ts is frozen: do not change its shape. normalize keeps only the steps with a positive ms and returns them in name order, without changing the input array. scale multiplies every step's ms by the factor and rounds down. total sums the ms of the steps it is given. summarize renders every step it was given on its own line with renderStep, in the order given, then renders one more step named \"total\" whose ms is the total it was given.", + "editable": [ + "evals/ooo-execution/fixtures/pipeline/normalize.ts", + "evals/ooo-execution/fixtures/pipeline/scale.ts", + "evals/ooo-execution/fixtures/pipeline/total.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.ts" + ], + "canned": { + "evals/ooo-execution/fixtures/pipeline/normalize.ts": "evals/ooo-execution/fixtures/pipeline/normalize.canned.ts", + "evals/ooo-execution/fixtures/pipeline/scale.ts": "evals/ooo-execution/fixtures/pipeline/scale.canned.ts", + "evals/ooo-execution/fixtures/pipeline/total.ts": "evals/ooo-execution/fixtures/pipeline/total.canned.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.ts": "evals/ooo-execution/fixtures/pipeline/summarize.canned.ts" + } + } + }, + "checks": [ + { + "label": "normalize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts" + ] + }, + { + "label": "scale", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts" + ] + }, + { + "label": "total", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/total.test.ts" + ] + }, + { + "label": "summarize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts" + ] + }, + { + "label": "pipeline", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ] + } + ], + "parentChecks": [ + { + "label": "composed pipeline", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ] + } + ], + "worker": { "kind": "canned" } +} diff --git a/evals/ooo-execution/fixtures/pipeline/fine.spec.json b/evals/ooo-execution/fixtures/pipeline/fine.spec.json new file mode 100644 index 00000000..b73cf783 --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/fine.spec.json @@ -0,0 +1,114 @@ +{ + "baseline": [ + "evals/ooo-execution/fixtures/pipeline/frozen.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.ts", + "evals/ooo-execution/fixtures/pipeline/scale.ts", + "evals/ooo-execution/fixtures/pipeline/total.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.ts", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ], + "plan": [ + { "id": "normalize", "effect": "isolated-artifact" }, + { "id": "scale", "effect": "isolated-artifact" }, + { "id": "total", "effect": "isolated-artifact" }, + { + "id": "summarize", + "effect": "isolated-artifact", + "dependencies": ["normalize", "scale", "total"] + } + ], + "units": { + "normalize": { + "instruction": "Implement normalize in this directory so that normalize.test.ts passes: keep only the steps with a positive ms and return them in name order, without changing the input array. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": ["evals/ooo-execution/fixtures/pipeline/normalize.ts"], + "canned": { + "evals/ooo-execution/fixtures/pipeline/normalize.ts": "evals/ooo-execution/fixtures/pipeline/normalize.canned.ts" + }, + "checks": [ + { + "label": "normalize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts" + ] + } + ] + }, + "scale": { + "instruction": "Implement scale in this directory so that scale.test.ts passes: multiply every step's ms by the factor it is given and round down to whole milliseconds. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": ["evals/ooo-execution/fixtures/pipeline/scale.ts"], + "canned": { + "evals/ooo-execution/fixtures/pipeline/scale.ts": "evals/ooo-execution/fixtures/pipeline/scale.canned.ts" + }, + "checks": [ + { + "label": "scale", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts" + ] + } + ] + }, + "total": { + "instruction": "Implement total in this directory so that total.test.ts passes: sum the ms of the steps it is given, and answer 0 for no steps. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": ["evals/ooo-execution/fixtures/pipeline/total.ts"], + "canned": { + "evals/ooo-execution/fixtures/pipeline/total.ts": "evals/ooo-execution/fixtures/pipeline/total.canned.ts" + }, + "checks": [ + { + "label": "total", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/total.test.ts" + ] + } + ] + }, + "summarize": { + "instruction": "Implement summarize in this directory so that summarize.test.ts passes: render every step it was given with renderStep, in the order given, then render one more step named \"total\" whose ms is the total it was given, and join the lines with newlines. The three builders it is handed are already accepted. frozen.ts is frozen: do not change its shape, and do not edit any other file.", + "editable": ["evals/ooo-execution/fixtures/pipeline/summarize.ts"], + "canned": { + "evals/ooo-execution/fixtures/pipeline/summarize.ts": "evals/ooo-execution/fixtures/pipeline/summarize.canned.ts" + }, + "checks": [ + { + "label": "summarize", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts" + ] + } + ] + } + }, + "parentChecks": [ + { + "label": "composed pipeline", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/pipeline/normalize.test.ts", + "evals/ooo-execution/fixtures/pipeline/scale.test.ts", + "evals/ooo-execution/fixtures/pipeline/total.test.ts", + "evals/ooo-execution/fixtures/pipeline/summarize.test.ts", + "evals/ooo-execution/fixtures/pipeline/pipeline.test.ts" + ] + } + ], + "worker": { "kind": "canned" } +} diff --git a/evals/ooo-execution/fixtures/pipeline/frozen.ts b/evals/ooo-execution/fixtures/pipeline/frozen.ts new file mode 100644 index 00000000..0102e635 --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/frozen.ts @@ -0,0 +1,15 @@ +/** + * The frozen interface of the pipeline report this task family builds. + * + * Nothing here is anyone's unit: it is the contract the builders are written to, and the only + * renderer. A finer plan is the same work precisely because this file stays as it is. + */ +export interface Step { + readonly name: string; + readonly ms: number; +} + +/** The one renderer: a step is its name and its duration. */ +export function renderStep(step: Step): string { + return `${step.name}: ${step.ms}ms`; +} diff --git a/evals/ooo-execution/fixtures/pipeline/normalize-wrong.canned.ts b/evals/ooo-execution/fixtures/pipeline/normalize-wrong.canned.ts new file mode 100644 index 00000000..0daff2f7 --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/normalize-wrong.canned.ts @@ -0,0 +1,10 @@ +/** A wrong answer, not a stub: it returns the steps in reverse name order where the frozen check says + * name order, so a run that accepts it would be accepting on something other than the check. */ +import type { Step } from "./frozen.ts"; + +export function normalize(steps: readonly Step[]): Step[] { + return steps + .filter((step) => step.ms > 0) + .slice() + .sort((a, b) => b.name.localeCompare(a.name)); +} diff --git a/evals/ooo-execution/fixtures/pipeline/normalize.canned.ts b/evals/ooo-execution/fixtures/pipeline/normalize.canned.ts new file mode 100644 index 00000000..74c26b2d --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/normalize.canned.ts @@ -0,0 +1,9 @@ +/** + * The reference answer for the normalize unit, used by the canned worker: the instrument has to show + * the task family accepts a correct submission before a model is paid to produce one. + */ +import type { Step } from "./frozen.ts"; + +export function normalize(steps: readonly Step[]): Step[] { + return steps.filter((step) => step.ms > 0).slice().sort((a, b) => a.name.localeCompare(b.name)); +} diff --git a/evals/ooo-execution/fixtures/pipeline/normalize.test.ts b/evals/ooo-execution/fixtures/pipeline/normalize.test.ts new file mode 100644 index 00000000..0a94e86f --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/normalize.test.ts @@ -0,0 +1,19 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { normalize } from "./normalize.ts"; + +test("normalize keeps the steps that took time, in name order", () => { + assert.deepEqual( + normalize([ + { name: "b", ms: 2 }, + { name: "x", ms: 0 }, + { name: "a", ms: 1 }, + ]), + [ + { name: "a", ms: 1 }, + { name: "b", ms: 2 }, + ], + ); + assert.deepEqual(normalize([]), []); +}); diff --git a/evals/ooo-execution/fixtures/pipeline/normalize.ts b/evals/ooo-execution/fixtures/pipeline/normalize.ts new file mode 100644 index 00000000..9656a3b1 --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/normalize.ts @@ -0,0 +1,6 @@ +import type { Step } from "./frozen.ts"; + +/** The steps the report is built from: only the ones that took time, in name order. */ +export function normalize(_steps: readonly Step[]): Step[] { + throw new Error("not implemented"); +} diff --git a/evals/ooo-execution/fixtures/pipeline/pipeline.test.ts b/evals/ooo-execution/fixtures/pipeline/pipeline.test.ts new file mode 100644 index 00000000..192a01f1 --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/pipeline.test.ts @@ -0,0 +1,18 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { normalize } from "./normalize.ts"; +import { scale } from "./scale.ts"; +import { summarize } from "./summarize.ts"; +import { total } from "./total.ts"; + +/** The composition's own acceptance: the four builders over one input, rendered. */ +test("the composed report is the four builders over one input", () => { + const raw = [ + { name: "b", ms: 4 }, + { name: "x", ms: 0 }, + { name: "a", ms: 3 }, + ]; + const scaled = scale(normalize(raw), 2); + assert.equal(summarize(scaled, total(scaled)), "a: 6ms\nb: 8ms\ntotal: 14ms"); +}); diff --git a/evals/ooo-execution/fixtures/pipeline/scale.canned.ts b/evals/ooo-execution/fixtures/pipeline/scale.canned.ts new file mode 100644 index 00000000..d8b8bfa0 --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/scale.canned.ts @@ -0,0 +1,6 @@ +/** The reference answer for the scale unit. */ +import type { Step } from "./frozen.ts"; + +export function scale(steps: readonly Step[], factor: number): Step[] { + return steps.map((step) => ({ name: step.name, ms: Math.floor(step.ms * factor) })); +} diff --git a/evals/ooo-execution/fixtures/pipeline/scale.test.ts b/evals/ooo-execution/fixtures/pipeline/scale.test.ts new file mode 100644 index 00000000..6ce64a21 --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/scale.test.ts @@ -0,0 +1,22 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { scale } from "./scale.ts"; + +test("scale multiplies every step and rounds down to whole milliseconds", () => { + assert.deepEqual(scale([{ name: "a", ms: 3 }], 1.5), [{ name: "a", ms: 4 }]); + assert.deepEqual( + scale( + [ + { name: "a", ms: 2 }, + { name: "b", ms: 1 }, + ], + 2, + ), + [ + { name: "a", ms: 4 }, + { name: "b", ms: 2 }, + ], + ); + assert.deepEqual(scale([], 3), []); +}); diff --git a/evals/ooo-execution/fixtures/pipeline/scale.ts b/evals/ooo-execution/fixtures/pipeline/scale.ts new file mode 100644 index 00000000..97548535 --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/scale.ts @@ -0,0 +1,6 @@ +import type { Step } from "./frozen.ts"; + +/** The same steps at another size: whole milliseconds, rounded down. */ +export function scale(_steps: readonly Step[], _factor: number): Step[] { + throw new Error("not implemented"); +} diff --git a/evals/ooo-execution/fixtures/pipeline/summarize.canned.ts b/evals/ooo-execution/fixtures/pipeline/summarize.canned.ts new file mode 100644 index 00000000..6436d2fd --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/summarize.canned.ts @@ -0,0 +1,7 @@ +/** The reference answer for the summarize unit. */ +import type { Step } from "./frozen.ts"; +import { renderStep } from "./frozen.ts"; + +export function summarize(normalized: readonly Step[], totalMs: number): string { + return [...normalized.map(renderStep), renderStep({ name: "total", ms: totalMs })].join("\n"); +} diff --git a/evals/ooo-execution/fixtures/pipeline/summarize.test.ts b/evals/ooo-execution/fixtures/pipeline/summarize.test.ts new file mode 100644 index 00000000..485dd1a3 --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/summarize.test.ts @@ -0,0 +1,18 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import type { Step } from "./frozen.ts"; +import { summarize } from "./summarize.ts"; + +/** The summary's own acceptance needs the frozen interface and nothing else: which steps and which + * total it is handed is the composition's business, and the composed check is where the real + * builders meet. */ +const steps: Step[] = [ + { name: "a", ms: 1 }, + { name: "b", ms: 2 }, +]; + +test("the report lists the steps it was given, then the total it was given", () => { + assert.equal(summarize(steps, 3), "a: 1ms\nb: 2ms\ntotal: 3ms"); + assert.equal(summarize([], 0), "total: 0ms"); +}); diff --git a/evals/ooo-execution/fixtures/pipeline/summarize.ts b/evals/ooo-execution/fixtures/pipeline/summarize.ts new file mode 100644 index 00000000..d470b224 --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/summarize.ts @@ -0,0 +1,7 @@ +import type { Step } from "./frozen.ts"; + +/** The report: every step it was given, and the total it was given, rendered. Its input is the other + * three builders' results, which is why it can only be built once they exist. */ +export function summarize(_normalized: readonly Step[], _totalMs: number): string { + throw new Error("not implemented"); +} diff --git a/evals/ooo-execution/fixtures/pipeline/total.canned.ts b/evals/ooo-execution/fixtures/pipeline/total.canned.ts new file mode 100644 index 00000000..aa85b228 --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/total.canned.ts @@ -0,0 +1,6 @@ +/** The reference answer for the total unit. */ +import type { Step } from "./frozen.ts"; + +export function total(steps: readonly Step[]): number { + return steps.reduce((sum, step) => sum + step.ms, 0); +} diff --git a/evals/ooo-execution/fixtures/pipeline/total.test.ts b/evals/ooo-execution/fixtures/pipeline/total.test.ts new file mode 100644 index 00000000..b0356701 --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/total.test.ts @@ -0,0 +1,15 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { total } from "./total.ts"; + +test("total adds the steps it was given, and an empty pipeline is zero", () => { + assert.equal( + total([ + { name: "a", ms: 1 }, + { name: "b", ms: 2 }, + ]), + 3, + ); + assert.equal(total([]), 0); +}); diff --git a/evals/ooo-execution/fixtures/pipeline/total.ts b/evals/ooo-execution/fixtures/pipeline/total.ts new file mode 100644 index 00000000..0b86c90a --- /dev/null +++ b/evals/ooo-execution/fixtures/pipeline/total.ts @@ -0,0 +1,6 @@ +import type { Step } from "./frozen.ts"; + +/** The whole pipeline in one number: the sum of the steps it was given. */ +export function total(_steps: readonly Step[]): number { + throw new Error("not implemented"); +} diff --git a/evals/ooo-execution/fixtures/rename-baseline.ts b/evals/ooo-execution/fixtures/rename-baseline.ts new file mode 100644 index 00000000..db42c6ac --- /dev/null +++ b/evals/ooo-execution/fixtures/rename-baseline.ts @@ -0,0 +1,62 @@ +/** + * The patch probe's frozen rename target. + * + * This was the live `src/integration/ooo-execution.ts` until that file outgrew the bounds this probe + * passes its artifact through: the patch artifact is the whole file, and a dependent probe task + * carries it inside its snapshot, where the shared work contract bounds a dependency's serialized + * bytes at 8 KB. Both limits are deliberate - the probe just has to stay inside them, and pointing it + * at a growing product file made its own limits depend on how big that file had grown (it was within + * about a kilobyte of the second one when a legitimate edit crossed it, and a correct rename was then + * reported as "rejected"). + * + * The probe asserts a mechanism, not the identity of the file: an exact local rename is what the host + * accepts, and anything else is refused. So it freezes a small self-contained copy. + * + * The host's oracle (`expectedRename`) requires this shape: exactly one exported `nextTask`, the local + * map identifier declared inside it (the one the probe renames), no `planIndex` anywhere, and no + * further export after it. The comment deliberately avoids naming that identifier: the oracle renames + * only inside the target function, so a mention up here would survive the rename and make a candidate + * that is correct look wrong. + */ +export interface FrozenDispatchTask { + id: string; + effect: string; + sourceVersion?: string; + observedVersion?: string; + dependencies: readonly string[]; + accepted: boolean; + claimed: boolean; + delivered: boolean; + externalEvent?: string; + externalReady?: boolean; + cancelled: boolean; +} + +export function nextTask(plan: readonly FrozenDispatchTask[]): string | null { + const byId = new Map(plan.map((task) => [task.id, task])); + if (byId.size !== plan.length) throw new Error("duplicate task"); + const current = (task: FrozenDispatchTask) => + !!task.sourceVersion && task.sourceVersion === task.observedVersion; + const valid = (id: string, visiting = new Set()): boolean => { + const task = byId.get(id); + if (!task || !task.accepted || task.cancelled || !current(task) || visiting.has(id)) + return false; + const path = new Set(visiting).add(id); + return task.dependencies.every((dependency) => valid(dependency, path)); + }; + const waiting = (task: FrozenDispatchTask) => !!task.externalEvent && !task.externalReady; + const ready = (task: FrozenDispatchTask) => + current(task) && + !task.cancelled && + !waiting(task) && + ["read-only", "isolated-artifact"].includes(task.effect) && + task.dependencies.every((id) => valid(id)); + const selectable = plan.filter((task) => task.accepted || !task.delivered); + const pending = selectable.filter((task) => !valid(task.id)); + if (pending.some((task) => task.claimed) || pending.filter(waiting).length > 1) return null; + const first = pending[0]; + if (!first) return null; + if (ready(first)) return first.id; + if (!current(first) || !waiting(first)) return null; + return pending.slice(1).find(ready)?.id ?? null; +} diff --git a/evals/ooo-execution/fixtures/report/alpha-wrong.canned.ts b/evals/ooo-execution/fixtures/report/alpha-wrong.canned.ts new file mode 100644 index 00000000..14d0dcf4 --- /dev/null +++ b/evals/ooo-execution/fixtures/report/alpha-wrong.canned.ts @@ -0,0 +1,8 @@ +import type { Section } from "./interface.ts"; + +/** A wrong answer, not a stub: it sorts descending where the frozen check says ascending, so a run + * that accepts it would be accepting on something other than the check. */ +export function alphaSection(rows: readonly string[]): Section { + const kept = rows.filter((row) => row.trim() !== "").sort((a, b) => b.localeCompare(a)); + return { id: "alpha", title: "Alpha", lines: kept }; +} diff --git a/evals/ooo-execution/fixtures/report/alpha.canned.ts b/evals/ooo-execution/fixtures/report/alpha.canned.ts new file mode 100644 index 00000000..32c3664d --- /dev/null +++ b/evals/ooo-execution/fixtures/report/alpha.canned.ts @@ -0,0 +1,13 @@ +/** + * The reference answer for the alpha unit, used by the canned worker: the instrument has to show the + * task family accepts a correct submission before a model is paid to produce one. + */ +import type { Section } from "./interface.ts"; + +export function alphaSection(rows: readonly string[]): Section { + return { + id: "alpha", + title: "Alpha", + lines: rows.filter((row) => row !== "").slice().sort(), + }; +} diff --git a/evals/ooo-execution/fixtures/report/alpha.test.ts b/evals/ooo-execution/fixtures/report/alpha.test.ts new file mode 100644 index 00000000..dcaf24b0 --- /dev/null +++ b/evals/ooo-execution/fixtures/report/alpha.test.ts @@ -0,0 +1,13 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { alphaSection } from "./alpha.ts"; + +test("alpha lists the rows it was given, sorted, without blanks", () => { + assert.deepEqual(alphaSection(["b=2", "", "a=1"]), { + id: "alpha", + title: "Alpha", + lines: ["a=1", "b=2"], + }); + assert.deepEqual(alphaSection([]), { id: "alpha", title: "Alpha", lines: [] }); +}); diff --git a/evals/ooo-execution/fixtures/report/alpha.ts b/evals/ooo-execution/fixtures/report/alpha.ts new file mode 100644 index 00000000..8a829712 --- /dev/null +++ b/evals/ooo-execution/fixtures/report/alpha.ts @@ -0,0 +1,6 @@ +import type { Section } from "./interface.ts"; + +/** The alpha section: the rows it was given, sorted, with blanks dropped. */ +export function alphaSection(_rows: readonly string[]): Section { + throw new Error("not implemented"); +} diff --git a/evals/ooo-execution/fixtures/report/beta.canned.ts b/evals/ooo-execution/fixtures/report/beta.canned.ts new file mode 100644 index 00000000..b2a7db28 --- /dev/null +++ b/evals/ooo-execution/fixtures/report/beta.canned.ts @@ -0,0 +1,5 @@ +import type { Section } from "./interface.ts"; + +export function betaSection(rows: readonly string[]): Section { + return { id: "beta", title: "Beta", lines: rows.map((row, index) => `${index + 1}. ${row}`) }; +} diff --git a/evals/ooo-execution/fixtures/report/beta.test.ts b/evals/ooo-execution/fixtures/report/beta.test.ts new file mode 100644 index 00000000..d2e76e1f --- /dev/null +++ b/evals/ooo-execution/fixtures/report/beta.test.ts @@ -0,0 +1,13 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { betaSection } from "./beta.ts"; + +test("beta numbers the rows from one, in the order given", () => { + assert.deepEqual(betaSection(["x", "y"]), { + id: "beta", + title: "Beta", + lines: ["1. x", "2. y"], + }); + assert.deepEqual(betaSection(["only"]), { id: "beta", title: "Beta", lines: ["1. only"] }); +}); diff --git a/evals/ooo-execution/fixtures/report/beta.ts b/evals/ooo-execution/fixtures/report/beta.ts new file mode 100644 index 00000000..a4e2aec5 --- /dev/null +++ b/evals/ooo-execution/fixtures/report/beta.ts @@ -0,0 +1,6 @@ +import type { Section } from "./interface.ts"; + +/** The beta section: the rows it was given, numbered from one, in the order given. */ +export function betaSection(_rows: readonly string[]): Section { + throw new Error("not implemented"); +} diff --git a/evals/ooo-execution/fixtures/report/coarse.spec.json b/evals/ooo-execution/fixtures/report/coarse.spec.json new file mode 100644 index 00000000..c5e2a725 --- /dev/null +++ b/evals/ooo-execution/fixtures/report/coarse.spec.json @@ -0,0 +1,77 @@ +{ + "baseline": [ + "evals/ooo-execution/fixtures/report/interface.ts", + "evals/ooo-execution/fixtures/report/alpha.ts", + "evals/ooo-execution/fixtures/report/beta.ts", + "evals/ooo-execution/fixtures/report/gamma.ts", + "evals/ooo-execution/fixtures/report/summary.ts", + "evals/ooo-execution/fixtures/report/alpha.test.ts", + "evals/ooo-execution/fixtures/report/beta.test.ts", + "evals/ooo-execution/fixtures/report/gamma.test.ts", + "evals/ooo-execution/fixtures/report/summary.test.ts", + "evals/ooo-execution/fixtures/report/report.test.ts" + ], + "plan": [ + { "id": "report", "effect": "isolated-artifact" } + ], + "units": { + "report": { + "instruction": "Implement the three section builders and the summary in this directory so that all five frozen test files pass. The interface file is frozen: do not change its shape. alphaSection drops blank rows and sorts the rest. betaSection numbers the rows from one in the order given. gammaSection upper-cases the rows and keeps the first of a duplicate. summarySection lists one line per section, in the order it was given, each as '- ' plus that section's title, and describes itself with id \"summary\" and title \"Summary\".", + "editable": [ + "evals/ooo-execution/fixtures/report/alpha.ts", + "evals/ooo-execution/fixtures/report/beta.ts", + "evals/ooo-execution/fixtures/report/gamma.ts", + "evals/ooo-execution/fixtures/report/summary.ts" + ], + "canned": { + "evals/ooo-execution/fixtures/report/alpha.ts": "evals/ooo-execution/fixtures/report/alpha.canned.ts", + "evals/ooo-execution/fixtures/report/beta.ts": "evals/ooo-execution/fixtures/report/beta.canned.ts", + "evals/ooo-execution/fixtures/report/gamma.ts": "evals/ooo-execution/fixtures/report/gamma.canned.ts", + "evals/ooo-execution/fixtures/report/summary.ts": "evals/ooo-execution/fixtures/report/summary.canned.ts" + } + } + }, + "checks": [ + { + "label": "alpha", + "command": "node", + "args": ["--experimental-strip-types", "--test", "evals/ooo-execution/fixtures/report/alpha.test.ts"] + }, + { + "label": "beta", + "command": "node", + "args": ["--experimental-strip-types", "--test", "evals/ooo-execution/fixtures/report/beta.test.ts"] + }, + { + "label": "gamma", + "command": "node", + "args": ["--experimental-strip-types", "--test", "evals/ooo-execution/fixtures/report/gamma.test.ts"] + }, + { + "label": "summary", + "command": "node", + "args": ["--experimental-strip-types", "--test", "evals/ooo-execution/fixtures/report/summary.test.ts"] + }, + { + "label": "report", + "command": "node", + "args": ["--experimental-strip-types", "--test", "evals/ooo-execution/fixtures/report/report.test.ts"] + } + ], + "parentChecks": [ + { + "label": "composed report", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/alpha.test.ts", + "evals/ooo-execution/fixtures/report/beta.test.ts", + "evals/ooo-execution/fixtures/report/gamma.test.ts", + "evals/ooo-execution/fixtures/report/summary.test.ts", + "evals/ooo-execution/fixtures/report/report.test.ts" + ] + } + ], + "worker": { "kind": "canned" } +} diff --git a/evals/ooo-execution/fixtures/report/fine.spec.json b/evals/ooo-execution/fixtures/report/fine.spec.json new file mode 100644 index 00000000..a5a1b4e3 --- /dev/null +++ b/evals/ooo-execution/fixtures/report/fine.spec.json @@ -0,0 +1,110 @@ +{ + "baseline": [ + "evals/ooo-execution/fixtures/report/interface.ts", + "evals/ooo-execution/fixtures/report/alpha.ts", + "evals/ooo-execution/fixtures/report/beta.ts", + "evals/ooo-execution/fixtures/report/gamma.ts", + "evals/ooo-execution/fixtures/report/summary.ts", + "evals/ooo-execution/fixtures/report/alpha.test.ts", + "evals/ooo-execution/fixtures/report/beta.test.ts", + "evals/ooo-execution/fixtures/report/gamma.test.ts", + "evals/ooo-execution/fixtures/report/summary.test.ts", + "evals/ooo-execution/fixtures/report/report.test.ts" + ], + "plan": [ + { "id": "alpha", "effect": "isolated-artifact" }, + { "id": "beta", "effect": "isolated-artifact" }, + { "id": "gamma", "effect": "isolated-artifact" }, + { "id": "summary", "effect": "isolated-artifact", "dependencies": ["alpha", "beta", "gamma"] } + ], + "units": { + "alpha": { + "instruction": "Implement alphaSection in this directory so that alpha.test.ts passes. It drops blank rows and sorts the rest, and describes itself with id \"alpha\" and title \"Alpha\". The interface file is frozen: do not change its shape, and do not edit any other file.", + "editable": ["evals/ooo-execution/fixtures/report/alpha.ts"], + "canned": { + "evals/ooo-execution/fixtures/report/alpha.ts": "evals/ooo-execution/fixtures/report/alpha.canned.ts" + }, + "checks": [ + { + "label": "alpha", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/alpha.test.ts" + ] + } + ] + }, + "beta": { + "instruction": "Implement betaSection in this directory so that beta.test.ts passes. It numbers the rows from one in the order given, and describes itself with id \"beta\" and title \"Beta\". The interface file is frozen: do not change its shape, and do not edit any other file.", + "editable": ["evals/ooo-execution/fixtures/report/beta.ts"], + "canned": { + "evals/ooo-execution/fixtures/report/beta.ts": "evals/ooo-execution/fixtures/report/beta.canned.ts" + }, + "checks": [ + { + "label": "beta", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/beta.test.ts" + ] + } + ] + }, + "gamma": { + "instruction": "Implement gammaSection in this directory so that gamma.test.ts passes. It upper-cases the rows and keeps the first of a duplicate, and describes itself with id \"gamma\" and title \"Gamma\". The interface file is frozen: do not change its shape, and do not edit any other file.", + "editable": ["evals/ooo-execution/fixtures/report/gamma.ts"], + "canned": { + "evals/ooo-execution/fixtures/report/gamma.ts": "evals/ooo-execution/fixtures/report/gamma.canned.ts" + }, + "checks": [ + { + "label": "gamma", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/gamma.test.ts" + ] + } + ] + }, + "summary": { + "instruction": "Implement summarySection in this directory so that summary.test.ts passes. It lists one line per section, in the order it was given, each as '- ' plus that section's title, and describes itself with id \"summary\" and title \"Summary\". The three sections it is handed are already accepted; the interface file is frozen: do not change its shape, and do not edit any other file.", + "editable": ["evals/ooo-execution/fixtures/report/summary.ts"], + "canned": { + "evals/ooo-execution/fixtures/report/summary.ts": "evals/ooo-execution/fixtures/report/summary.canned.ts" + }, + "checks": [ + { + "label": "summary", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/summary.test.ts" + ] + } + ] + } + }, + "parentChecks": [ + { + "label": "composed report", + "command": "node", + "args": [ + "--experimental-strip-types", + "--test", + "evals/ooo-execution/fixtures/report/alpha.test.ts", + "evals/ooo-execution/fixtures/report/beta.test.ts", + "evals/ooo-execution/fixtures/report/gamma.test.ts", + "evals/ooo-execution/fixtures/report/summary.test.ts", + "evals/ooo-execution/fixtures/report/report.test.ts" + ] + } + ], + "worker": { "kind": "canned" } +} diff --git a/evals/ooo-execution/fixtures/report/gamma.canned.ts b/evals/ooo-execution/fixtures/report/gamma.canned.ts new file mode 100644 index 00000000..f642891a --- /dev/null +++ b/evals/ooo-execution/fixtures/report/gamma.canned.ts @@ -0,0 +1,10 @@ +import type { Section } from "./interface.ts"; + +export function gammaSection(rows: readonly string[]): Section { + const lines: string[] = []; + for (const row of rows) { + const upper = row.toUpperCase(); + if (!lines.includes(upper)) lines.push(upper); + } + return { id: "gamma", title: "Gamma", lines }; +} diff --git a/evals/ooo-execution/fixtures/report/gamma.test.ts b/evals/ooo-execution/fixtures/report/gamma.test.ts new file mode 100644 index 00000000..763c04b0 --- /dev/null +++ b/evals/ooo-execution/fixtures/report/gamma.test.ts @@ -0,0 +1,13 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { gammaSection } from "./gamma.ts"; + +test("gamma upper-cases the rows and keeps the first of a duplicate", () => { + assert.deepEqual(gammaSection(["a", "b", "a"]), { + id: "gamma", + title: "Gamma", + lines: ["A", "B"], + }); + assert.deepEqual(gammaSection([]), { id: "gamma", title: "Gamma", lines: [] }); +}); diff --git a/evals/ooo-execution/fixtures/report/gamma.ts b/evals/ooo-execution/fixtures/report/gamma.ts new file mode 100644 index 00000000..e0317815 --- /dev/null +++ b/evals/ooo-execution/fixtures/report/gamma.ts @@ -0,0 +1,6 @@ +import type { Section } from "./interface.ts"; + +/** The gamma section: the rows it was given, upper-cased, with later duplicates dropped. */ +export function gammaSection(_rows: readonly string[]): Section { + throw new Error("not implemented"); +} diff --git a/evals/ooo-execution/fixtures/report/interface.ts b/evals/ooo-execution/fixtures/report/interface.ts new file mode 100644 index 00000000..d96c3f07 --- /dev/null +++ b/evals/ooo-execution/fixtures/report/interface.ts @@ -0,0 +1,17 @@ +/** + * The frozen interface of the report this task family builds. + * + * Nothing in this file is anyone's unit: it is the contract the sections are built to, and the only + * renderer. A refinement of the work is legal precisely because this file stays as it is - the units + * change bodies, never this shape. + */ +export interface Section { + readonly id: string; + readonly title: string; + readonly lines: readonly string[]; +} + +/** The one renderer: a section is its title and its lines, and the composition is the join. */ +export function render(section: Section): string { + return [`## ${section.title}`, ...section.lines].join("\n") + "\n"; +} diff --git a/evals/ooo-execution/fixtures/report/report.test.ts b/evals/ooo-execution/fixtures/report/report.test.ts new file mode 100644 index 00000000..b79cd74d --- /dev/null +++ b/evals/ooo-execution/fixtures/report/report.test.ts @@ -0,0 +1,18 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { alphaSection } from "./alpha.ts"; +import { betaSection } from "./beta.ts"; +import { gammaSection } from "./gamma.ts"; +import { render } from "./interface.ts"; +import { summarySection } from "./summary.ts"; + +/** The composition's own acceptance: the frozen renderer over the composed sections. */ +test("the composed report renders the frozen interface's shape", () => { + const report = summarySection([ + alphaSection(["b=2", "a=1"]), + betaSection(["x"]), + gammaSection(["a", "a"]), + ]); + assert.equal(render(report), "## Summary\n- Alpha\n- Beta\n- Gamma\n"); +}); diff --git a/evals/ooo-execution/fixtures/report/summary.canned.ts b/evals/ooo-execution/fixtures/report/summary.canned.ts new file mode 100644 index 00000000..4c1a4b66 --- /dev/null +++ b/evals/ooo-execution/fixtures/report/summary.canned.ts @@ -0,0 +1,5 @@ +import type { Section } from "./interface.ts"; + +export function summarySection(sections: readonly Section[]): Section { + return { id: "summary", title: "Summary", lines: sections.map((section) => `- ${section.title}`) }; +} diff --git a/evals/ooo-execution/fixtures/report/summary.test.ts b/evals/ooo-execution/fixtures/report/summary.test.ts new file mode 100644 index 00000000..2e468dd2 --- /dev/null +++ b/evals/ooo-execution/fixtures/report/summary.test.ts @@ -0,0 +1,22 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import type { Section } from "./interface.ts"; +import { summarySection } from "./summary.ts"; + +/** The summary's own acceptance needs the frozen shape and nothing else: which sections it is handed + * is the composition's business, and the composed check is where the real builders meet. */ +const section = (id: string, title: string): Section => ({ id, title, lines: [] }); + +test("the summary lists the sections it was given, in the order they were given", () => { + assert.deepEqual(summarySection([section("alpha", "Alpha"), section("beta", "Beta")]), { + id: "summary", + title: "Summary", + lines: ["- Alpha", "- Beta"], + }); + assert.deepEqual(summarySection([section("gamma", "Gamma"), section("alpha", "Alpha")]), { + id: "summary", + title: "Summary", + lines: ["- Gamma", "- Alpha"], + }); +}); diff --git a/evals/ooo-execution/fixtures/report/summary.ts b/evals/ooo-execution/fixtures/report/summary.ts new file mode 100644 index 00000000..fcfe9d3b --- /dev/null +++ b/evals/ooo-execution/fixtures/report/summary.ts @@ -0,0 +1,7 @@ +import type { Section } from "./interface.ts"; + +/** The summary section: one line per section, in the order the sections were given. Its input is the + * other three sections, which is why it can only be built once they exist. */ +export function summarySection(_sections: readonly Section[]): Section { + throw new Error("not implemented"); +} diff --git a/evals/ooo-execution/fusion-ceiling.ts b/evals/ooo-execution/fusion-ceiling.ts new file mode 100644 index 00000000..342ddf7f --- /dev/null +++ b/evals/ooo-execution/fusion-ceiling.ts @@ -0,0 +1,114 @@ +/** + * Fusion ceiling: how few sessions a plan could need, and what the caps the repository can actually + * declare would get. + * + * This is the offline half of fusion planning (`docs/design/ooo-fusion-planning.md`). It costs no + * model calls and it is not a policy: a run may never read these numbers to decide a move, because + * they price an optimistic projection of the plan - every unit accepted, nothing cancelled, no + * external wait pending - instead of the facts a run holds. + * + * Usage: node --experimental-strip-types evals/ooo-execution/fusion-ceiling.ts [--spec ] + * + * It prints, per plan: the unit count, whether the successor relation is transitive, the + * chain-cover floor (the least number of sessions any order-respecting schedule could use), the + * sessions greedy list scheduling actually opens at each cap, and the milliseconds that saves against + * running one unit per session at the measured ~1_900 ms startup. Not modelled: the union tool + * surface's extra first-unit turn and the context a longer chain resends. + */ +import { readFileSync } from "node:fs"; +import { + chainCoverFloor, + fusionGraph, + listScheduleSessions, + MEASURED_SESSION_STARTUP_MS, +} from "../../src/integration/ooo-fusion-plan.ts"; +import type { DispatchTask, SessionPlan } from "../../src/integration/ooo-execution.ts"; + +const DEFAULT_SPEC = "evals/ooo-execution/fixtures/report/fine.spec.json"; + +interface SpecRow { + id: string; + effect?: string; + dependencies?: readonly string[]; +} + +interface SpecFile { + plan: readonly SpecRow[]; + units: Readonly>; + revision?: string; + fusion?: { + declarations?: Readonly>; + }; +} + +function readSpec(path: string): SpecFile { + const spec = JSON.parse(readFileSync(path, "utf8")) as SpecFile; + if (!Array.isArray(spec.plan) || spec.plan.length === 0) + throw new Error("spec.plan must be non-empty"); + return spec; +} + +/** + * The plan as the offline half sees it: every unit accepted and ready. The projection is deliberate - + * a run's verdicts, cancellations and pending branches are facts it does not have yet - and it leaves + * the five legality conditions to `sharedSessionLegal`, which still decides condition 1. + */ +function planFrom(spec: SpecFile): SessionPlan { + const revision = spec.revision ?? "v1"; + const tasks: DispatchTask[] = spec.plan.map((row) => ({ + id: row.id, + effect: row.effect ?? "isolated-artifact", + sourceVersion: revision, + observedVersion: revision, + dependencies: [...(row.dependencies ?? [])], + accepted: true, + claimed: false, + externalReady: true, + })); + const declarations = Object.fromEntries( + spec.plan.map((row) => [ + row.id, + { + capability: spec.fusion?.declarations?.[row.id]?.capability ?? "patch", + authority: spec.fusion?.declarations?.[row.id]?.authority ?? "host", + visible: spec.units[row.id]?.visible ?? [], + }, + ]), + ); + return { tasks, declarations }; +} + +function report(path: string): Record { + const spec = readSpec(path); + const plan = planFrom(spec); + const graph = fusionGraph(plan); + const floor = chainCoverFloor(graph); + const caps = Array.from({ length: graph.units.length }, (_, index) => index + 1); + const perCap = caps.map((cap) => { + const sessions = listScheduleSessions(graph, cap); + return { + cap, + sessions: sessions.length, + savedStartupMs: (graph.units.length - sessions.length) * MEASURED_SESSION_STARTUP_MS, + sizes: sessions.map((session) => session.length), + }; + }); + if (perCap.some((row) => row.sessions < floor)) { + throw new Error( + `${path}: a schedule used fewer sessions than the floor, so the floor is wrong`, + ); + } + return { + spec: path, + units: graph.units.length, + transitive: graph.transitive, + chainCoverFloor: floor, + oneUnitPerSession: graph.units.length, + perCap, + note: "savedStartupMs is against one unit per session, at the measured ~1_900 ms startup; the union tool surface's extra turn and a longer chain's context are not modelled", + }; +} + +const argument = process.argv.indexOf("--spec"); +const paths = argument === -1 ? [DEFAULT_SPEC] : [process.argv[argument + 1] ?? DEFAULT_SPEC]; +console.log(JSON.stringify(paths.map(report), null, 2)); diff --git a/evals/ooo-execution/live-continuation.ts b/evals/ooo-execution/live-continuation.ts index 439e4308..fdec285a 100644 --- a/evals/ooo-execution/live-continuation.ts +++ b/evals/ooo-execution/live-continuation.ts @@ -58,6 +58,8 @@ interface Task { parent: Case[]; } +const throws = (name: string, call: Case["call"]): Case => ({ name, call, expected: "throws" }); + /** Five tasks, one shape: a frozen contract, a part-1 slice, and edge cases behind the boundary. */ const TASKS: Task[] = [ { @@ -111,12 +113,8 @@ const TASKS: Task[] = [ }, { name: "empty input", call: (m) => m.chunk([] as never, 3 as never), expected: [] }, { name: "size one", call: (m) => m.chunk([7] as never, 1 as never), expected: [[7]] }, - { name: "size zero", call: (m) => m.chunk([1] as never, 0 as never), expected: "throws" }, - { - name: "fractional size", - call: (m) => m.chunk([1] as never, 1.5 as never), - expected: "throws", - }, + throws("size zero", (m) => m.chunk([1] as never, 0 as never)), + throws("fractional size", (m) => m.chunk([1] as never, 1.5 as never)), ], }, { @@ -271,16 +269,8 @@ const TASKS: Task[] = [ expected: [], }, { name: "empty input", call: (m) => m.movingAverage([] as never, 3 as never), expected: [] }, - { - name: "window zero", - call: (m) => m.movingAverage([1] as never, 0 as never), - expected: "throws", - }, - { - name: "window not an integer", - call: (m) => m.movingAverage([1] as never, 1.5 as never), - expected: "throws", - }, + throws("window zero", (m) => m.movingAverage([1] as never, 0 as never)), + throws("window not an integer", (m) => m.movingAverage([1] as never, 1.5 as never)), ], }, { @@ -577,6 +567,8 @@ if (role === "part1" || role === "part2") { writeFileSync(keptPath, execution.artifact, "utf8"); throw new Error( `${message} (conclusion=${conclusionKind ?? "none"}, submitted artifact kept at ${keptPath})`, + // The message quotes the failure; the failure itself stays reachable as data, so a + // reader that needs the original (or its own cause) is not left parsing prose. { cause: error }, ); } diff --git a/evals/ooo-execution/live-cycle.ts b/evals/ooo-execution/live-cycle.ts deleted file mode 100644 index 40f03280..00000000 --- a/evals/ooo-execution/live-cycle.ts +++ /dev/null @@ -1,365 +0,0 @@ -// Explicit --live required. Real model calls, real git-worktree candidate checks, -// real background check concurrent with B. Nothing in the working tree is modified. -// Requires PI_PROVIDER and PI_MODEL; use the provider the user authorized. -import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { execFileSync } from "node:child_process"; -import { runCycle, type Requirement } from "../../src/integration/ooo-cycle.ts"; -import { mutate, type Mutation } from "../../src/integration/ooo-mutation.ts"; -import { - RoundLog, - compareFrozen, - compareTerminal, - readRoundLog, - recordedPlan, - recordedWorker, - terminalEvent, -} from "../../src/integration/ooo-round-log.ts"; -import { verifyCandidate } from "../../src/integration/ooo-candidate.ts"; -import { - executePiPatch, - type CheckTool, - type PushbackSpec, -} from "../../.pi/extensions/nmg/ooo-execution.ts"; - -const provider = process.env.PI_PROVIDER; -const model = process.env.PI_MODEL; -/** Replay mode re-runs this round's host checks against the answers its log recorded. It - * needs no provider: the model is the activity being replayed, not re-executed. */ -const replayMode = process.argv.includes("--replay"); -if (!replayMode && !process.argv.includes("--live")) - throw new Error("pass --live explicitly (this calls the configured model), or --replay"); -if (!replayMode && (!provider || !model)) - throw new Error("Set PI_PROVIDER and PI_MODEL explicitly"); -const logPath = ".nmg/ooo-live/round.jsonl"; -const recordedEvents = replayMode ? readRoundLog(readFileSync(logPath, "utf8")) : []; -/** A replay checks out the revision the round used, not whatever HEAD is now: a round's - * identity is its inputs, and a moved HEAD would silently verify different code. */ -const recordedRevision = replayMode ? (recordedPlan(recordedEvents)?.revision ?? null) : null; -const replayWorker = recordedWorker(recordedEvents); - -const repository = process.cwd(); -const revision = replayMode - ? (recordedRevision ?? "") - : execFileSync("git", ["rev-parse", "HEAD"], { - cwd: repository, - encoding: "utf8", - }).trim(); -if (replayMode && !recordedRevision) - console.warn(`${logPath} records no revision: the replay reads the current checkout`); -const read = (path: string) => readFileSync(new URL(`../../${path}`, import.meta.url), "utf8"); - -/** Round-frozen baseline: every untracked OoO file the fixed check needs. */ -const baselinePaths = [ - "src/integration/ooo-execution.ts", - "src/integration/ooo-patch.ts", - "src/integration/ooo-check.ts", - "evals/ooo-execution/board-admission.ts", - "evals/ooo-execution/patch-verifier.ts", - "evals/ooo-execution/check-events.test.ts", - "evals/ooo-execution/patch-cycle.test.ts", -]; -const baseline = Object.fromEntries(baselinePaths.map((path) => [path, read(path)])); -const checks = [ - { - label: "protocol-regression", - command: process.execPath, - args: [ - "--experimental-strip-types", - "--test", - "evals/ooo-execution/check-events.test.ts", - "evals/ooo-execution/patch-cycle.test.ts", - ], - }, -]; - -// Only the waiting task's cases; B is in proven-gap mode, which replaces the token -// rule with "passes intact and kills a declared mutant". -const cases = { - A: [{ name: "check-identity", token: "samecheck rejects a ticket whose attempt differs" }], -}; - -/** Host-owned mutants of the editable implementation. Each is proven to survive the - * frozen suite before the round starts (cycle.ts refuses the round otherwise), so - * "this fault goes undetected" is evidence rather than an assumption. - * - * This list is refreshed by probing candidates against the *current* frozen suite - * (`MUTATION_SPEC= mutation-probe.ts`); the fault the previous round proved is - * now detected by the baseline that round produced, so re-declaring it would be a - * false premise. All five below were reported `survived` against this revision. */ -const mutants: readonly Mutation[] = [ - { - id: "bound-ignores-runid", - path: "evals/ooo-execution/board-admission.ts", - from: " ticket.runId === this.runId &&\n", - to: "", - }, - { - id: "bound-ignores-input-digest", - path: "evals/ooo-execution/board-admission.ts", - from: " ticket.inputDigest === row.input_digest &&\n", - to: "", - }, - { - id: "issueCheck-skips-stale-input-guard", - path: "evals/ooo-execution/board-admission.ts", - from: ' if (row.source_revision !== row.observed_revision) throw new Error("stale check input");\n', - to: "", - }, - { - id: "cancel-keeps-check-tickets", - path: "evals/ooo-execution/board-admission.ts", - from: " this.db.prepare(\"UPDATE ooo_probe_checks SET terminal='cancelled', cancelled=1\").run();\n", - to: "", - }, - { - id: "withdraw-handoff-keeps-entry", - path: "evals/ooo-execution/board-admission.ts", - from: " this.fenceRow(this.row(taskId), [], reason);", - to: " void reason;", - }, -]; - -/** The worker gets exactly the round's own check, bounded, so it can verify its own - * patch. Every earlier round failed on a test the worker could not run. */ -const checkTool: CheckTool = { - label: "run_check", - maxRuns: 4, - run: async (files) => { - const result = await verifyCandidate({ - repository, - revision, - files: { ...baseline, ...Object.fromEntries(files.map((file) => [file.path, file.content])) }, - checks, - }); - const failed = result.outcomes.find((outcome) => outcome.status !== "passed"); - return { - verdict: result.verdict, - log: - result.outcomes.map((outcome) => `${outcome.label}=${outcome.status}`).join(", ") + - (failed?.log ? ` | ${failed.log.slice(-2_000)}` : ""), - }; - }, -}; - -/** What the composition task declares it needs from its dependencies. The host - * evaluates these mechanically; the worker may also push back mid-attempt when what - * it received cannot satisfy one of them. */ -const requirements: readonly Requirement[] = [ - // Derived from the round's own declaration, never written by hand: a hard-coded id goes - // stale as soon as a round closes the fault it names, and the stale requirement was - // exactly what B pushed back on with first-hand evidence from the frozen test file. - { kind: "mutant-killed", task: "B", id: mutants[0]!.id }, -]; -const pushback: PushbackSpec = { - requirements: requirements.map((requirement) => ({ - task: requirement.task, - requirement: - requirement.kind === "mutant-killed" - ? `${requirement.task} kills ${requirement.id}` - : `${requirement.task} satisfies ${requirement.kind}`, - })), -}; - -const runs: number[] = []; - -const result = await runCycle({ - repository, - revision, - baseline, - checks, - noChangeCases: cases, - // The round records itself as it runs, so it can be replayed later without a model: - // replay re-checks the recorded answers through the host and re-derives every verdict. - roundLog: replayMode ? undefined : new RoundLog(logPath), - mutations: { B: mutants }, - requires: { C: requirements }, - maxReopens: 1, - // The whole baseline is re-sent on every model turn. Each task reads only the files - // its acceptance rule can point at (the implementation under test and the tests that - // exercise it); the large harness files stay frozen and hidden, and the host still - // verifies against all of them. - visible: { - A: [ - "src/integration/ooo-check.ts", - "evals/ooo-execution/board-admission.ts", - "evals/ooo-execution/patch-verifier.ts", - "evals/ooo-execution/check-events.test.ts", - "evals/ooo-execution/patch-cycle.test.ts", - ], - B: [ - "src/integration/ooo-check.ts", - "evals/ooo-execution/board-admission.ts", - "evals/ooo-execution/patch-verifier.ts", - "evals/ooo-execution/check-events.test.ts", - "evals/ooo-execution/patch-cycle.test.ts", - ], - C: [ - "src/integration/ooo-check.ts", - "evals/ooo-execution/board-admission.ts", - "evals/ooo-execution/patch-verifier.ts", - "evals/ooo-execution/check-events.test.ts", - "evals/ooo-execution/patch-cycle.test.ts", - ], - }, - // Only the kind each task's rule can admit. B must return files (a proven gap cannot - // be answered with prose or with a promotion), C must promote the composed candidate. - admitted: { - A: ["no-change-needed", "cannot-complete"], - B: ["cannot-complete"], - C: ["promote-candidate", "cannot-complete"], - }, - aEditable: ["src/integration/ooo-check.ts"], - bEditable: ["evals/ooo-execution/check-events.test.ts"], - budget: { perFile: 24_000, output: 48_000 }, - limits: { turns: 10, reads: 6, timeoutMs: 240_000 }, - aInstruction: - "You are the repair task of an out-of-order development round. The editable file owns the " + - "external-check protocol: host-issued check identity, terminal evidence, fencing and expiry. " + - "A fixed regression check has run against the frozen revision; its outcome is stated below. " + - "If it exposed a real defect inside your editable file, return a patch fixing exactly that. " + - "If nothing in your editable file is at fault, return a no-change conclusion citing the exact " + - "existing test title that already establishes check identity. Do not change behavior the check " + - "does not justify.", - bInstruction: - "You are the independent regression task of an out-of-order development round. Frozen here are " + - "the check protocol, the coordination board and the tests that exercise them. The host has " + - "already proved, by mutation, that the frozen suite does not detect the faults listed below; " + - "each is stated as the exact code change that introduces it. Add regression tests to the " + - "editable test file so that the suite now detects them. The host accepts your patch only when " + - "it passes on the intact implementation and fails on at least one listed mutation; a test that " + - "passes in both cases proves nothing and is rejected. Closing one declared fault is enough: " + - "pick the one you can prove, and keep the added tests few and exact rather than covering all " + - "five, which is how the previous attempt overran its output budget and was recorded as a " + - "failed attempt. A no-change conclusion is also rejected, " + - "because the host's own evidence already shows a gap. Change only that test file.", - worker: replayMode - ? replayWorker - : async (_taskId, frozen) => { - const run = await executePiPatch(frozen, provider!, model!, { check: checkTool, pushback }); - runs.push(run.checks); - const metrics = { - tokens: run.tokens, - turns: run.turns, - checks: run.checks, - cacheRead: run.cacheRead, - cacheWrite: run.cacheWrite, - }; - if (run.pushback) return { artifact: "", pushback: run.pushback, metrics }; - return { artifact: run.artifact, metrics }; - }, -}); - -/** In replay mode the round's own vocabulary is not enough: the question is whether the - * re-derived terminal state matches what the log recorded — and whether this is the same - * round at all. A replay handed different frozen work is a different round, which can finish - * with the same verdicts by coincidence; that case must be named, not reported as a - * reproduction. */ -if (replayMode) { - const recorded = terminalEvent(recordedEvents); - if (!recorded) throw new Error(`${logPath} has no terminal event`); - const plan = recordedPlan(recordedEvents); - const foreign = compareFrozen(recordedEvents, result.log); - const unverified = plan?.revision - ? [] - : [`the log records no revision: this replay is not bound to a round identity`]; - const differences = [...unverified, ...foreign, ...compareTerminal(recorded, result)]; - console.log( - foreign.length - ? `replay refused: the frozen work differs from the log, so this is a different round:\n` + - ` - ${foreign.join("\n - ")}` - : differences.length - ? `replay diverged from the log:\n - ${differences.join("\n - ")}` - : `replay reproduced the round: ${recorded.composed.files.length} composed file(s), ` + - `${Object.keys(recorded.accepted).length} accepted task(s), no model call, ` + - `revision ${plan!.revision!.slice(0, 12)}`, - ); - if (differences.length) process.exitCode = 1; -} - -const report = { - finishedAt: new Date().toISOString(), - provider, - model, - revision, - checks: checks.map((check) => check.label), - caseRules: cases, - mutants: mutants.map((mutation) => ({ - id: mutation.id, - killed: result.killed.B?.includes(mutation.id) ?? false, - survived: result.survived.B?.includes(mutation.id) ?? false, - })), - verdicts: result.verdicts, - composed: result.composed, - measurements: result.measurements, - // How often the worker verified its own proposal before answering. - workerCheckRuns: runs, - timeline: result.timeline, - rejections: result.rejections.map((entry) => ({ - ...entry, - artifact: entry.artifact.slice(0, 600), - })), - submissions: Object.fromEntries( - Object.entries(result.submissions).map(([task, submission]) => [ - task, - submission.kind === "patch" - ? { - kind: "patch", - files: Object.keys(submission.files).filter( - (path) => baseline[path] !== submission.files[path], - ), - } - : { - kind: "conclusion", - conclusion: submission.conclusion, - citations: submission.citations, - }, - ]), - ), -}; -mkdirSync(".nmg/ooo-live", { recursive: true }); -// An accepted candidate is the round's product: keep the changed files, otherwise a -// useful result exists only inside the process that produced it. The changed files are -// also written out as real files, so promotion is a reviewable copy and a stale -// hand-written "candidate" can never be mistaken for a round's product. -for (const [task, submission] of Object.entries(result.submissions)) { - if (submission.kind !== "patch") continue; - const changed = Object.fromEntries( - Object.entries(submission.files).filter(([path, text]) => baseline[path] !== text), - ); - writeFileSync( - `.nmg/ooo-live/accepted-${task}.json`, - JSON.stringify({ task, changed, mutantKills: result.killed[task] ?? [] }, null, 2) + "\n", - ); - for (const [path, text] of Object.entries(changed)) { - const target = join(".nmg/ooo-live/candidate", path); - mkdirSync(dirname(target), { recursive: true }); - writeFileSync(target, text); - } -} -// The mutated text is kept so a later reader can reproduce the round's premise. -writeFileSync( - ".nmg/ooo-live/mutants.json", - JSON.stringify( - mutants.map((mutation) => ({ - ...mutation, - mutated: mutate(baseline, mutation)[mutation.path], - })), - null, - 2, - ) + "\n", -); -writeFileSync(".nmg/ooo-live/cycle.json", JSON.stringify(report, null, 2) + "\n"); -// Rejected artifacts are kept in full (still bounded), because the reason a round -// failed is often visible only in the text the contract refused. -writeFileSync( - ".nmg/ooo-live/rejections.json", - JSON.stringify( - result.rejections.map((entry) => ({ ...entry, caseRules: cases })), - null, - 2, - ) + "\n", -); -console.log(JSON.stringify(report, null, 2)); -if (result.verdicts.A !== "accepted" || result.verdicts.B !== "accepted") - throw new Error("round did not accept A and B"); diff --git a/evals/ooo-execution/live-patch.ts b/evals/ooo-execution/live-patch.ts index 3e861561..cf3169c7 100644 --- a/evals/ooo-execution/live-patch.ts +++ b/evals/ooo-execution/live-patch.ts @@ -2,10 +2,11 @@ // Requires PI_PROVIDER and PI_MODEL. Shared source files are never modified. // Host verifies the exact rename and parses it in a disposable candidate directory. // This is S0 evidence only: check-event admission and A/B/C integration remain open. -import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; +import { writeFileSync, mkdirSync } from "node:fs"; import { patchCandidate, preparePatchWork } from "../../src/integration/ooo-patch.ts"; import { executePiPatch } from "../../.pi/extensions/nmg/ooo-execution.ts"; -import { expectedRename, verifyRenameCandidate } from "../../src/integration/ooo-verifier.ts"; +import { verifyRenameCandidate } from "../../src/integration/check-runner.ts"; +import { expectedRenameOf, RENAME_TARGET, renameSource } from "./rename-probe.ts"; import { randomUUID } from "node:crypto"; const provider = process.env.PI_PROVIDER; @@ -14,9 +15,11 @@ if (!process.argv.includes("--live")) throw new Error("pass --live explicitly: this calls the configured model"); if (!provider || !model) throw new Error("Set PI_PROVIDER and PI_MODEL explicitly"); -const path = "src/integration/ooo-execution.ts"; -const source = readFileSync(path, "utf8"); -const expected = expectedRename(source); +// The probe's frozen target, not a live product file: the candidate is the whole file and a +// dependent task carries it inside a snapshot, where the shared work contract bounds it. +const path = RENAME_TARGET; +const source = renameSource(); +const expected = expectedRenameOf(source); const startedAt = new Date().toISOString(); const frozen = preparePatchWork({ taskId: `s0-patch-probe:${randomUUID()}`, diff --git a/evals/ooo-execution/mutation-probe.ts b/evals/ooo-execution/mutation-probe.ts index 7e616162..0ea63060 100644 --- a/evals/ooo-execution/mutation-probe.ts +++ b/evals/ooo-execution/mutation-probe.ts @@ -8,7 +8,7 @@ import { mutate, type Mutation } from "../../src/integration/ooo-mutation.ts"; const repository = process.cwd(); const revision = process.env.MUTATION_REVISION ?? "HEAD"; -const impl = "src/integration/ooo-check.ts"; +const impl = "src/integration/check-ticket.ts"; const defaultPaths = [ impl, "evals/ooo-execution/check-events.test.ts", diff --git a/evals/ooo-execution/narrow-dispatch.test.ts b/evals/ooo-execution/narrow-dispatch.test.ts index 04ae5b9f..64f97d3b 100644 --- a/evals/ooo-execution/narrow-dispatch.test.ts +++ b/evals/ooo-execution/narrow-dispatch.test.ts @@ -2,8 +2,10 @@ import assert from "node:assert/strict"; import test from "node:test"; import { nextTask, + selectableTasks, snapshotAnswer, snapshotPrompt, + startableTasks, type DispatchTask, } from "../../src/integration/ooo-execution.ts"; @@ -74,3 +76,56 @@ test("unknown/shared effects and stale inputs are ineligible; changes invalidate ); assert.throws(() => nextTask([task("A"), task("A")]), /duplicate/); }); + +test("a declared budget is spent by claims in flight, not by the next task's rank", () => { + const plan = [task("A"), task("B"), task("C")]; + // One claim in flight: a two-slot run still has room, and the claimed task itself is not on offer. + assert.deepEqual( + startableTasks([task("A", { claimed: true }), task("B"), task("C")], 2), + ["B"], + "a claimed task is not a candidate, and it spends exactly one slot", + ); + assert.deepEqual( + selectableTasks([task("A", { claimed: true }), task("B"), task("C")], 2), + ["B", "C"], + "the legal set is what a source may rank; the budget cuts the start, not the set", + ); + // Budget spent: nothing is startable, however ready the rest of the plan is. + assert.deepEqual( + startableTasks([task("A", { claimed: true }), task("B", { claimed: true }), task("C")], 2), + [], + "two claims in flight spend a two-slot budget", + ); + assert.deepEqual( + selectableTasks([task("A", { claimed: true }), task("B", { claimed: true }), task("C")], 2), + [], + "a spent budget is the rule's own answer, not the caller's subtraction", + ); + // The cut happens after ordering: the plan's third task is startable only when the budget pays for it. + assert.deepEqual(startableTasks(plan, 2), ["A", "B"]); + assert.deepEqual(startableTasks(plan, 1), ["A"], "one slot is the default and the history"); + assert.deepEqual(selectableTasks(plan, 2), ["A", "B", "C"], "the ordered legal set is unchanged"); + assert.equal(nextTask(plan, 2), "A", "the head is the head whatever the budget"); +}); + +test("a claim in flight does not release a dependent, and half a slot is not a budget", () => { + const dependent = [task("A", { claimed: true }), task("B", { dependencies: ["A"] })]; + assert.deepEqual( + selectableTasks(dependent, 4), + [], + "B's dependency is unaccepted while A is in flight, so no budget makes B selectable", + ); + // An external wait ahead of it still licenses the tasks after it - at any budget. + const behindWait = [waiting(), task("B", { claimed: true }), task("C")]; + assert.deepEqual(startableTasks(behindWait, 2), ["C"]); + assert.deepEqual( + startableTasks(behindWait, 3), + ["C"], + "B is claimed, not startable, at any budget", + ); + for (const slots of [0, -1, 1.5, Number.NaN]) { + assert.throws(() => selectableTasks([task("A")], slots), /slots/, `slots=${slots}`); + assert.throws(() => startableTasks([task("A")], slots), /slots/, `slots=${slots}`); + assert.throws(() => nextTask([task("A")], slots), /slots/, `slots=${slots}`); + } +}); diff --git a/evals/ooo-execution/patch-cycle.test.ts b/evals/ooo-execution/patch-cycle.test.ts index 5f827ae3..841bfe00 100644 --- a/evals/ooo-execution/patch-cycle.test.ts +++ b/evals/ooo-execution/patch-cycle.test.ts @@ -1,22 +1,19 @@ import assert from "node:assert/strict"; import test, { type TestContext } from "node:test"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { BoardAdmission, - channel, type PatchTaskSpec, type ProbePlan, } from "../../src/integration/ooo-board.ts"; -import { expectedRename } from "../../src/integration/ooo-verifier.ts"; +import { expectedRenameOf, RENAME_TARGET, renameSource } from "./rename-probe.ts"; import type { PatchSubmission } from "../../src/integration/ooo-patch.ts"; -const source = readFileSync( - new URL("../../src/integration/ooo-execution.ts", import.meta.url), - "utf8", -); -const expected = expectedRename(source); +const TARGET = RENAME_TARGET; +const source = renameSource(); +const expected = expectedRenameOf(source); const proposal = () => JSON.stringify({ digest: "", files: [] }); /** Host check: an exact-rename patch, or a conclusion. Conclusions are admitted @@ -24,14 +21,11 @@ const proposal = () => JSON.stringify({ digest: "", files: [] }); async function verifyRename(submission: PatchSubmission) { if (submission.kind === "conclusion") return submission.evidence ? "accept" : "reject"; const candidate = submission.files; - if (candidate["src/integration/ooo-execution.ts"] !== expected) return "reject" as const; + if (candidate[TARGET] !== expected) return "reject" as const; const directory = mkdtempSync(join(tmpdir(), "ooo-candidate-")); try { - mkdirSync(join(directory, "src/integration"), { recursive: true }); - writeFileSync( - join(directory, "src/integration/ooo-execution.ts"), - candidate["src/integration/ooo-execution.ts"]!, - ); + mkdirSync(join(directory, dirname(TARGET)), { recursive: true }); + writeFileSync(join(directory, TARGET), candidate[TARGET]!); return "accept" as const; } finally { rmSync(directory, { recursive: true, force: true }); @@ -44,12 +38,12 @@ const plan: ProbePlan = [ ]; function fixture(t: TestContext, verify: PatchTaskSpec["verify"] = verifyRename) { - const dir = mkdtempSync(join(tmpdir(), "ooo-cycle-")); + const dir = mkdtempSync(join(tmpdir(), "ooo-patch-cycle-")); const gate = new BoardAdmission(join(dir, "store.sqlite"), plan, { P: { instruction: "Rename byId to planIndex in nextTask only.", - files: { "src/integration/ooo-execution.ts": source }, - editable: ["src/integration/ooo-execution.ts"], + files: { [TARGET]: source }, + editable: [TARGET], verify, }, }); @@ -66,7 +60,7 @@ function submitPatch( artifact: string, ) { const entry = gate.putTaskBoardEntry({ - taskId: "ooo-process-probe", + taskId: gate.channel, agentId: ticket.owner, kind: "result", content: JSON.stringify({ ticket, artifact }), @@ -80,14 +74,14 @@ test("contract: a verified patch candidate is what dependents bind to, and only const ticket = gate.claim("P", "worker-p"); const artifact = JSON.stringify({ digest: ticket.patch!.digest, - files: [{ path: "src/integration/ooo-execution.ts", content: expected }], + files: [{ path: TARGET, content: expected }], }); assert.deepEqual(gate.accepted(), {}); assert.equal(await submitPatch(gate, ticket, artifact), "accepted"); assert.deepEqual(Object.keys(gate.accepted()), ["P"]); assert.deepEqual(JSON.parse(gate.accepted().P!), { kind: "patch", - files: { "src/integration/ooo-execution.ts": expected }, + files: { [TARGET]: expected }, }); const dependent = gate.claim("D", "worker-d"); assert.deepEqual(dependent.dependencies, { P: gate.accepted().P! }); @@ -100,11 +94,11 @@ test("acceptance lands on the board as a deliverable and an outside verdict, not const gate = fixture(t); const ticket = gate.claim("P", "worker-p"); const entryId = gate - .readTaskBoard({ taskId: channel }) + .readTaskBoard({ taskId: gate.channel }) .entries.find((entry) => entry.claimedBy === "worker-p")!.id; const artifact = JSON.stringify({ digest: ticket.patch!.digest, - files: [{ path: "src/integration/ooo-execution.ts", content: expected }], + files: [{ path: TARGET, content: expected }], }); assert.equal(await submitPatch(gate, ticket, artifact), "accepted"); @@ -112,7 +106,7 @@ test("acceptance lands on the board as a deliverable and an outside verdict, not // "coordinator") — a self-report. What must be true now is that the artifact // is recorded against the claim that produced it and that someone other than // its producer judged it. - const judged = gate.getTaskBoardEntryById(channel, entryId)!; + const judged = gate.getTaskBoardEntryById(gate.channel, entryId)!; assert.equal(judged.deliveredBy, "worker-p"); assert.equal(judged.verdict, "accepted"); assert.equal(judged.judgedBy, "coordinator"); @@ -126,11 +120,11 @@ test("the board verdict is what accepts an artifact, not the round's own column" const gate = fixture(t); const ticket = gate.claim("P", "worker-p"); const entryId = gate - .readTaskBoard({ taskId: channel }) + .readTaskBoard({ taskId: gate.channel }) .entries.find((entry) => entry.claimedBy === "worker-p")!.id; const artifact = JSON.stringify({ digest: ticket.patch!.digest, - files: [{ path: "src/integration/ooo-execution.ts", content: expected }], + files: [{ path: TARGET, content: expected }], }); assert.equal(await submitPatch(gate, ticket, artifact), "accepted"); assert.deepEqual(Object.keys(gate.accepted()), ["P"]); @@ -139,7 +133,7 @@ test("the board verdict is what accepts an artifact, not the round's own column" // must follow the verdict, not the round's own row: the value is still stored there, // and it must stop counting as accepted. gate.judgeTaskBoardEntry({ - taskId: channel, + taskId: gate.channel, entryId, agentId: "auditor", verdict: "rejected", @@ -156,7 +150,7 @@ test("cancellation is announced on the board, not only in the round's own store" // The terminal decision has to be visible to agents that were not the caller, which is // what makes a cross-process cancel work at all. const announcement = gate - .readTaskBoard({ taskId: channel, includeResolved: true }) + .readTaskBoard({ taskId: gate.channel, includeResolved: true }) .entries.find( (entry) => entry.kind === "decision" && entry.content.includes("cancel: operator stopped the round"), @@ -168,9 +162,9 @@ test("cancellation is announced on the board, not only in the round's own store" test("safety: worker-supplied approval is ignored and a reissued attempt fences the old artifact", async (t) => { const gate = fixture(t, async () => "accept"); - const files = (content: string) => [{ path: "src/integration/ooo-execution.ts", content }]; + const files = (content: string) => [{ path: TARGET, content }]; const first = gate.claim("P", "worker-p"); - // Extra fields are not a verdict channel: the artifact shape must be exact. + // Extra fields are not a verdict gate.channel: the artifact shape must be exact. const selfApproved = JSON.stringify({ digest: first.patch!.digest, files: files(expected), @@ -200,13 +194,13 @@ test("safety: a rejected proposal never accepts worker text and the same attempt assert.equal(gate.next(), null); const corrected = JSON.stringify({ digest: first.patch!.digest, - files: [{ path: "src/integration/ooo-execution.ts", content: expected }], + files: [{ path: TARGET, content: expected }], }); assert.equal(await submitPatch(gate, first, corrected), "accepted"); assert.equal(await submitPatch(gate, first, corrected), "duplicate"); assert.deepEqual(JSON.parse(gate.accepted().P!), { kind: "patch", - files: { "src/integration/ooo-execution.ts": expected }, + files: { [TARGET]: expected }, }); }); @@ -245,7 +239,7 @@ test("safety: a host check that throws or is undecidable cannot accept a candida const ticket = gate.claim("P", "worker-p"); const artifact = JSON.stringify({ digest: ticket.patch!.digest, - files: [{ path: "src/integration/ooo-execution.ts", content: expected }], + files: [{ path: TARGET, content: expected }], }); assert.equal(await submitPatch(gate, ticket, artifact), "rejected"); assert.deepEqual(gate.accepted(), {}); @@ -255,17 +249,17 @@ test("an outside rejection withdraws the release of a dependent, and the round f const gate = fixture(t); const ticket = gate.claim("P", "worker-p"); const entryId = gate - .readTaskBoard({ taskId: channel }) + .readTaskBoard({ taskId: gate.channel }) .entries.find((entry) => entry.claimedBy === "worker-p")!.id; const artifact = JSON.stringify({ digest: ticket.patch!.digest, - files: [{ path: "src/integration/ooo-execution.ts", content: expected }], + files: [{ path: TARGET, content: expected }], }); assert.equal(await submitPatch(gate, ticket, artifact), "accepted"); assert.equal(gate.next(), "D", "the dependent is selected while P's artifact is accepted"); gate.judgeTaskBoardEntry({ - taskId: channel, + taskId: gate.channel, entryId, agentId: "auditor", verdict: "rejected", @@ -295,14 +289,14 @@ test("acceptance survives the entry's own TTL, because the round retains what it const ticket = gate.claim("P", "worker-p"); const artifact = JSON.stringify({ digest: ticket.patch!.digest, - files: [{ path: "src/integration/ooo-execution.ts", content: expected }], + files: [{ path: TARGET, content: expected }], }); assert.equal(await submitPatch(gate, ticket, artifact), "accepted"); const entryId = gate - .readTaskBoard({ taskId: channel, includeResolved: true }) + .readTaskBoard({ taskId: gate.channel, includeResolved: true }) .entries.find((entry) => entry.deliveredBy === "worker-p")!.id; assert.ok( - gate.listTaskBoardRetentions({ taskId: channel }).length > 0, + gate.listTaskBoardRetentions({ taskId: gate.channel }).length > 0, "publishing a handoff pins it: the round references an entry whose own TTL is 24h", ); @@ -311,7 +305,7 @@ test("acceptance survives the entry's own TTL, because the round retains what it // it is that the entry this run derives from is still there and still says accepted. gate.pruneExpiredTaskBoardEntries("2099-01-01T00:00:00.000Z"); assert.ok( - gate.getTaskBoardEntryById(channel, entryId), + gate.getTaskBoardEntryById(gate.channel, entryId), "the referenced entry survives its own expiry", ); assert.deepEqual( @@ -325,13 +319,13 @@ test("acceptance survives the entry's own TTL, because the round retains what it // prunable again: retention defers the prune, it does not exempt the entry from it. gate.reopen("P", "the artifact is no longer wanted"); assert.deepEqual( - gate.listTaskBoardRetentions({ taskId: channel }).filter((row) => row.entryId === entryId), + gate.listTaskBoardRetentions({ taskId: gate.channel }).filter((row) => row.entryId === entryId), [], "the pin on the entry this verdict lived in is gone (D's own handoff pin is not P's business)", ); gate.pruneExpiredTaskBoardEntries("2099-01-01T00:00:00.000Z"); assert.equal( - gate.getTaskBoardEntryById(channel, entryId), + gate.getTaskBoardEntryById(gate.channel, entryId), null, "with the pin gone the expired entry is finally pruned", ); @@ -342,11 +336,11 @@ test("cancelling a round releases the pins it held, so nothing it referenced lea const ticket = gate.claim("P", "worker-p"); const artifact = JSON.stringify({ digest: ticket.patch!.digest, - files: [{ path: "src/integration/ooo-execution.ts", content: expected }], + files: [{ path: TARGET, content: expected }], }); assert.equal(await submitPatch(gate, ticket, artifact), "accepted"); assert.ok( - gate.listTaskBoardRetentions({ taskId: channel }).length > 0, + gate.listTaskBoardRetentions({ taskId: gate.channel }).length > 0, "the round holds pins while it is running", ); @@ -354,5 +348,5 @@ test("cancelling a round releases the pins it held, so nothing it referenced lea // the handoff published for the successor, and the entry whose verdict accepted P. A // leaked pin would keep an entry alive forever in a round nobody is running. gate.cancel("operator stopped the round"); - assert.deepEqual(gate.listTaskBoardRetentions({ taskId: channel }), []); + assert.deepEqual(gate.listTaskBoardRetentions({ taskId: gate.channel }), []); }); diff --git a/evals/ooo-execution/pilot.ts b/evals/ooo-execution/pilot.ts new file mode 100644 index 00000000..d05bcf97 --- /dev/null +++ b/evals/ooo-execution/pilot.ts @@ -0,0 +1,270 @@ +// The paid pilot: the same frozen work as one unit, as four units at one slot, and as four units at +// two slots, run against the configured model, with the arm order randomly drawn and every run +// recorded, including the ones that failed. +// +// What it measures is F1's three expectations on a real worker: whether a finer plan costs more wall +// time in the host, whether a slot actually buys overlap once a model - not a stub - is the worker, +// and whether the arms reach the same accepted work. The sample is small by construction and is +// reported as such; a difference the sample cannot resolve is a difference this script does not claim. +// +// Usage: +// node --experimental-strip-types evals/ooo-execution/pilot.ts --live \ +// --out [--family pipeline] [--reps 3,3,2] [--seed 1] [--runs-dir ] +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { piWorker, runPlan, specFrom, type SpecFile } from "./plan-driver.ts"; + +const USAGE = + "usage: pilot.ts --live --out [--family ] [--reps A,B,C] [--seed ] " + + "[--runs-dir ]\n" + + " --live required: this calls the configured model (" + + "PI_PROVIDER/PI_MODEL) and spends tokens\n" + + " --family the fixture directory under evals/ooo-execution/fixtures (default: pipeline)\n" + + " --reps runs per arm, in the order coarse,fine-1-slot,fine-2-slots (default: 3,3,2)\n" + + " --seed seed for the arm order draw, so the order is reproducible (default: 1)\n" + + " --runs-dir where each run's own result is written (default: --out's directory + /runs)\n" + + " --report re-aggregate result files already recorded (comma-separated); no model call"; + +/** The pilot fixes the envelope's limits and never the arms' variables: the first model turn comes + * back aborted often enough that the host default of three turns spends the attempt on it. */ +const PILOT_LIMITS = { turns: 6, reads: 3, timeoutMs: 120_000 } as const; + +function flags(argv: readonly string[]): Record { + const parsed: Record = {}; + for (let index = 0; index < argv.length; index += 1) { + const flag = argv[index]!; + if (!flag.startsWith("--")) continue; + const value = argv[index + 1]; + // A flag with no value of its own is a switch: `--live --out x` must not read `--out` as one. + if (value === undefined || value.startsWith("--")) { + parsed[flag.slice(2)] = "true"; + continue; + } + parsed[flag.slice(2)] = value; + index += 1; + } + return parsed; +} + +/** A deterministic draw, so a reader can re-run the same arm order. */ +function shuffled(items: readonly T[], seed: number): T[] { + const out = [...items]; + let state = seed >>> 0 || 1; + for (let index = out.length - 1; index > 0; index -= 1) { + state = (state * 1_664_525 + 1_013_904_223) >>> 0; + const swap = state % (index + 1); + [out[index], out[swap]] = [out[swap]!, out[index]!]; + } + return out; +} + +const values = flags(process.argv.slice(2)); +const out = values["out"]; +if (!out) throw new Error(`--out is required\n${USAGE}`); +const family = values["family"] ?? "pipeline"; + +/** `--report a.json,b.json` re-derives the aggregate from runs already recorded - no model call, and + * the only way to combine two sittings. The runs are the evidence; this refuses a set that was not + * produced by the same instrument. */ +const reportOnly = values["report"] + ? [values["report"]] + .flatMap((paths) => paths.split(",")) + .map((path) => path.trim()) + .filter((path) => path.length > 0) + .map((path) => resolve(path)) + : undefined; +if (reportOnly && !reportOnly.length) throw new Error(`--report needs at least one file\n${USAGE}`); +if (!reportOnly && !process.argv.includes("--live")) + throw new Error(`refusing a paid run without --live\n${USAGE}`); +const provider = reportOnly ? "" : (process.env.PI_PROVIDER ?? ""); +const model = reportOnly ? "" : (process.env.PI_MODEL ?? ""); +if (!reportOnly && (!provider || !model)) + throw new Error( + "Set PI_PROVIDER and PI_MODEL: the pilot's model is an input, not a default\n" + USAGE, + ); +const reps = (values["reps"] ?? "3,3,2") + .split(",") + .map((value) => Number(value.trim())); +if (reps.length !== 3 || reps.some((count) => !Number.isSafeInteger(count) || count < 1)) + throw new Error(`--reps wants three positive integers, one per arm\n${USAGE}`); +const seed = Number(values["seed"] ?? 1); +if (!Number.isSafeInteger(seed)) throw new Error(`--seed wants an integer\n${USAGE}`); +const runsDir = values["runs-dir"] ?? resolve(out, "..", "runs"); +mkdirSync(runsDir, { recursive: true }); + +const directory = resolve("evals/ooo-execution/fixtures", family); +const read = (name: string): SpecFile => { + const file = JSON.parse(readFileSync(resolve(directory, name), "utf8")) as SpecFile; + if (file.worker.kind !== "canned") + throw new Error(`${name}: the pilot supplies the worker, so the spec must not name one`); + return { ...file, worker: { kind: "pi", provider, model }, limits: { ...PILOT_LIMITS } }; +}; + +/** A = the whole task in one unit, B = the same work in four units, C = B with two slots. */ +const ARMS = [ + { arm: "A", plan: "coarse (1 unit)", slots: 1, spec: "coarse.spec.json" }, + { arm: "B", plan: "fine (4 units)", slots: 1, spec: "fine.spec.json" }, + { arm: "C", plan: "fine (4 units)", slots: 2, spec: "fine.spec.json" }, +] as const; +const specs = new Map( + [...new Set(ARMS.map((arm) => arm.spec))].map((name) => [name, read(name)]), +); + +const schedule = shuffled( + ARMS.flatMap((arm, index) => + Array.from({ length: reps[index]! }, (_, rep) => ({ ...arm, rep: rep + 1 })), + ), + seed, +); + +interface Recorded { + arm: string; + plan: string; + rep: number; + slotsRequested: number; + slotsUsed: number; + slotRefusal?: string; + wallMs: number; + hostMs: number; + tokens: number; + units: number; + accepted: number; + parent?: string; + verdicts: Record; + incomplete: readonly string[]; +} + +const recorded: Recorded[] = []; +let recordedProvider = provider; +let recordedModel = model; +let recordedFamily = family; +if (reportOnly) { + const previous = reportOnly.map( + (file) => + JSON.parse(readFileSync(file, "utf8")) as { + provider: string; + model: string; + family: string; + limits: unknown; + runs: Recorded[]; + }, + ); + const first = previous[0]!; + for (const [index, file] of reportOnly.entries()) { + const one = previous[index]!; + if ( + one.provider !== first.provider || + one.model !== first.model || + one.family !== first.family || + JSON.stringify(one.limits) !== JSON.stringify(first.limits) + ) + throw new Error( + `${file}: recorded by ${one.provider}/${one.model} on ${one.family} with other limits, ` + + `while ${reportOnly[0]} used ${first.provider}/${first.model}; merging them would put two ` + + "instruments in one table", + ); + recorded.push(...one.runs); + } + if (!recorded.length) throw new Error("the result files hold no runs"); + if (values["family"] !== undefined && values["family"] !== first.family) + throw new Error( + `--family ${values["family"]} does not match the recorded family ${first.family}`, + ); + recordedProvider = first.provider; + recordedModel = first.model; + recordedFamily = first.family; +} else await runArms(); + +/** The per-arm view of a set of runs. Kept separate from the running so the same aggregation can be + * re-derived from runs that were recorded in more than one sitting - the evidence is the runs, and + * an aggregate that cannot be recomputed from them is not evidence. */ +function aggregate(runs: readonly Recorded[], planned: readonly number[]) { + return ARMS.map((arm, index) => { + const own = runs.filter((entry) => entry.arm === arm.arm); + const wall = own.map((entry) => entry.wallMs).sort((left, right) => left - right); + return { + arm: arm.arm, + plan: arm.plan, + slots: arm.slots, + planned: planned[index] ?? own.length, + runs: own.length, + acceptedUnits: own.map((entry) => entry.accepted), + parents: own.map((entry) => entry.parent ?? null), + slotsUsed: own.map((entry) => entry.slotsUsed), + completeRuns: own.filter((entry) => entry.incomplete.length === 0).length, + wallMsTotal: wall.reduce((sum, ms) => sum + ms, 0), + wallMsMedian: wall.length ? wall[Math.floor(wall.length / 2)]! : null, + hostMsTotal: own.reduce((sum, entry) => sum + entry.hostMs, 0), + tokensTotal: own.reduce((sum, entry) => sum + entry.tokens, 0), + unitsTotal: own.reduce((sum, entry) => sum + entry.units, 0), + failures: own.flatMap((entry) => entry.incomplete), + }; + }); +} + +const report = { + measuredAt: new Date().toISOString(), + provider: recordedProvider, + model: recordedModel, + family: recordedFamily, + limits: PILOT_LIMITS, + reps: reportOnly ? [] : reps, + seed, + order: schedule.map((step) => `${step.arm}${step.rep}`), + mergedFrom: reportOnly, + arms: aggregate(recorded, reportOnly ? [] : reps), + runs: recorded, +}; +writeFileSync(out, `${JSON.stringify(report, null, 2)}\n`); +for (const arm of report.arms) + process.stdout.write( + `${arm.arm} ${arm.plan} @${arm.slots}: ${arm.runs} runs (${arm.completeRuns} complete), ` + + `accepted ${arm.acceptedUnits.join("/")}, parents ${arm.parents.join("/")}, ` + + `wall ${arm.wallMsTotal}ms (median ${String(arm.wallMsMedian)}), host ${arm.hostMsTotal}ms, ` + + `tokens ${arm.tokensTotal}, slots used ${arm.slotsUsed.join("/")}\n`, + ); +process.stdout.write(`results: ${out}${reportOnly ? "" : `\nper-run: ${runsDir}`}\n`); + +async function runArms(): Promise { + for (const [index, step] of schedule.entries()) { + process.stdout.write( + `[${index + 1}/${schedule.length}] arm ${step.arm} rep ${step.rep} (${step.plan}, ` + + `${step.slots} slot${step.slots === 1 ? "" : "s"})\n`, + ); + const startedAt = Date.now(); + const run = await runPlan( + specFrom(specs.get(step.spec)!, piWorker({ provider, model }, true), step.slots), + ); + const verdicts: Record = {}; + for (const unit of run.units) verdicts[unit.verdict] = (verdicts[unit.verdict] ?? 0) + 1; + const one: Recorded = { + arm: step.arm, + plan: step.plan, + rep: step.rep, + slotsRequested: run.slotsRequested, + slotsUsed: run.slotsUsed, + ...(run.slotRefusal ? { slotRefusal: run.slotRefusal } : {}), + wallMs: run.wallMs, + hostMs: run.hostMs, + tokens: run.tokens, + units: run.units.length, + accepted: Object.keys(run.accepted).length, + ...(run.parent ? { parent: run.parent.verdict } : {}), + verdicts, + incomplete: run.incomplete, + }; + recorded.push(one); + writeFileSync( + resolve(runsDir, `${step.arm}-${step.rep}-${startedAt}.json`), + `${JSON.stringify(one, null, 2)}\n`, + ); + process.stdout.write( + ` wall ${one.wallMs}ms, host ${one.hostMs}ms, tokens ${one.tokens}, slots ${one.slotsUsed}/` + + `${one.slotsRequested}, verdicts ${JSON.stringify(verdicts)}, parent ${String(one.parent)}` + + `${one.incomplete.length ? `, incomplete ${JSON.stringify(one.incomplete)}` : ""}\n`, + ); + } + if (recorded.length !== reps.reduce((sum, count) => sum + count, 0)) + throw new Error(`planned ${reps.join("+")} runs and recorded ${recorded.length}`); +} + diff --git a/evals/ooo-execution/plan-driver.test.ts b/evals/ooo-execution/plan-driver.test.ts new file mode 100644 index 00000000..ee4e930b --- /dev/null +++ b/evals/ooo-execution/plan-driver.test.ts @@ -0,0 +1,433 @@ +// The granularity driver's own properties, offline and deterministic. +// +// The arms compare two slot counts on the same plan, so the driver has exactly two jobs: start only +// what the rules allow, and start as many of those as the slot count says at once. The second job was +// once unreachable - the shared admission layer published a handoff only for the task it had selected, +// so a run could hold exactly one claim - and these cases pin the mechanism that made it reachable (a +// declared slot budget, with each handoff directed at its own claimant) from the driver's side. +import assert from "node:assert/strict"; +import test from "node:test"; +import type { ProbePlan } from "../../src/integration/ooo-board.ts"; +import { + comparePlanSlots, + piWorker, + runPlan, + specFrom, + type PlanDriverSpec, + type PlanWorker, +} from "./plan-driver.ts"; + +const baseline = { "src/unit.ts": "export const value = 1;\n" }; +const ok = [{ label: "unit check", command: process.execPath, args: ["-e", "process.exit(0)"] }]; +/** Four units that share a frozen interface, and a summary over three of them: the shape the arms + * use, and one whose independence is real rather than a relabelling of test groups. */ +const plan: ProbePlan = [ + ["first", "", [], "isolated-artifact", null, null], + ["second", "", [], "isolated-artifact", null, null], + ["third", "", [], "isolated-artifact", null, null], + ["summary", "", ["first", "second", "third"], "isolated-artifact", null, null], +]; + +function spec(overrides: Partial = {}): PlanDriverSpec { + const units = Object.fromEntries( + plan.map((row) => [ + String(row[0]), + { instruction: `work on ${String(row[0])}`, editable: ["src/unit.ts"], checks: ok }, + ]), + ); + return { + plan, + units, + worker: recordingWorker(), + repository: process.cwd(), + revision: "HEAD", + baseline, + parentChecks: ok, + join: "summary", + slots: 4, + ...overrides, + }; +} + +/** A worker that takes a fixed time and records when it ran, so overlap is visible in the data and + * not only in a wall clock. It answers in the protocol's own patch shape: a list of whole-file + * replacements that must actually differ from the frozen input, which is what the store checks. */ +function recordingWorker(latencyMs = 60, log: string[] = []): PlanWorker { + return async (taskId, frozen) => { + log.push(`start:${taskId}`); + await new Promise((done) => setTimeout(done, latencyMs)); + return { + artifact: JSON.stringify({ + digest: frozen.digest, + files: frozen.work.editable.map((path) => ({ + path, + content: `${frozen.work.files[path] ?? ""}// ${taskId}\n`, + })), + }), + metrics: { tokens: 7, turns: 1, checks: 0 }, + }; + }; +} + +/** The session a fused run must actually reuse: this worker echoes the session it was handed, so a + * chain that only *looked* fused - a fresh session per unit - would show up as distinct ids. */ +function sessionWorker(latencyMs = 20): PlanWorker { + return async (taskId, frozen, dependencies, session) => { + const produced = await recordingWorker(latencyMs)(taskId, frozen, dependencies); + if (typeof produced === "string" || !session) return produced; + return { ...produced, metrics: { ...produced.metrics, sessionId: session.id } }; + }; +} + +test("a fused run runs several units in one session, each with its own ticket and verdict", async () => { + const run = await runPlan( + spec({ slots: 1, fusion: { unitsPerSession: 2 }, worker: sessionWorker(10) }), + ); + // One session of two units, then a yield boundary and a second session: `summary` becomes a + // candidate only once `third` is accepted, so the chain continues into it. + assert.deepEqual(run.sessions, [ + ["first", "second"], + ["third", "summary"], + ]); + for (const unit of run.units) + assert.equal( + unit.sessionId, + `session:${["first", "second"].includes(unit.taskId) ? "first" : "third"}`, + `${unit.taskId} must name the session it really ran in`, + ); + // Fusion changes the execution resource, never the acceptance facts: each unit keeps its own + // claim, verdict, attempt and token count. + assert.equal(run.units.length, 4); + assert.deepEqual( + run.units.map((unit) => unit.verdict), + Array(4).fill("accepted"), + ); + assert.equal(run.hostChecks, 4, "every fused unit still crosses the host boundary on its own"); + assert.deepEqual( + run.units.map((unit) => unit.attempt), + [1, 1, 1, 1], + ); +}); + +test("a fused session ends where the next unit needs another capability", async () => { + const run = await runPlan( + spec({ + slots: 1, + fusion: { unitsPerSession: 4, declarations: { third: { capability: "other" } } }, + worker: sessionWorker(10), + }), + ); + // `third` is not a legal successor of `second`, and `summary` is not one of `third`: the run keeps + // each of them in its own session rather than widening what one session may do. + assert.deepEqual(run.sessions, [["first", "second"], ["third"], ["summary"]]); +}); + +test("a fused chain stops at the declared bound and does not swallow the plan", async () => { + const run = await runPlan( + spec({ slots: 1, fusion: { unitsPerSession: 1 }, worker: sessionWorker(10) }), + ); + // A bound of one is the control arm: fusion is on, and every session is one unit. + assert.deepEqual(run.sessions, [["first"], ["second"], ["third"], ["summary"]]); +}); + +test("a unit with no verdict ends the session it was running in", async () => { + const worker: PlanWorker = async (taskId, frozen, dependencies, session) => { + if (taskId === "second") return { failure: "gave up on the second unit" }; + const produced = await sessionWorker(10)(taskId, frozen, dependencies, session); + return produced; + }; + const run = await runPlan(spec({ slots: 1, fusion: { unitsPerSession: 4 }, worker })); + assert.equal(run.failures, 1); + assert.ok(run.incomplete.some((entry) => entry.includes("second"))); + assert.deepEqual(run.sessions, [["first"]], "the session ended at the unit with no verdict"); + assert.equal(run.accepted["summary"], undefined, "the summary cannot be accepted on a failure"); +}); + +test("a fused session does not continue from a unit the host rejected", async () => { + // A verdict is not a failure: the unit ran, the host looked at it and refused. The session must end + // there all the same, because the next unit would be working on top of an unverified answer. + const bad = [{ label: "unit check", command: process.execPath, args: ["-e", "process.exit(1)"] }]; + const base = spec({ slots: 1, fusion: { unitsPerSession: 4 }, worker: sessionWorker(10) }); + const run = await runPlan({ + ...base, + units: { ...base.units, second: { ...base.units["second"]!, checks: bad } }, + }); + assert.equal(run.units.find((unit) => unit.taskId === "second")?.verdict, "rejected"); + assert.deepEqual( + run.sessions.find((chain) => chain.includes("second")), + ["first", "second"], + "the session ended at the rejected unit, not after it", + ); + assert.ok( + !run.units.some( + (unit) => + unit.taskId === "third" && + unit.sessionId === run.units.find((x) => x.taskId === "second")?.sessionId, + ), + `no unit may run on top of a rejected answer: ${JSON.stringify(run.sessions)}`, + ); +}); + +test("a worker that starts its own session is not reported as fusion", async () => { + // `recordingWorker` never echoes a session, which is what a worker that opens a fresh session per + // unit looks like from the driver's side. The run must report boundaries, not fusion. + const run = await runPlan( + spec({ slots: 1, fusion: { unitsPerSession: 4 }, worker: recordingWorker(10) }), + ); + assert.deepEqual(run.sessions, [["first"], ["second"], ["third"], ["summary"]]); + assert.ok( + run.units.every((unit) => unit.sessionId === undefined), + "no unit claims a session the worker never reported", + ); +}); + +test("the live worker refuses to continue a session it cannot hold, and spends nothing", async () => { + const worker = piWorker({ provider: "deepseek", model: "deepseek-v4-flash" }, true); + const result = await worker("second", {} as never, {}, { id: "session:first", units: ["first"] }); + assert.ok(typeof result !== "string"); + assert.match( + String(result.failure), + /cannot continue session session:first \(it has run first\)/, + "a continuation the harness cannot hold is refused by name, not answered with a new session", + ); +}); + +test("one slot runs the units in plan order, each to acceptance", async () => { + const run = await runPlan(spec({ slots: 1, worker: recordingWorker(20) })); + assert.deepEqual(run.order, ["first", "second", "third", "summary"]); + assert.deepEqual( + run.units.map((unit) => unit.verdict), + Array(4).fill("accepted"), + ); + assert.equal(run.slotsUsed, 1); + assert.equal(run.slotRefusal, undefined); + assert.equal(run.parent?.verdict, "accept", "the fixed parent check runs over the composition"); + assert.deepEqual(run.parent?.files, ["summary"], "the join unit stands for the parent's result"); + assert.equal(run.hostChecks, 4, "the host checks every candidate, in both arms"); + assert.deepEqual(run.incomplete, []); +}); + +test("a declared slot count is reached, and the claims overlap in time", async () => { + const log: string[] = []; + let inFlight = 0; + let peak = 0; + const inner = recordingWorker(60, log); + const worker: PlanWorker = async (taskId, frozen, dependencies) => { + inFlight += 1; + peak = Math.max(peak, inFlight); + try { + return await inner(taskId, frozen, dependencies); + } finally { + inFlight -= 1; + } + }; + const run = await runPlan(spec({ slots: 4, worker })); + assert.equal(run.slotsRequested, 4); + assert.equal( + run.slotsUsed, + 3, + "three units are independent, and the summary waits for all three", + ); + assert.equal(run.slotRefusal, undefined, "the board allowed every claim it was asked for"); + assert.equal(peak, 3, `the worker saw the claims overlap: ${log.join(",")}`); + assert.deepEqual( + run.units.map((unit) => unit.verdict), + Array(4).fill("accepted"), + "the same verdicts as the one-slot run, which is what makes the two arms one experiment", + ); + assert.deepEqual(run.incomplete, []); +}); + +test("a dependent unit waits for its dependencies and is never dispatched early", async () => { + const log: string[] = []; + const run = await runPlan(spec({ slots: 1, worker: recordingWorker(10, log) })); + assert.equal(run.order.at(-1), "summary", `dispatch order: ${run.order.join(",")}`); + assert.deepEqual(run.units.at(-1)?.taskId, "summary"); + assert.equal(run.failures, 0); +}); + +test("a failed worker is recorded as incomplete rather than silently skipped", async () => { + const worker: PlanWorker = async (taskId, frozen) => { + if (taskId === "second") return { failure: "stub: the model returned nothing" }; + return { + artifact: JSON.stringify({ + digest: frozen.digest, + files: [{ path: "src/unit.ts", content: `${frozen.work.files["src/unit.ts"]}// x\n` }], + }), + }; + }; + const run = await runPlan(spec({ slots: 1, worker })); + assert.equal(run.failures, 1); + assert.ok( + run.incomplete.some((entry) => entry.includes("second")), + `incomplete: ${JSON.stringify(run.incomplete)}`, + ); + assert.equal(run.accepted["summary"], undefined, "the summary cannot be accepted on a failure"); +}); + +test("the parent check is the composed acceptance, and a failing check is reported as such", async () => { + const failing = [ + { label: "parent check", command: process.execPath, args: ["-e", "process.exit(1)"] }, + ]; + const run = await runPlan(spec({ slots: 1, worker: recordingWorker(10), parentChecks: failing })); + assert.equal(run.parent?.verdict, "reject"); + assert.deepEqual( + run.units.map((unit) => unit.verdict), + Array(4).fill("accepted"), + "the units are still accepted; only the composed result fails", + ); +}); + +test("a comparison refuses a time verdict when the slot count or the quality differs", async () => { + const oneSlot = await comparePlanSlots(spec({ slots: 4, worker: recordingWorker(10) }), { + runs: 2, + }); + assert.deepEqual( + oneSlot.arms.map((arm) => arm.slots), + [1, 4], + ); + assert.equal(oneSlot.qualityParity, true, JSON.stringify(oneSlot.differences)); + assert.equal( + oneSlot.slotShortfalls.length, + 2, + "both four-slot runs reached three of four: the plan has three independent units", + ); + assert.equal( + oneSlot.comparable, + false, + "a fallback slot count makes the two arms one experiment", + ); + + const twoArms = await comparePlanSlots(spec({ slots: 1, worker: recordingWorker(10) }), { + runs: 1, + arms: [1], + }); + assert.equal(twoArms.comparable, true); + assert.deepEqual(twoArms.slotShortfalls, []); + + let calls = 0; + const flaky: PlanWorker = async (_taskId, frozen) => { + calls += 1; + if (calls === 1) return { failure: "stub: the first call fails" }; + return { + artifact: JSON.stringify({ + digest: frozen.digest, + files: [{ path: "src/unit.ts", content: `${frozen.work.files["src/unit.ts"]}// y\n` }], + }), + }; + }; + const unfair = await comparePlanSlots(spec({ slots: 1, worker: flaky }), { + runs: 1, + arms: [1, 2], + }); + assert.equal(unfair.qualityParity, false, "a lost unit has to show up as different verdicts"); + assert.equal(unfair.comparable, false); + assert.ok(unfair.differences.length > 0); +}); + +test("impossible input is refused rather than defaulted", async () => { + await assert.rejects(() => runPlan(spec({ slots: 0 })), /slots must be a positive integer/); + await assert.rejects( + () => runPlan(spec({ units: { ghost: { instruction: "x", editable: [], checks: ok } } })), + /not in the plan; the plan is the authority/, + ); + await assert.rejects( + () => comparePlanSlots(spec(), { runs: 0 }), + /runs must be a positive integer/, + ); +}); + +/** The out-of-order property the design is about, carried by this driver and not by roles: an + * independent unit's worker runs while another unit's check is still outstanding. The retired round + * (`docs/decisions/implemented/2026-09-18-retire-the-round-instrument.md`) was the only end-to-end + * carrier of it before this case; if the batch loop ever stops overlapping units, this fails. */ +test("a unit's check is outstanding while an independent unit's worker runs", async () => { + const slowCheckMs = 1500; + const windows: Record = {}; + const worker: PlanWorker = async (taskId, frozen) => { + const window = { start: Date.now(), end: 0 }; + windows[taskId] = window; + // The second unit's work is shorter than the first unit's check, so "inside it" is a fact about + // the dispatch policy rather than about the two durations. + if (taskId === "second") await new Promise((done) => setTimeout(done, 700)); + window.end = Date.now(); + return { + artifact: JSON.stringify({ + digest: frozen.digest, + files: frozen.work.editable.map((path) => ({ + path, + content: `${frozen.work.files[path] ?? ""}// ${taskId}\n`, + })), + }), + metrics: { tokens: 1, turns: 1, checks: 0 }, + }; + }; + const slow = [ + { + label: "slow", + command: process.execPath, + args: ["-e", `setTimeout(() => {}, ${slowCheckMs})`], + }, + ]; + const twoUnits: ProbePlan = [ + ["first", "", [], "isolated-artifact", null, null], + ["second", "", [], "isolated-artifact", null, null], + ]; + const run = await runPlan({ + plan: twoUnits, + units: { + first: { instruction: "work on first", editable: ["src/unit.ts"], checks: slow }, + second: { instruction: "work on second", editable: ["src/unit.ts"], checks: ok }, + }, + worker, + repository: process.cwd(), + revision: "HEAD", + baseline, + slots: 2, + }); + const first = run.units.find((unit) => unit.taskId === "first"); + assert.ok(first, "the first unit ran"); + assert.equal(run.slotsUsed, 2, `two slots were declared and used: ${run.slotRefusal ?? ""}`); + assert.ok( + first.hostMs >= slowCheckMs, + `the first unit's check is the slow one; measured ${first.hostMs} ms`, + ); + const firstWindow = windows.first!; + const secondWindow = windows.second!; + assert.ok( + secondWindow.start >= firstWindow.end && secondWindow.end <= firstWindow.end + first.hostMs, + "the second unit's worker must run entirely inside the first unit's check window: " + + `first=${firstWindow.start}-${firstWindow.end}, check=${first.hostMs}ms, second=${secondWindow.start}-${secondWindow.end}`, + ); + assert.equal(run.failures, 0, "both units are accepted, so the overlap is not a failure path"); +}); + +test("a spec file's fusion block reaches the run it describes", () => { + const target = "evals/ooo-execution/fixtures/report/alpha.ts"; + const file = { + baseline: [target], + plan: [{ id: "first", effect: "isolated-artifact" }], + units: { + first: { + instruction: "work on first", + editable: [target], + checks: [{ label: "ok", command: "node", args: ["-e", "process.exit(0)"] }], + }, + }, + worker: { kind: "stub" as const, latencyMs: 1 }, + fusion: { unitsPerSession: 2 }, + }; + const declared = specFrom(file, recordingWorker(), 1); + assert.deepEqual( + declared.fusion, + { unitsPerSession: 2 }, + "a spec that asked for fusion must not be run as the control arm", + ); + const without: Record = { ...file }; + delete without.fusion; + assert.equal( + specFrom(without, recordingWorker(), 1).fusion, + undefined, + "a spec that declared none has none: the two readings must not be the same run", + ); +}); diff --git a/evals/ooo-execution/plan-driver.ts b/evals/ooo-execution/plan-driver.ts new file mode 100644 index 00000000..bc91b502 --- /dev/null +++ b/evals/ooo-execution/plan-driver.ts @@ -0,0 +1,906 @@ +// The granularity arms' driver: run one legal plan, with a chosen number of execution slots. +// +// Why this driver exists at all: the arms need the same parent task at two granularities, which is +// its own driver and not a parameter of some other one. The decision, with the couplings measured +// behind it, is `docs/decisions/implemented/2026-09-17-arms-get-their-own-driver.md`; the round this +// header used to be written beside was retired in +// `docs/decisions/implemented/2026-09-18-retire-the-round-instrument.md`, which is also where the +// interleaving this driver carries (a unit's check outstanding while another unit works) is pinned. +// +// What it does **not** duplicate: the rules and the ordering. `BoardAdmission.candidates()` is the +// ordered legal set from the shared semantics, and this driver only decides how *many* of them to +// start at once: +// +// - `slots: 1` - one legal unit at a time, each to acceptance: the B arm. +// - `slots: N` - every legal unit at once, up to N in flight: the C arm. +// +// Everything else is the shared layer's: the claim/attempt/fact writes, the candidate verification a +// unit's own `verify` performs, the accepted-artifact identity a dependent binds to. The driver adds +// only the dispatch policy, the timing, and the parent check at the end. +// +// Usage: +// node --experimental-strip-types evals/ooo-execution/plan-driver.ts run --spec \ +// --slots --out [--live] +// node --experimental-strip-types evals/ooo-execution/plan-driver.ts compare --spec \ +// --out [--live] + +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { + BoardAdmission, + type PatchTaskSpec, + type ProbePlan, +} from "../../src/integration/ooo-board.ts"; +import { verifyCandidate } from "../../src/integration/ooo-candidate.ts"; +import { + preparePatchWork, + type FrozenPatchWork, + type PatchSubmission, +} from "../../src/integration/ooo-patch.ts"; +import type { CandidateCheck } from "../../src/integration/ooo-candidate.ts"; +import type { SessionPlan } from "../../src/integration/ooo-execution.ts"; +import { nextSessionMove } from "../../src/integration/ooo-fusion-plan.ts"; + +/** What a worker reports about its own run. The product's run surface records no worker metrics + * today, so this shape lives with its only consumer rather than in `src/`. */ +export type WorkerMetrics = { + tokens?: number; + turns?: number; + checks?: number; + cacheRead?: number; + cacheWrite?: number; + /** The session the worker actually ran this unit in. Omitted by a worker that has no session to + * report (the stub and canned arms), and the field a fused run is judged on. */ + sessionId?: string; +}; + +/** What a worker returns for one unit. A failure is a recorded attempt, not a crashed run. */ +export type PlanWorkerResult = + string | { artifact?: string; metrics?: WorkerMetrics; failure?: string }; + +export type PlanWorker = ( + taskId: string, + frozen: FrozenPatchWork, + dependencies: Readonly>, + session?: PlanSession, +) => Promise; + +/** The execution resource a fused run reuses across units. `id` names the session, and `units` is what + * it has already run - so a worker can refuse a continuation it cannot honour instead of quietly + * starting a new session and having the run call that fusion. */ +export interface PlanSession { + id: string; + units: readonly string[]; +} + +/** One unit of the plan: what it is asked for, what it may edit, and what its own candidate must + * pass. The last one is the unit's acceptance; the parent check is separate and fixed. */ +export interface PlanUnit { + instruction: string; + editable: readonly string[]; + visible?: readonly string[]; + checks: readonly CandidateCheck[]; +} + +export interface PlanDriverSpec { + plan: ProbePlan; + units: Readonly>; + worker: PlanWorker; + repository: string; + revision: string; + baseline: Readonly>; + /** The parent check: what the composed artifacts must pass, run once at the end. Omission means + * the arms are comparing cost only, which the report must say. */ + parentChecks?: readonly CandidateCheck[]; + /** The unit whose acceptance stands for the parent's composed result. Omission: every accepted + * unit contributes to the parent's files. */ + join?: string; + databasePath?: string; + slots: number; + /** Execution fusion: how many units one session may run in a row, and the host declarations that + * decide which of them may share one. Declared, not derived - the runtime's policy is short ready + * chains, so this bound is what keeps a fused run from swallowing the plan. Omitted: no fusion. */ + fusion?: { + unitsPerSession: number; + /** Per-unit session declarations. Omitted: every unit shares the run's one capability and + * authority, and only its own visibility decides. */ + declarations?: Readonly>; + }; + budget?: { perFile: number; output: number }; + limits?: { turns: number; reads: number; timeoutMs: number }; +} + +export interface UnitRun { + taskId: string; + verdict: string; + /** Claim to return: the worker's own time, which is what parallelism can overlap. */ + workerMs: number; + /** The host's check for this unit's candidate. Host time is serial in every arm. */ + hostMs: number; + tokens: number; + /** Cache accounting for this unit's own turns. Recorded beside tokens because a chain carries its + * context forward, so a later unit's input is mostly a cache read - a different price, and the + * reason a token count alone cannot be read as a cost. */ + cacheRead: number; + cacheWrite: number; + attempt: number; + /** The session the worker reported for this unit. A fused run's evidence is that two units name + * the same session; a worker that quietly starts a new one is not fusing, and its unit says so. */ + sessionId?: string; +} + +export interface PlanRun { + plan: readonly string[]; + /** Start order, which is the evidence for the slot count: with one slot it is the legal order. */ + order: readonly string[]; + units: readonly UnitRun[]; + accepted: Readonly>; + /** Wall time from the first claim to the last verdict. */ + wallMs: number; + /** Sum of the host's candidate checks. */ + hostMs: number; + hostChecks: number; + /** The slot count the caller asked for, and what the shared admission layer actually allowed. + * They differ today, and the difference is the finding: see `slotRefusal`. */ + slotsRequested: number; + slotsUsed: number; + /** Set when the requested slot count could not be used, with the board's own reason. A run that + * wanted N slots and got one must say so, or its wall time is read as the C arm's. */ + slotRefusal?: string; + tokens: number; + cacheRead: number; + cacheWrite: number; + failures: number; + /** Each session's units, in the order one session ran them. One entry per session: a fused run's + * cost claim rests on these, and a session of one unit is a yield boundary, not fusion. */ + sessions: readonly (readonly string[])[]; + /** The parent check's verdict, when the spec declares one. */ + parent?: { verdict: string; files: readonly string[]; ms: number }; + /** Set when a worker failed or a unit was never accepted: a comparison of such runs must say so + * rather than compare times. */ + incomplete: readonly string[]; +} + +/** A unit's acceptance: its own candidate check, run by the store through the spec it was given. */ +function unitVerifier(spec: PlanDriverSpec, unit: PlanUnit) { + return async (submission: PatchSubmission): Promise<"accept" | "reject" | "undecidable"> => { + if (submission.kind !== "patch") return "reject"; + const result = await verifyCandidate({ + repository: spec.repository, + revision: spec.revision, + files: { ...spec.baseline, ...submission.files }, + checks: [...unit.checks], + }); + return result.verdict === "accept" + ? "accept" + : result.verdict === "reject" + ? "reject" + : "undecidable"; + }; +} + +/** Who a unit's handoff is offered to, and who therefore claims it. One name, one home: the board + * directs the handoff to it and the claim names it, so a run that declares more than one slot cannot + * offer work to one name and claim it as another. */ +export const ownerOf = (taskId: string): string => `plan-driver:${taskId}`; + +/** One unit's own record of the session it ran in: absent when the worker reported none, so a fused run + * is judged on a session the worker named rather than on the one the driver asked for. */ +function reportedSession(result: { metrics?: WorkerMetrics }): { sessionId?: string } { + const sessionId = result.metrics?.sessionId; + return sessionId === undefined ? {} : { sessionId }; +} + +/** The cache accounting a worker reports, summed over its own turns. It is recorded beside tokens + * because the two are not the same currency: a chain carries its context forward, so most of what a + * later unit sends is a cache read, which is priced far below a fresh input token. Without these two + * fields a token count cannot be turned into a cost. */ +function reportedCache(result: { metrics?: WorkerMetrics }): { + cacheRead: number; + cacheWrite: number; +} { + return { + cacheRead: result.metrics?.cacheRead ?? 0, + cacheWrite: result.metrics?.cacheWrite ?? 0, + }; +} + +/** One unit through the board: claim, run the worker, put the result on the channel, submit. The + * store decides the verdict; the driver never reads a worker's claim about itself. */ +async function runOneUnit( + spec: PlanDriverSpec, + gate: BoardAdmission, + taskId: string, + session?: PlanSession, +): Promise { + const unit = spec.units[taskId]; + if (!unit) return { failure: `${taskId}: the plan selected it, but no spec describes it` }; + const claimedAt = Date.now(); + let ticket: ReturnType; + try { + ticket = gate.claim(taskId, ownerOf(taskId)); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + // The board publishes a handoff only for the task it has selected, so while one unit is claimed + // no other unit is claimable. That is the boundary the C arm needs and does not have; it is + // reported as a refusal to use the slot count, never as a failed unit. + if (/no published handoff|not selected by narrow dispatch/.test(reason)) + return { refused: reason }; + return { failure: `${taskId}: ${reason}` }; + } + if (!ticket.patch) return { failure: `${taskId}: not a patch task` }; + const frozen = preparePatchWork({ + taskId: ticket.patch.taskId, + attempt: ticket.attempt, + instruction: ticket.patch.instruction, + files: ticket.patch.files, + editable: ticket.patch.editable, + visible: ticket.patch.visible, + admittedConclusions: ticket.patch.admittedConclusions, + budget: ticket.patch.budget, + limits: ticket.patch.limits, + }); + let produced: PlanWorkerResult; + try { + produced = await spec.worker(taskId, frozen, ticket.dependencies, session); + } catch (error) { + return { failure: `${taskId}: ${error instanceof Error ? error.message : String(error)}` }; + } + const workerMs = Date.now() - claimedAt; + const result = typeof produced === "string" ? { artifact: produced } : produced; + const tokens = result.metrics?.tokens ?? 0; + if (result.failure !== undefined || result.artifact === undefined) + return { failure: `${taskId}: ${result.failure ?? "the worker returned no artifact"}` }; + const entry = gate.putTaskBoardEntry({ + taskId: gate.channel, + agentId: ownerOf(taskId), + kind: "result", + content: JSON.stringify({ ticket, artifact: result.artifact }), + expiresAt: new Date(gate.now + 86_400_000).toISOString(), + }); + const checkStartedAt = Date.now(); + const verdict = await gate.submit(entry.id); + return { + taskId, + verdict, + workerMs, + hostMs: Date.now() - checkStartedAt, + tokens, + attempt: ticket.attempt, + ...reportedCache(result), + ...reportedSession(result), + }; +} + +/** The parent check: the fixed acceptance over the composed artifacts, run once at the end. */ +async function runParentCheck( + spec: PlanDriverSpec, + accepted: Readonly>, + planIds: readonly string[], +): Promise { + if (!spec.parentChecks?.length) return undefined; + const startedAt = Date.now(); + const files: Record = { ...spec.baseline }; + const acceptedIds = spec.join ? [spec.join] : planIds; + for (const id of acceptedIds) { + const artifact = accepted[id]; + if (!artifact) continue; + const submission = JSON.parse(artifact) as PatchSubmission; + // A submitted patch carries the unit's whole frozen view, so merging it wholesale would put the + // last unit's untouched copies of its siblings' files - the stubs it was frozen with - over the + // work they actually did. Only what the unit changed is its work; the rest stays as the baseline. + if (submission.kind === "patch") + for (const [path, content] of Object.entries(submission.files)) + if (spec.baseline[path] !== content) files[path] = content; + } + const verified = await verifyCandidate({ + repository: spec.repository, + revision: spec.revision, + files, + checks: [...spec.parentChecks], + }); + return { + verdict: verified.verdict, + files: acceptedIds.filter((id) => accepted[id] !== undefined), + ms: Date.now() - startedAt, + }; +} + +export async function runPlan(spec: PlanDriverSpec): Promise { + if (!Number.isInteger(spec.slots) || spec.slots < 1) + throw new Error(`slots must be a positive integer, got ${spec.slots}`); + const planIds = spec.plan.map((row) => String(row[0])); + for (const id of Object.keys(spec.units)) + if (!planIds.includes(id)) + throw new Error(`unit ${id} has a spec but is not in the plan; the plan is the authority`); + const gate = new BoardAdmission( + spec.databasePath ?? ":memory:", + spec.plan, + {}, + { + slots: spec.slots, + // Each slot's work is offered point-to-point, because the store queues a second un-directed + // actionable behind the first: without a target the second claim is refused, not parallel. + handoffTarget: ownerOf, + }, + ); + const startedAt = Date.now(); + const units: UnitRun[] = []; + const order: string[] = []; + const failures: string[] = []; + const slotRefusals: string[] = []; + + for (const [taskId, unit] of Object.entries(spec.units)) { + const patch: PatchTaskSpec = { + instruction: unit.instruction, + files: spec.baseline, + editable: unit.editable, + ...(unit.visible ? { visible: unit.visible } : {}), + ...(spec.budget ? { budget: spec.budget } : {}), + ...(spec.limits ? { limits: spec.limits } : {}), + verify: unitVerifier(spec, unit), + }; + gate.installPatchTask(taskId, patch); + } + const dispatch = async (taskId: string, session?: PlanSession): Promise => { + order.push(taskId); + const result = await runOneUnit(spec, gate, taskId, session); + if ("refused" in result) { + slotRefusals.push(result.refused); + order.pop(); + return false; + } + if ("failure" in result) failures.push(result.failure); + else units.push(result); + return true; + }; + /** What a unit's session declaration is when the spec declares no bound for it: one capability and + * one authority for the whole run, and only the unit's own visibility decides. */ + const declarationOf = (id: string) => ({ + capability: spec.fusion?.declarations?.[id]?.capability ?? "patch", + authority: spec.fusion?.declarations?.[id]?.authority ?? "host", + visible: spec.units[id]?.visible ?? [], + }); + /** The fusion legality view. The board's own candidate answer stays the authority on staleness, + * cancellation, delivery and a declared external wait, because the driver cannot read those facts + * back out of it; the shared pair rule adds what fusion alone cares about - compatible capability, + * authority and visibility, a dependency that is accepted rather than merely delivered, and no + * reuse across a branch that is still pending. */ + const sessionPlan = (pendingBranches: readonly string[]): SessionPlan => { + const accepted = gate.accepted(); + return { + tasks: spec.plan.map((row) => ({ + id: String(row[0]), + effect: String(row[3]), + sourceVersion: spec.revision, + observedVersion: spec.revision, + dependencies: [...row[2]], + accepted: accepted[String(row[0])] !== undefined, + claimed: false, + externalReady: true, + })), + declarations: Object.fromEntries( + spec.plan.map((row) => [String(row[0]), declarationOf(String(row[0]))]), + ), + ...(pendingBranches.length ? { pendingBranches } : {}), + }; + }; + /** The next unit that may continue this session, asked of the one owner of the move: legal by the + * shared rule, still on offer by the board, and inside the declared bound. A bound is what keeps a + * fused run from swallowing the plan, and the move is repair-first - it either admits the next + * legal successor or closes the session, which is the irreversible commitment the design describes. */ + const nextInChain = (current: string, session: PlanSession): string | undefined => { + const move = nextSessionMove({ + plan: sessionPlan([]), + current, + size: session.units.length, + bound: spec.fusion?.unitsPerSession ?? 1, + onOffer: gate.candidates(), + }); + return move.kind === "admit" ? move.unit : undefined; + }; + /** One session: claim and check each unit in turn, and continue only from a unit the host has + * accepted. A unit that fails, or a successor that is not legal at the boundary, ends the session + * there - which is the yield boundary the design asks for, and why the accepted prefix survives. */ + const runChain = async (first: string, chains: string[][]): Promise => { + // The session's units are this driver's own array, handed out under a readonly view: a session's + // shape is a fact consumers read, and the driver is the one place that grows it. + const sessionUnits: string[] = []; + const session: PlanSession = { id: `session:${first}`, units: sessionUnits }; + /** The units the worker reported running in this session. Fusion's evidence: the driver asking for + * a session is not the fact - the worker's own report is. */ + const fused: string[] = []; + let started = false; + for (let current: string | undefined = first; current !== undefined;) { + const id = current; + const before = units.length; + const ok = await dispatch(id, session); + if (!ok) break; + // A worker that failed recorded no unit: the session ends with the units that did run. + const unit = units.length > before ? units[before] : undefined; + if (!unit) break; + sessionUnits.push(id); + started = true; + if (unit?.sessionId !== session.id) { + // It ran somewhere of its own: that is a session of one, reported as one, and the chain ends. + chains.push([id]); + break; + } + fused.push(id); + // Continuing is the shared rule's decision, not a second copy of it: `fusionSuccessors` already + // requires the unit that just ran to be *accepted*, so a rejected or undecidable verdict ends the + // session through the same predicate that guards every other boundary. + current = nextInChain(id, session); + } + // One entry per session, whatever its length: a session of one unit is a yield boundary, and a run + // whose sessions are all singletons did not fuse anything. A chain that could not start is not a + // session at all - that refusal is reported as a refusal, not as an empty session. + if (fused.length) chains.push(fused); + return started; + }; + /** Fusion is on when the spec declares a bound, and a bound of one is the control arm: the same + * accounting with no session reuse, so a fused run is compared against the same code path. */ + const fusionDeclared = spec.fusion !== undefined; + /** The most units ever claimed at the same time: requested slots are a wish, this is the fact. */ + let widestHeld = 0; + const chains: string[][] = []; + for (;;) { + const legal = gate.candidates(); + if (!legal.length) break; + // A fused run holds one session per slot: the bound decides how far a session goes, and the slot + // count decides how many sessions are open at once. + const batch = legal.slice(0, fusionDeclared ? 1 : spec.slots); + const held = await Promise.all( + batch.map((id) => (fusionDeclared ? runChain(id, chains) : dispatch(id))), + ); + widestHeld = Math.max(widestHeld, held.filter(Boolean).length); + } + const wallMs = Date.now() - startedAt; + const accepted = gate.accepted(); + const incomplete = [ + ...failures, + ...planIds.filter( + (id) => accepted[id] === undefined && units.some((unit) => unit.taskId === id), + ), + ]; + + const parent = await runParentCheck(spec, accepted, planIds); + gate.close(); + return { + plan: planIds, + order, + units, + accepted, + wallMs, + slotsRequested: spec.slots, + slotsUsed: Math.max(1, widestHeld), + sessions: chains, + ...(slotRefusals.length + ? { slotRefusal: `wanted ${spec.slots} slots, the board allowed one: ${slotRefusals[0]}` } + : {}), + hostMs: units.reduce((total, unit) => total + unit.hostMs, 0), + hostChecks: units.filter((unit) => unit.verdict !== "worker-failed").length, + tokens: units.reduce((total, unit) => total + unit.tokens, 0), + cacheRead: units.reduce((total, unit) => total + unit.cacheRead, 0), + cacheWrite: units.reduce((total, unit) => total + unit.cacheWrite, 0), + failures: failures.length, + ...(parent ? { parent } : {}), + incomplete, + }; +} + +export interface PlanComparison { + plan: readonly string[]; + runs: number; + arms: { slots: number; runs: PlanRun[] }[]; + /** True when every run of every arm accepted the same units and reached the same parent verdict. + * A comparison with different verdicts is not a comparison, and the report says so. */ + qualityParity: boolean; + differences: string[]; + caveat: string; + /** Every run whose requested slot count the board did not allow, with its reason. */ + slotShortfalls: string[]; + /** False when a time difference must not be read as an arm's result. */ + comparable: boolean; +} + +/** Runs the same spec at two slot counts. The slot count is the only difference: same plan, same + * units, same checks, same parent acceptance - which is what the B/C arms require. */ +export async function comparePlanSlots( + spec: PlanDriverSpec, + options: { runs: number; arms?: readonly number[] }, +): Promise { + if (!Number.isInteger(options.runs) || options.runs < 1) + throw new Error(`runs must be a positive integer, got ${options.runs}`); + const arms = options.arms ?? [1, spec.slots]; + const results: PlanComparison["arms"] = []; + for (const slots of arms) { + const runs: PlanRun[] = []; + for (let index = 0; index < options.runs; index += 1) + runs.push(await runPlan({ ...spec, plan: [...spec.plan], slots })); + results.push({ slots, runs }); + } + const differences: string[] = []; + const slotShortfalls: string[] = []; + const shape = (run: PlanRun) => + JSON.stringify({ + order: [...run.order].sort(), + verdicts: run.units.map((unit) => `${unit.taskId}:${unit.verdict}`).sort(), + ...(run.parent ? { parent: run.parent.verdict } : {}), + }); + const first = results[0]!.runs[0]!; + for (const arm of results) + for (const run of arm.runs) { + if (shape(run) !== shape(first)) + differences.push( + `slots=${arm.slots} disagreed with the first run: ${shape(run)} vs ${shape(first)}`, + ); + if (run.slotsUsed < run.slotsRequested) + slotShortfalls.push( + `slots=${arm.slots} ran with ${run.slotsUsed}: ` + + (run.slotRefusal ?? "the plan had no more startable units"), + ); + } + return { + plan: first.plan, + runs: options.runs, + arms: results, + qualityParity: differences.length === 0, + differences, + slotShortfalls, + /** A slot count that was never reached makes the two arms the same experiment, so the time + * comparison is refused rather than reported as "no gain". */ + comparable: differences.length === 0 && slotShortfalls.length === 0, + caveat: + "cost and scheduling only where the slot counts were actually reached and quality matches: " + + "with different verdicts, or with a slot count the board refused, these are not a faster and " + + "a slower run of one experiment.", + }; +} + +const USAGE = `usage: + plan-driver.ts run --spec --slots --out [--live] + plan-driver.ts compare --spec --out [--runs ] [--live] + +A spec is JSON: + baseline: repository-relative paths read at the round's revision + plan: the units, each { id, revision?, dependencies?, effect, operation? } + units: per unit id { instruction, editable, visible?, checks?, canned? } + checks: this unit's own candidate check; without it the global list is used + canned: only for worker.kind "canned": editable path -> file holding the answer + checks: the unit's own candidate check, [{ label, command, args }] + parentChecks: the fixed parent acceptance, run once over the composed artifacts + worker: { kind: "stub", latencyMs, fail? } | { kind: "canned" } | { kind: "pi", provider, model }`; + +/** The spec file's shape: JSON, with `worker` naming how an artifact is produced. */ +export interface SpecFile { + baseline: readonly string[]; + revision?: string; + /** The frozen envelope's fixed budget and limits. Optional because a run that does not name them + * takes the host's own defaults; a paid run fixes them so two arms differ only in the plan. */ + budget?: { perFile: number; output: number }; + limits?: { turns: number; reads: number; timeoutMs: number }; + plan: readonly { + id: string; + revision?: string; + dependencies?: readonly string[]; + effect: string; + operation?: string | null; + }[]; + units: Readonly>; + /** The fallback check list: a unit that declares none of its own is checked by this. A file that + * declares neither is refused, because a unit nothing checks is not a unit. */ + checks?: readonly { label: string; command: string; args: string[] }[]; + parentChecks?: readonly { label: string; command: string; args: string[] }[]; + join?: string; + /** Execution fusion, declared in the spec file the same way the driver's own spec declares it. It is + * copied through by `specFrom`: a spec that asked for fusion and silently got none would be read as + * the control arm. */ + fusion?: { + unitsPerSession: number; + declarations?: Readonly>; + }; + worker: + | { kind: "stub"; latencyMs: number; fail?: readonly string[] } + | { kind: "canned" } + | { kind: "pi"; provider: string; model: string }; +} + +/** One unit's work: what to ask for, which files it may write, what its candidate is checked by, and + * (for the canned worker) where the instrument's own answer is read from. */ +interface SpecUnit { + instruction: string; + editable: string[]; + visible?: string[]; + /** This unit's own checks. Without them every unit is checked by the whole list, which a fine plan + * cannot use: a unit whose siblings are still unimplemented would never pass its own candidate. */ + checks?: readonly { label: string; command: string; args: string[] }[]; + /** The instrument's answer, as editable path -> the file holding the content to return. A canned + * run is how the task family is shown to accept a correct submission without paying a model. */ + canned?: Readonly>; +} + +function checkList( + raw: readonly { label: string; command: string; args: readonly string[] }[], +): CandidateCheck[] { + if (!raw.length) throw new Error("a check list may not be empty"); + return raw.map((check) => ({ + label: check.label, + command: check.command, + args: [...check.args], + })); +} + +function readSpecFile(path: string): SpecFile { + const file = JSON.parse(readFileSync(path, "utf8")) as SpecFile; + if (!Array.isArray(file.plan) || !file.plan.length) + throw new Error("spec.plan must be non-empty"); + if (!Array.isArray(file.baseline) || !file.baseline.length) + throw new Error("spec.baseline must name at least one file"); + if (!file.units || Object.keys(file.units).length === 0) + throw new Error("spec.units must describe at least one unit"); + const planIds = new Set(file.plan.map((row) => row.id)); + for (const id of Object.keys(file.units)) + if (!planIds.has(id)) + throw new Error(`spec.units.${id} is not in the plan; the plan is the authority`); + return file; +} + +function planOf(file: SpecFile): ProbePlan { + return file.plan.map((row) => [ + row.id, + row.revision ?? "", + [...(row.dependencies ?? [])], + row.effect, + null, + row.operation ?? null, + ]) as unknown as ProbePlan; +} + +function baselineOf(file: SpecFile, repository: string): Record { + return Object.fromEntries( + file.baseline.map((path) => [path, readFileSync(resolve(repository, path), "utf8")]), + ); +} + +/** The stub worker answers by leaving its editable files as they are, so acceptance depends on the + * unit's own check and the run measures the driver rather than a model. `fail` names units that + * must fail, which is how the driver's failure path is exercised without a model. */ +function stubWorker(worker: { latencyMs: number; fail?: readonly string[] }): PlanWorker { + return async (taskId, frozen) => { + await new Promise((done) => setTimeout(done, worker.latencyMs)); + if (worker.fail?.includes(taskId)) return { failure: `stub worker: ${taskId} fails` }; + // The protocol's patch shape is a list of whole-file replacements, and the store checks that each + // one really differs from the frozen input: an object map (or an unchanged file) is not a proposal + // the host can compare, so a stub that sent one would be rejected for the wrong reason. + const files = frozen.work.editable.map((path) => ({ + path, + content: frozen.work.files[path] ?? "", + })); + return { + artifact: JSON.stringify({ digest: frozen.digest, files }), + metrics: { tokens: 0, turns: 0, checks: 0 }, + }; + }; +} + +/** The canned worker: the instrument's own answer, read from the paths the spec names. It exists so + * that the task family's acceptance is checked offline - and so that a canned answer which is wrong + * is rejected - before any model is asked to write one. */ +export function cannedWorker(file: SpecFile): PlanWorker { + const answers = new Map>>(); + for (const [id, unit] of Object.entries(file.units)) { + if (!unit.canned) + throw new Error( + `the canned worker needs an answer for every unit: ${id} has none, ` + + "and an instrument cannot be checked by a unit that answers nothing", + ); + answers.set(id, unit.canned); + } + return async (taskId, frozen) => { + const answer = answers.get(taskId); + if (!answer) return { failure: `${taskId}: the canned worker has no answer for it` }; + // The protocol's patch shape: whole-file replacements that differ from the frozen input. + const files: { path: string; content: string }[] = []; + for (const path of frozen.work.editable) { + const source = answer[path]; + if (!source) return { failure: `${taskId}: the canned answer has no content for ${path}` }; + files.push({ path, content: readFileSync(resolve(process.cwd(), source), "utf8") }); + } + return { + artifact: JSON.stringify({ digest: frozen.digest, files }), + metrics: { tokens: 0, turns: 0, checks: 0 }, + }; + }; +} + +export function piWorker(worker: { provider: string; model: string }, live: boolean): PlanWorker { + if (!live) + throw new Error( + "refusing a live model run without --live: the spec names the provider, the operator " + + "authorizes the spend", + ); + return async (taskId, frozen, _dependencies, session) => { + // Measured, not assumed: `executePiPatch` creates a session per call (`SessionManager.inMemory()`), + // so this worker can start a session but cannot continue one. A fused live run therefore needs the + // extension to hold a session across calls - until it does, the continuation is refused by name + // here rather than answered with a fresh session that would be reported as fusion. + if (session && session.units.length > 0) + return { + failure: + `${taskId}: the live worker cannot continue session ${session.id} (it has run ` + + `${session.units.join(", ")}); a fused live arm needs the extension to hold one session ` + + "across calls, and this harness creates a session per call", + }; + const { executePiPatch } = await import("../../.pi/extensions/nmg/ooo-execution.ts"); + const run = await executePiPatch(frozen, worker.provider, worker.model); + if (!run.artifact) return { failure: `${taskId}: the worker returned no artifact` }; + return { + artifact: run.artifact, + metrics: { + tokens: run.tokens, + turns: run.turns, + checks: run.checks, + cacheRead: run.cacheRead, + cacheWrite: run.cacheWrite, + ...(session ? { sessionId: session.id } : {}), + }, + }; + }; +} + +/** The live worker that can honour a continuation: one runner per session id, created on that + * session's first unit and re-pointed for the rest. `piWorker` cannot do this - the extension it calls + * creates a session per call - so a fused live arm uses this one, and its `sessionId` is read from the + * session the unit actually ran in. `close()` disposes them; a run should call it once. */ +export function piSessionWorker( + worker: { provider: string; model: string }, + live: boolean, +): { worker: PlanWorker; close: () => void } { + if (!live) + throw new Error( + "refusing a live model run without --live: the spec names the provider, the operator " + + "authorizes the spend", + ); + const runners = new Map< + string, + import("../../.pi/extensions/nmg/ooo-execution.ts").PiSessionRunner + >(); + const planWorker: PlanWorker = async (taskId, frozen, _dependencies, session) => { + const key = session?.id ?? `unit:${taskId}`; + const { createPiSessionRunner, patchSessionInput } = + await import("../../.pi/extensions/nmg/ooo-execution.ts"); + let runner = runners.get(key); + if (!runner) { + const input = patchSessionInput(frozen); + runner = await createPiSessionRunner({ + provider: worker.provider, + modelId: worker.model, + patchMode: true, + first: input, + // A chain's surface is fixed when the session is created, so it registers the union of what + // its units may need rather than the first unit's subset. + chain: session !== undefined, + }); + runners.set(key, runner); + } + const run = await runner.runUnit(patchSessionInput(frozen)); + if (!run.artifact) return { failure: `${taskId}: the worker returned no artifact` }; + return { + artifact: run.artifact, + metrics: { + tokens: run.tokens, + turns: run.turns, + checks: run.checks, + cacheRead: run.cacheRead, + cacheWrite: run.cacheWrite, + // The driver's name for the session it asked for, reported only because this runner is the one + // held under that name: a worker that answered with a session of its own reports a different id + // and the driver ends the chain, which is how a fused run is told from a wish. + ...(session ? { sessionId: session.id } : {}), + }, + }; + }; + return { + worker: planWorker, + close: () => { + for (const runner of runners.values()) runner.dispose(); + runners.clear(); + }, + }; +} + +export function specFrom(file: SpecFile, worker: PlanWorker, slots: number): PlanDriverSpec { + const repository = process.cwd(); + const fallback = file.checks ? checkList(file.checks) : undefined; + return { + plan: planOf(file), + units: Object.fromEntries( + Object.entries(file.units).map(([id, unit]) => { + const checks = unit.checks ? checkList(unit.checks) : fallback; + if (!checks) + throw new Error( + `${id}: no checks - a unit is checked by what it declares, or by the spec's own list`, + ); + return [ + id, + { + instruction: unit.instruction, + editable: unit.editable, + ...(unit.visible ? { visible: unit.visible } : {}), + checks, + }, + ]; + }), + ), + worker, + repository, + revision: file.revision ?? "HEAD", + baseline: baselineOf(file, repository), + ...(file.parentChecks ? { parentChecks: checkList(file.parentChecks) } : {}), + ...(file.join ? { join: file.join } : {}), + ...(file.budget ? { budget: file.budget } : {}), + ...(file.limits ? { limits: file.limits } : {}), + ...(file.fusion ? { fusion: file.fusion } : {}), + slots, + }; +} + +async function main(): Promise { + const [command, ...rest] = process.argv.slice(2); + if (command !== "run" && command !== "compare") throw new Error(USAGE); + const { values } = parseArgs({ + args: rest, + options: { + spec: { type: "string" }, + slots: { type: "string" }, + runs: { type: "string" }, + out: { type: "string" }, + live: { type: "boolean" }, + // Holds one Pi session across a chain's units. Off by default: the flag is what makes a fused + // live arm possible, and a spec naming `pi` without it keeps the per-call worker (and its + // refusal to continue a session) as the recorded behaviour. + "session-runner": { type: "boolean" }, + }, + allowPositionals: false, + }); + if (!values.spec || !values.out) throw new Error(USAGE); + const file = readSpecFile(values.spec); + const slots = values.slots === undefined ? 2 : Number(values.slots); + const sessions = + file.worker.kind === "pi" && values["session-runner"] === true + ? piSessionWorker(file.worker, values.live === true) + : undefined; + const worker = + file.worker.kind === "stub" + ? stubWorker(file.worker) + : file.worker.kind === "canned" + ? cannedWorker(file) + : (sessions?.worker ?? piWorker(file.worker, values.live === true)); + const spec = specFrom(file, worker, slots); + let report: Awaited> | PlanComparison; + try { + report = + command === "run" + ? await runPlan(spec) + : await comparePlanSlots(spec, { + runs: values.runs === undefined ? 3 : Number(values.runs), + }); + } finally { + sessions?.close(); + } + const out = resolve(values.out); + mkdirSync(dirname(out), { recursive: true }); + writeFileSync( + out, + `${JSON.stringify({ measuredAt: new Date().toISOString(), spec: values.spec, report }, null, 2)}\n`, + ); + console.log(JSON.stringify(report, null, 2)); +} + +const invokedPath = process.argv[1] ? resolve(process.argv[1]) : ""; +if (invokedPath === fileURLToPath(import.meta.url)) await main(); + +export { USAGE }; diff --git a/evals/ooo-execution/probe-check-duration.ts b/evals/ooo-execution/probe-check-duration.ts deleted file mode 100644 index 5e01a6ee..00000000 --- a/evals/ooo-execution/probe-check-duration.ts +++ /dev/null @@ -1,245 +0,0 @@ -// Does the share of a round that out-of-order execution hides actually grow with the check's -// duration? That is the premise the design rests on, and it has never been measured — the in-round -// check in real rounds is ~1.9 s against ~56 s of worker time. -// -// What is real: the orchestrator, the board, the candidate worktree, the check processes, the -// acceptance rules and the frozen baseline. What is chosen rather than observed: the check's -// duration, and the workers' duration. A replay returns instantly, so an offline arm can never be -// overlapped at all — hence workers that do CPU-bounded work of a chosen duration. CPU-bound -// overlap is a LOWER bound for the real case: two CPU-bound activities contend for cores, while a -// model call mostly waits on the network. -// -// Zero model tokens: no provider is contacted. -import { execFile } from "node:child_process"; -import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; -import { promisify } from "node:util"; -import type { FrozenPatchWork } from "../../src/integration/ooo-patch.ts"; -import { compareModes } from "./round-compare.ts"; -import { readRoundLog } from "../../src/integration/ooo-round-log.ts"; -import { parseRoundSpec, type RoundSpec } from "./round-spec.ts"; - -const run = promisify(execFile); - -/** Real CPU work for a chosen duration. Not a sleep: a sleep would not contend for anything. */ -const busy = (ms: number): string => - `const end=Date.now()+${ms};let a=1;while(Date.now() { - return { - revision: "HEAD", - baseline: BASELINE, - checks: [ - { - label: "durational-check", - command: process.execPath, - args: ["-e", busy(checkMs)], - }, - ], - a: { instruction: "repair what the check exposes", editable: BASELINE }, - b: { instruction: "add the missing regressions", editable: BASELINE }, - worker: { kind: "replay", log: "unused-in-this-probe" }, - noChangeCases: { A: [{ name: "check-identity", token: CASE_TOKEN }] }, - admitted: { - A: ["no-change-needed", "cannot-complete"], - B: ["cannot-complete"], - C: ["promote-candidate", "cannot-complete"], - }, - budget: { perFile: 40_000, output: 40_000 }, - limits: { turns: 2, reads: 2, timeoutMs: 120_000 }, - }; -} - -/** The same accepted answers the comparison test uses: the probe studies scheduling, so the inputs - * must not differ between the arms or between the grid points. */ -function answers(task: string, frozen: FrozenPatchWork): string { - if (task === "A") - return JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "no-change-needed", - summary: "the check passes, so there is nothing to repair", - evidence: "cited from the frozen suite", - citations: [ - { - case: "check-identity", - test: "cancelling a round kills its running check and leaves no accepted work", - }, - ], - }); - if (task === "B") - return JSON.stringify({ - digest: frozen.digest, - files: [ - { - path: BASELINE[0]!, - content: `${frozen.work.files[BASELINE[0]!]}\ntest("compare added", () => {});\n`, - }, - ], - }); - return JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "promote-candidate", - summary: "combined candidate verified by the host", - evidence: "composed-check accept", - citations: [], - }); -} - -interface Window { - label: string; - fromMs: number; - toMs: number; -} - -/** The true question: how much of the round check was covered by the independent task running, - * as opposed to two host checks simply running at the same time. */ -function coverage(logPath: string): { - checks: Window[]; - checkCoveredByBMs: number; -} { - const events = readRoundLog(readFileSync(logPath, "utf8")); - const at = (event: { at?: string }) => (event.at ? Date.parse(event.at) : Number.NaN); - const start = Math.min( - ...events.map((event) => at(event as { at?: string })).filter(Number.isFinite), - ); - const issued = new Map(); - const checks: Window[] = []; - let bFrom = Number.NaN; - let bTo = Number.NaN; - for (const event of events) { - const time = at(event as { at?: string }) - start; - if (event.kind === "check-issued") issued.set(event.ticket.label, time); - if (event.kind === "check-result" && issued.has(event.label)) - checks.push({ label: event.label, fromMs: issued.get(event.label)!, toMs: time }); - if (event.kind === "claim" && event.taskId === "B") bFrom = time; - if (event.kind === "artifact" && event.taskId === "B") bTo = time; - } - let covered = 0; - for (const check of checks) - covered = Math.max( - covered, - Math.max(0, Math.min(check.toMs, bTo) - Math.max(check.fromMs, bFrom)), - ); - return { checks, checkCoveredByBMs: covered }; -} - -interface Point { - checkMs: number; - workerMs: number; - oooWallMs: number; - sequentialWallMs: number; - hiddenWaitMs: number; - hiddenShare: number; - hostMs: number; - /** What the harness reports as hidden, versus how much of a check the independent task really - * covered. When these differ, the harness number is host-check concurrency, not wait hiding. */ - checkCoveredByBMs: number; - checkCount: number; - verdicts: string; -} - -function parseFlags(argv: readonly string[]): { out: string; checks: number[]; workers: number[] } { - const values = new Map(); - for (let i = 0; i < argv.length; i += 1) { - const flag = argv[i]!; - if (!flag.startsWith("--")) throw new Error(`unexpected argument: ${flag}`); - const value = argv[i + 1]; - if (value === undefined || value.startsWith("--")) - throw new Error(`${flag} needs a value (never default to a guess)`); - values.set(flag, value); - i += 1; - } - const out = values.get("--out"); - if (!out) throw new Error("--out is required: the probe records what it measured"); - const numbers = (flag: string, fallback: number[]) => - values.has(flag) ? values.get(flag)!.split(",").map(Number) : fallback; - return { - out, - checks: numbers("--check-ms", [0, 2_000, 8_000, 40_000]), - workers: numbers("--worker-ms", [2_000, 8_000]), - }; -} - -async function main(argv: readonly string[]): Promise { - const { out, checks, workers } = parseFlags(argv); - if (!BASELINE.every((file) => file.length > 0)) throw new Error("baseline is empty"); - const points: Point[] = []; - { - // The mandatory {0,0} control plus the requested grid, with a repeated point counted once: - // a duplicate would run twice into the same output directory, and the second run would find - // the first run's state there. - const requested: Array<{ checkMs: number; workerMs: number }> = [ - { checkMs: 0, workerMs: 0 }, - ...checks.flatMap((checkMs) => workers.map((workerMs) => ({ checkMs, workerMs }))), - ]; - const seen = new Set(); - const grid = requested.filter((point) => { - const key = `${point.checkMs}/${point.workerMs}`; - if (seen.has(key)) return false; - seen.add(key); - return true; - }); - for (const { checkMs, workerMs } of grid) { - const directory = join(out, `check-${checkMs}-worker-${workerMs}`); - mkdirSync(directory, { recursive: true }); - const spec: RoundSpec = parseRoundSpec(specObject(checkMs)); - const comparison = await compareModes({ - spec, - repository: REPOSITORY, - outputDirectory: directory, - times: 1, - workerFor: async () => async (task: string, frozen: FrozenPatchWork) => { - if (task === "B" && workerMs > 0) await run(process.execPath, ["-e", busy(workerMs)]); - return answers(task, frozen); - }, - }); - // Refuse to compare incomparable arms, exactly as the product comparison does. - if (!comparison.qualityParity) - throw new Error( - `check=${checkMs} worker=${workerMs}: the arms disagree, so their times are not comparable: ${JSON.stringify(comparison.differences)}`, - ); - const ooo = comparison.arms.find((entry) => entry.mode === "ooo")!; - const sequential = comparison.arms.find((entry) => entry.mode === "sequential")!; - if (ooo.hostChecks < 1) throw new Error("the check never ran, so nothing was measured"); - if (sequential.hiddenWaitMs !== 0) - throw new Error("the control must not hide a wait; its hidden wait is non-zero"); - points.push({ - checkMs, - workerMs, - oooWallMs: ooo.wallMs, - sequentialWallMs: sequential.wallMs, - hiddenWaitMs: ooo.hiddenWaitMs, - hiddenShare: ooo.hiddenWaitMs / sequential.wallMs, - hostMs: ooo.hostMs, - checkCoveredByBMs: coverage(join(directory, "ooo-1", "round.jsonl")).checkCoveredByBMs, - checkCount: coverage(join(directory, "ooo-1", "round.jsonl")).checks.length, - verdicts: Object.entries(ooo.verdicts) - .sort() - .map(([task, verdict]) => `${task}:${verdict}`) - .join(" "), - }); - } - } - mkdirSync(out, { recursive: true }); - writeFileSync(join(out, "check-duration.json"), JSON.stringify({ points }, null, 2), "utf8"); - process.stdout.write( - [ - "check ms | worker ms | ooo wall ms | sequential wall ms | ooo-seq ms | harness hidden ms | covered by B ms | checks", - ...points.map( - (p) => - `${String(p.checkMs).padStart(8)} | ${String(p.workerMs).padStart(9)} | ${String(p.oooWallMs).padStart(11)} | ${String(p.sequentialWallMs).padStart(18)} | ${String(p.oooWallMs - p.sequentialWallMs).padStart(10)} | ${String(p.hiddenWaitMs).padStart(17)} | ${String(p.checkCoveredByBMs).padStart(15)} | ${String(p.checkCount).padStart(6)}`, - ), - "", - `verdicts identical across the grid: ${new Set(points.map((p) => p.verdicts)).size === 1}`, - ].join("\n") + "\n", - ); - return 0; -} - -process.exitCode = await main(process.argv.slice(2)); diff --git a/evals/ooo-execution/process-fixture.ts b/evals/ooo-execution/process-fixture.ts index 4cdc47f3..461c5179 100644 --- a/evals/ooo-execution/process-fixture.ts +++ b/evals/ooo-execution/process-fixture.ts @@ -8,7 +8,6 @@ import type { ServerState } from "../../src/cli/lifecycle.ts"; import type { TaskBoardEntry } from "../../src/core/types.ts"; import type { BoardTicket, BoardAdmission } from "../../src/integration/ooo-board.ts"; -const channel = "ooo-process-probe"; type Command = { id: string; action: string; args: Record }; /** One command handler per action. A table rather than a switch, so adding an action * does not add a branch to the dispatcher's own complexity. */ @@ -178,11 +177,19 @@ if (process.argv[2] === "daemon" || liveDaemon) { host: "127.0.0.1", port: address.port, token, + // The run's board channel travels with the endpoint: a worker must poll the channel its + // own run publishes to, and only the coordinator knows which one that is. + channel: authority.channel, }, }); } else if (process.argv[2] === "worker" || process.argv[2] === "pi-worker") { let endpoint: ServerState; let agent: string; + let channel = ""; + const boardChannel = () => { + if (!channel) throw new Error("worker is not connected to a run"); + return channel; + }; // Local worker scratch only. Authority/generation/completion live in the daemon DB. const held = new Map(); const admission = async (body: Record): Promise => { @@ -199,7 +206,7 @@ if (process.argv[2] === "daemon" || liveDaemon) { const board = async (): Promise => { const result = (await httpCall(endpoint, "taskBoard", { action: "read", - taskId: channel, + taskId: boardChannel(), agentId: agent, includeResolved: true, limit: 200, @@ -212,7 +219,7 @@ if (process.argv[2] === "daemon" || liveDaemon) { const artifact = override ?? snapshotAnswer(ticket); const result = (await httpCall(endpoint, "taskBoard", { action: "put", - taskId: channel, + taskId: boardChannel(), agentId: agent, kind: "result", content: JSON.stringify({ ticket, artifact: override ?? artifact, passed: true }), @@ -250,6 +257,8 @@ if (process.argv[2] === "daemon" || liveDaemon) { connect: (args) => { endpoint = args.endpoint as ServerState; agent = String(args.agent); + channel = String((args.endpoint as { channel?: string }).channel ?? ""); + boardChannel(); return process.pid; }, board: () => board(), diff --git a/evals/ooo-execution/recovery.test.ts b/evals/ooo-execution/recovery.test.ts index 1c59d084..e5274bb8 100644 --- a/evals/ooo-execution/recovery.test.ts +++ b/evals/ooo-execution/recovery.test.ts @@ -15,7 +15,6 @@ import { type ProbePlan, } from "../../src/integration/ooo-board.ts"; -const channel = "ooo-process-probe"; const LEASE_MS = 60_000; const plan: ProbePlan = [ @@ -64,7 +63,7 @@ function submitPatch( artifact: string, ): Promise { const entry = gate.putTaskBoardEntry({ - taskId: channel, + taskId: gate.channel, agentId: ticket.owner, kind: "result", content: JSON.stringify({ ticket, artifact }), diff --git a/evals/ooo-execution/rename-probe.ts b/evals/ooo-execution/rename-probe.ts new file mode 100644 index 00000000..e0018e8c --- /dev/null +++ b/evals/ooo-execution/rename-probe.ts @@ -0,0 +1,23 @@ +/** + * The patch probe's frozen rename target, in one place. + * + * The probe's candidate is the whole file, and a dependent probe task carries it inside a snapshot, + * where the shared work contract bounds a dependency's serialized bytes at 8 KB. Pointing the probe at + * a live product file made its own limits depend on how big that file had grown, which is how a + * correct rename once came back as `rejected`; the fixture is frozen and small instead, and its own + * comment states the shape the oracle requires. Both suites that exercise the probe share this, so the + * path and the expected answer have one home. + */ +import { readFileSync } from "node:fs"; + +import { expectedRename } from "../../src/integration/check-runner.ts"; + +/** The path the probe's spec, artifact and verify all agree on. */ +export const RENAME_TARGET = "fixtures/rename-baseline.ts"; + +/** The frozen file the worker is asked to rename inside. */ +export const renameSource = (): string => + readFileSync(new URL("./fixtures/rename-baseline.ts", import.meta.url), "utf8"); + +/** The host oracle's answer: the exact rename of `byId` to `planIndex` in the frozen file. */ +export const expectedRenameOf = (source: string): string => expectedRename(source); diff --git a/evals/ooo-execution/replay.test.ts b/evals/ooo-execution/replay.test.ts deleted file mode 100644 index f830bed1..00000000 --- a/evals/ooo-execution/replay.test.ts +++ /dev/null @@ -1,282 +0,0 @@ -// S2: a round is replayed from its log, not re-run. -// -// The property under test is narrow and checkable: with the worker answers taken from the -// log and the host checks executed as before, the round reaches the *same* terminal state -// it recorded — and when the log is edited or truncated, it does not. The second half -// matters more than the first: a replay that trusts its own input is a transcript, not -// evidence. -import assert from "node:assert/strict"; -import test from "node:test"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { - runCycle, - type CheckRunner, - type CycleOptions, - type CycleResult, -} from "../../src/integration/ooo-cycle.ts"; -import { - RoundLog, - compareFrozen, - compareTerminal, - readRoundLog, - recordedPlan, - recordedWorker, - terminalEvent, -} from "../../src/integration/ooo-round-log.ts"; - -/** A deterministic round: the same three tasks the live cycle runs, with the host check - * replaced by a function so the test needs no worktrees. */ -function roundOptions(overrides: Partial = {}): CycleOptions { - // The frozen suite passes while the probe is intact, so the round has the shape the - // live one has: a real wait on a passing check, a gap-closing patch, a composition. - const verify: CheckRunner = async ({ files }) => { - // Substring tests on purpose: a check that depends on an exact trailing newline in a - // test fixture fails for reasons that have nothing to do with the round. - const intact = (files["src/probe.ts"] ?? "").includes("value = 1"); - const titled = (files["src/probe.test.ts"] ?? "").includes("probe identity"); - const passed = intact && titled; - return { - verdict: passed ? "accept" : "reject", - outcomes: [ - { - label: "protocol-regression", - status: passed ? "passed" : "failed", - log: "deterministic host check", - }, - ], - }; - }; - const baseline = { - "src/probe.ts": "export const value = 1;\n", - "src/probe.test.ts": "test('probe identity', () => {});\n", - }; - const artifacts: Record string> = { - // A repairs nothing: the check passed, so a cited no-change conclusion is the honest - // answer, which is also the answer the live rounds had trouble producing. - A: (digest) => - JSON.stringify({ - digest, - kind: "conclusion", - conclusion: "no-change-needed", - summary: "the check passed", - evidence: "protocol-regression=passed", - citations: [{ case: "check-identity", test: "probe identity" }], - }), - // B adds the regression its case rule asks for, leaving the implementation intact. - B: (digest) => - JSON.stringify({ - digest, - files: [ - { - path: "src/probe.test.ts", - content: "test('probe identity', () => {});\ntest('probe regression', () => {});\n", - }, - ], - }), - C: (digest) => - JSON.stringify({ - digest, - kind: "conclusion", - conclusion: "promote-candidate", - summary: "composed candidate", - evidence: "composed check passed", - citations: [], - }), - }; - return { - repository: process.cwd(), - revision: "HEAD", - baseline, - checks: [{ label: "protocol-regression", command: "node", args: ["-e", ""] }], - worker: async (taskId, frozen) => ({ artifact: artifacts[taskId]!(frozen.digest) }), - aInstruction: "repair the check", - bInstruction: "add a regression", - aEditable: ["src/probe.ts"], - bEditable: ["src/probe.test.ts"], - budget: { perFile: 8_000, output: 8_000 }, - limits: { turns: 4, reads: 2, timeoutMs: 60_000 }, - runChecks: verify, - noChangeCases: { - A: [{ name: "check-identity", token: "probe identity" }], - B: [{ name: "regression", token: "probe regression" }], - }, - admitted: { - A: ["no-change-needed", "cannot-complete"], - B: ["cannot-complete"], - C: ["promote-candidate", "cannot-complete"], - }, - maxReopens: 1, - ...overrides, - }; -} - -test("a round replays from its log to the same terminal state, with no worker call", async (t) => { - const directory = mkdtempSync(join(tmpdir(), "nmg-replay-")); - t.after(() => - rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }), - ); - const path = join(directory, "round.jsonl"); - - const original = await runCycle(roundOptions({ roundLog: new RoundLog(path) })); - const events = readRoundLog(readFileSync(path, "utf8")); - const recorded = terminalEvent(events); - assert.ok(recorded, "the round must record its own terminal state"); - assert.equal(recorded.accepted.C !== undefined, original.accepted.C !== undefined); - - // Replay: the same round, with the recorded answers instead of a worker. - let calls = 0; - const replayed = await runCycle( - roundOptions({ - worker: async (taskId, frozen, dependencies) => { - calls += 1; - return recordedWorker(events)(taskId, frozen, dependencies); - }, - }), - ); - assert.equal(calls, 3, "replay asks for exactly the recorded attempts"); - assert.deepEqual(compareTerminal(recorded, replayed), []); - // And the replayed round reaches the same state as the round it replaces. - assert.deepEqual(replayed.accepted, original.accepted); - assert.deepEqual(replayed.verdicts, original.verdicts); - assert.deepEqual(replayed.composed, original.composed); -}); - -test("an edited artifact in the log does not reproduce the verdict", async (t) => { - const directory = mkdtempSync(join(tmpdir(), "nmg-replay-edit-")); - t.after(() => - rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }), - ); - const path = join(directory, "round.jsonl"); - await runCycle(roundOptions({ roundLog: new RoundLog(path) })); - - // Rewrite B's recorded patch so it no longer carries the title its case rule asks for, - // keeping everything else — including its digest field — exactly as recorded. - const lines = readFileSync(path, "utf8").trim().split("\n"); - const edited = lines.map((line) => { - const event = JSON.parse(line) as { kind: string; taskId?: string; artifact?: string }; - if (event.kind !== "artifact" || event.taskId !== "B") return line; - const artifact = JSON.parse(event.artifact!) as { files: { path: string; content: string }[] }; - artifact.files[0]!.content = "export const value = 99;\n"; - return JSON.stringify({ ...event, artifact: JSON.stringify(artifact) }); - }); - const editedPath = join(directory, "edited.jsonl"); - writeFileSync(editedPath, edited.join("\n") + "\n", "utf8"); - - const events = readRoundLog(readFileSync(editedPath, "utf8")); - const recorded = terminalEvent(events)!; - const replayed = await runCycle(roundOptions({ worker: recordedWorker(events) })); - - // The round completes, but not with the recorded outcome: the host re-checked the - // artifact instead of trusting the log. - const differences = compareTerminal(recorded, replayed); - assert.ok(differences.length > 0, "an edited artifact must not reproduce the verdict"); - assert.ok( - differences.some((line) => line.startsWith("B:")), - `expected B to differ, got ${JSON.stringify(differences)}`, - ); -}); - -test("a truncated log is detected, not repaired", async (t) => { - const directory = mkdtempSync(join(tmpdir(), "nmg-replay-cut-")); - t.after(() => - rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }), - ); - const path = join(directory, "round.jsonl"); - await runCycle(roundOptions({ roundLog: new RoundLog(path) })); - - // Drop the last recorded artifact. The replay cannot invent the answer, and the missing - // attempt surfaces as a recorded failure rather than as a round that quietly differs. - const lines = readFileSync(path, "utf8").trim().split("\n"); - const artifactIndexes = lines - .map((line, index) => ((JSON.parse(line) as { kind: string }).kind === "artifact" ? index : -1)) - .filter((index) => index >= 0); - assert.equal(artifactIndexes.length, 3); - const truncated = lines.filter((_, index) => index !== artifactIndexes.at(-1)); - const events = readRoundLog(truncated.join("\n") + "\n"); - const recorded = terminalEvent(events)!; - - const replayed = await runCycle(roundOptions({ worker: recordedWorker(events) })); - assert.equal(replayed.verdicts.C, "rejected"); - assert.ok( - replayed.rejections.some((item) => /no artifact for C attempt 1/.test(item.artifact)), - `expected the missing attempt to be reported, got ${JSON.stringify(replayed.rejections)}`, - ); - assert.ok(compareTerminal(recorded, replayed).some((line) => line.startsWith("C:"))); -}); - -test("a malformed log line is refused, not skipped", () => { - assert.throws(() => readRoundLog("not json\n"), /not JSON/); - assert.throws(() => readRoundLog('{"at":"now","kind":"invented"}\n'), /unknown kind/); - assert.throws(() => readRoundLog('{"kind":"plan"}\n'), /no timestamp/); - // A log with no terminal event is readable; the caller decides what that means. - const events = readRoundLog('{"at":"now","kind":"plan","tasks":[],"checks":[]}\n'); - assert.equal(events.length, 1); - assert.equal(terminalEvent(events), null); -}); - -test("a replay is a round like any other: it cannot accept what the host refuses", async () => { - // Even a faithful log cannot talk the host into an acceptance it did not make. - const result: CycleResult = await runCycle( - roundOptions({ - worker: async () => ({ artifact: JSON.stringify({ digest: "f".repeat(64), files: [] }) }), - }), - ); - assert.deepEqual(result.accepted, {}); - assert.equal(result.verdicts.B, "rejected"); -}); - -test("a replay handed a different round is refused by name, not reported as reproduced", async (t) => { - const directory = mkdtempSync(join(tmpdir(), "nmg-replay-identity-")); - t.after(() => - rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }), - ); - const path = join(directory, "round.jsonl"); - const recorded = await runCycle(roundOptions({ roundLog: new RoundLog(path) })); - const events = readRoundLog(readFileSync(path, "utf8")); - const terminal = terminalEvent(events)!; - assert.deepEqual(compareFrozen(events, recorded.log), [], "a faithful round has no difference"); - // The plan records which revision the round verified, so a replay checks out the same one. - assert.equal(recordedPlan(events)?.revision, "HEAD"); - - // The trap this check exists for: a replay can be handed a different round and still produce - // the same verdicts. Comparing terminal states alone would call that a reproduction. - const withDifferentCheck = await runCycle( - roundOptions({ - worker: recordedWorker(events), - checks: [{ label: "protocol-regression", command: "node", args: ["-e", "process.exit(0)"] }], - }), - ); - assert.deepEqual( - compareTerminal(terminal, withDifferentCheck), - [], - "the trap: identical verdicts", - ); - const differences = compareFrozen(events, withDifferentCheck.log); - assert.equal(differences.length, 1, JSON.stringify(differences)); - assert.match(differences[0]!, /^verification rules: [0-9a-f]{12} -> [0-9a-f]{12}$/); - - // The per-task frozen work covers the baseline and every other task input; a change there - // makes the recorded artifacts stale, and the check names that instead of leaving it inferred. - const changedBaseline = await runCycle( - roundOptions({ - worker: recordedWorker(events), - baseline: { - "src/probe.ts": "export const value = 2;\n", - "src/probe.test.ts": "test('probe identity', () => {});\n", - }, - }), - ); - const frozen = compareFrozen(events, changedBaseline.log); - for (const task of ["A", "C"]) - assert.match( - frozen.find((line) => line.startsWith(`${task}#1`))!, - /not claimed in the replay$/, - JSON.stringify(frozen), - ); - assert.match( - frozen.find((line) => line.startsWith("B#1"))!, - /^B#1: [0-9a-f]{12} -> [0-9a-f]{12}$/, - ); -}); diff --git a/evals/ooo-execution/round-cli.test.ts b/evals/ooo-execution/round-cli.test.ts deleted file mode 100644 index 133284fe..00000000 --- a/evals/ooo-execution/round-cli.test.ts +++ /dev/null @@ -1,238 +0,0 @@ -// S3 exit: a supported round is repeated from a spec file without editing a research script, -// and is queryable and cancellable from other processes. -// -// The spec's `replay` worker is the point of the stage's repeatability: the same spec that ran -// a round can re-run it from its own record, and the round's identity check (round-log.ts -// compareFrozen) refuses a spec whose frozen work differs from the log's. -import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import test from "node:test"; -import { BoardAdmission } from "../../src/integration/ooo-board.ts"; -import { - ROUND_PLAN, - cancelRun, - describeRun, - logPath, - readBaseline, - readRecord, - runSpecifiedRound, - storePath, -} from "./round-runner.ts"; -import { parseRoundSpec, type RoundSpec } from "./round-spec.ts"; -import type { FrozenPatchWork } from "../../src/integration/ooo-patch.ts"; - -const CLI = join(import.meta.dirname, "round-cli.ts"); -/** Baseline small enough to keep the test quick, with a real test title the case cites. */ -const BASELINE = ["evals/ooo-execution/cancellation.test.ts"]; -const CASE_TOKEN = "cancelling a round kills"; - -function specObject(log: string): Record { - return { - revision: "HEAD", - baseline: BASELINE, - checks: [ - { label: "protocol-regression", command: process.execPath, args: ["-e", "process.exit(0)"] }, - ], - a: { instruction: "repair what the check exposes", editable: BASELINE }, - b: { instruction: "add the missing regressions", editable: BASELINE }, - worker: { kind: "replay", log }, - noChangeCases: { A: [{ name: "check-identity", token: CASE_TOKEN }] }, - admitted: { - A: ["no-change-needed", "cannot-complete"], - B: ["cannot-complete"], - C: ["promote-candidate", "cannot-complete"], - }, - budget: { perFile: 40_000, output: 40_000 }, - limits: { turns: 2, reads: 2, timeoutMs: 60_000 }, - }; -} - -/** The answers a round needs, written against the frozen envelope rather than fixed text: - * the digest has to be the one the round hands this attempt, and a patch has to be built from - * the frozen content, or the host refuses it for a reason the test would then be asserting. */ -function answers(task: string, frozen: FrozenPatchWork): string { - if (task === "A") - return JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "no-change-needed", - summary: "the check passes, so there is nothing to repair", - evidence: "cited from the frozen suite", - citations: [ - { - case: "check-identity", - test: "cancelling a round kills its running check and leaves no accepted work", - }, - ], - }); - if (task === "B") - return JSON.stringify({ - digest: frozen.digest, - files: [ - { - path: BASELINE[0]!, - content: `${frozen.work.files[BASELINE[0]!]}\ntest("round-cli added", () => {});\n`, - }, - ], - }); - return JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "promote-candidate", - summary: "combined candidate verified by the host", - evidence: "composed-check accept", - citations: [], - }); -} - -function writeSpec(directory: string, log: string): string { - const path = join(directory, "spec.json"); - writeFileSync(path, JSON.stringify(specObject(log), null, 2), "utf8"); - return path; -} - -test("a spec is refused unless it is fully understood", () => { - const base = specObject(".nmg/round.jsonl"); - assert.equal(parseRoundSpec(base).revision, "HEAD"); - // Each refusal names the field it refused: a spec that is half-understood must not run. - for (const [field, patch] of [ - ["surplus", { surplus: 1 }], - ["baseline[0]", { baseline: ["../outside.ts"] }], - ["worker.kind", { worker: { kind: "stub" } }], - ["a.instruction", { a: { editable: BASELINE } }], - ["admitted.A", { admitted: { A: [] } }], - ] as const) { - let message = ""; - try { - parseRoundSpec({ ...base, ...patch }); - } catch (error) { - message = error instanceof Error ? error.message : String(error); - } - assert.ok(message.includes(field), `${field}: got ${JSON.stringify(message)}`); - } -}); - -test( - "the CLI submits a spec, reports status, and re-runs the same round from its own record", - { timeout: 120_000 }, - async (t) => { - const directory = mkdtempSync(join(tmpdir(), "nmg-ooo-cli-")); - t.after(() => - rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }), - ); - const first = join(directory, "first"); - // First pass: record a round by running the spec with a stub worker, which is how a log that - // matches this spec's frozen work comes into existence at all. The spec is written twice - // because the recorded log only exists after this pass; the frozen inputs are identical. - const placeholder = writeSpec(directory, join(directory, "not-recorded-yet.jsonl")); - const spec = parseRoundSpec(JSON.parse(readFileSync(placeholder, "utf8"))) as RoundSpec; - const { revision, baseline } = readBaseline(spec, process.cwd()); - const recorded = await runSpecifiedRound({ - spec, - runDir: first, - repository: process.cwd(), - worker: async (task, frozen) => answers(task, frozen), - revision, - baseline, - }); - assert.deepEqual(recorded.verdicts, { B: "accepted", A: "accepted", C: "accepted" }); - assert.ok(readFileSync(logPath(first), "utf8").includes('"kind":"terminal"')); - // The same spec, rewritten to replay the log this pass produced. - const specPath = writeSpec(directory, logPath(first)); - - // Second pass: the same spec through the CLI, now replaying that record. No model call, - // and the round identity check has to accept the recorded answers. - const submitted = execFileSync( - process.execPath, - ["--experimental-strip-types", CLI, "submit", specPath, "--run-dir", join(directory, "run")], - { cwd: process.cwd(), encoding: "utf8" }, - ); - assert.match(submitted, /verdicts: \{"B":"accepted","A":"accepted","C":"accepted"\}/); - assert.match(submitted, /accepted: \{"A":"[\s\S]*"B":"[\s\S]*"C":"/); - const runDir = join(directory, "run"); - assert.ok(readRecord(runDir)?.finishedAt, "the run record is durable"); - - // status: a second process reads the same run directory. - const status = execFileSync( - process.execPath, - ["--experimental-strip-types", CLI, "status", "--run-dir", runDir], - { cwd: process.cwd(), encoding: "utf8" }, - ); - assert.match(status, /worker: replay/); - assert.match(status, /coordinator accepted: \{"A":"/); - assert.match(status, /measurements: host \d+ ms/); - - // An unknown flag is refused rather than ignored, and the refusal names the flag. - let refusal = ""; - try { - execFileSync( - process.execPath, - ["--experimental-strip-types", CLI, "status", "--run-dir", runDir, "--verbose"], - { cwd: process.cwd(), encoding: "utf8", stdio: "pipe" }, - ); - } catch (error) { - refusal = String((error as { stderr?: string }).stderr ?? ""); - } - assert.match(refusal, /unknown flag: --verbose/); - - // cancel: recorded in the round's own store by a different process. - const cancelled = execFileSync( - process.execPath, - [ - "--experimental-strip-types", - CLI, - "cancel", - "--run-dir", - runDir, - "--reason", - "operator stopped the round", - ], - { cwd: process.cwd(), encoding: "utf8" }, - ); - assert.match(cancelled, /cancellation recorded: operator stopped the round/); - assert.match(describeRun(runDir), /coordinator cancelled: operator stopped the round/); - assert.equal(cancelRun(runDir, "again"), "operator stopped the round"); - }, -); - -test( - "a round already cancelled in the store stops before it dispatches anything", - { timeout: 120_000 }, - async (t) => { - const directory = mkdtempSync(join(tmpdir(), "nmg-ooo-cli-cancel-")); - t.after(() => - rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }), - ); - const log = join(directory, "recorded.jsonl"); - const spec = parseRoundSpec(JSON.parse(readFileSync(writeSpec(directory, log), "utf8"))); - const runDir = join(directory, "run"); - const { revision, baseline } = readBaseline(spec, process.cwd()); - - // The cancellation is written by another process, before the round starts: the round polls - // the store, and must not spend a worker call after it. - mkdirSync(runDir, { recursive: true }); - const prepared = new BoardAdmission(storePath(runDir), ROUND_PLAN, {}); - prepared.cancel("stopped before dispatch"); - prepared.close(); - - let dispatched = 0; - const result = await runSpecifiedRound({ - spec, - runDir, - repository: process.cwd(), - worker: async (task, frozen) => { - dispatched += 1; - return answers(task, frozen); - }, - revision, - baseline, - }); - assert.equal(dispatched, 0, "a cancelled round must not spend a worker call"); - assert.equal(result.cancelled, "stopped before dispatch"); - assert.deepEqual(result.accepted, {}); - assert.match(describeRun(runDir), /cancelled: stopped before dispatch/); - }, -); diff --git a/evals/ooo-execution/round-cli.ts b/evals/ooo-execution/round-cli.ts deleted file mode 100644 index 951d1ed8..00000000 --- a/evals/ooo-execution/round-cli.ts +++ /dev/null @@ -1,114 +0,0 @@ -// S3 entry point: submit / status / cancel a round from a spec file. -// -// node --experimental-strip-types evals/ooo-execution/round-cli.ts submit --run-dir -// node --experimental-strip-types evals/ooo-execution/round-cli.ts status --run-dir -// node --experimental-strip-types evals/ooo-execution/round-cli.ts cancel --run-dir --reason -// -// `submit` runs in the foreground and reports what it did. `status` and `cancel` read or write -// the round's own durable state, so they work from other processes and across restarts — which -// is the point of the stage: a supported round is repeated without editing a research script. -import { readFileSync } from "node:fs"; -import { compareModes, describe } from "./round-compare.ts"; -import { parseRoundSpec } from "./round-spec.ts"; -import { - cancelRun, - describeRun, - readBaseline, - runSpecifiedRound, - specWorker, -} from "./round-runner.ts"; - -const USAGE = `usage: - round-cli.ts submit --run-dir [--live] - round-cli.ts status --run-dir - round-cli.ts cancel --run-dir --reason - round-cli.ts compare --out [--runs ]`; - -/** A tiny flag reader: unknown flags are refused rather than ignored. */ -function flags(argv: readonly string[], allowed: readonly string[]) { - const values: Record = {}; - for (let index = 0; index < argv.length; index += 1) { - const token = argv[index]!; - if (!token.startsWith("--")) throw new Error(`unexpected argument: ${token}`); - const name = token.slice(2); - if (!allowed.includes(name)) throw new Error(`unknown flag: ${token}`); - const next = argv[index + 1]; - if (next === undefined || next.startsWith("--")) values[name] = true; - else { - values[name] = next; - index += 1; - } - } - return values; -} - -function required(values: Record, name: string): string { - const value = values[name]; - if (typeof value !== "string" || !value.trim()) throw new Error(`--${name} is required`); - return value; -} - -async function submit(argv: readonly string[], repository: string) { - const [specPath, ...rest] = argv; - if (!specPath || specPath.startsWith("--")) throw new Error(USAGE); - const values = flags(rest, ["run-dir", "live"]); - const runDir = required(values, "run-dir"); - const spec = parseRoundSpec(JSON.parse(readFileSync(specPath, "utf8"))); - const { revision, baseline } = readBaseline(spec, repository); - const worker = await specWorker( - spec, - { repository, revision, baseline }, - { - live: values.live === true, - }, - ); - console.log(`submitting ${specPath} into ${runDir} (worker ${spec.worker.kind})`); - const result = await runSpecifiedRound({ spec, runDir, repository, worker, revision, baseline }); - console.log(describeRun(runDir)); - console.log( - `verdicts: ${JSON.stringify(result.verdicts)} accepted: ${JSON.stringify(result.accepted)}`, - ); - // A round that accepted nothing, or was cancelled, is not a success exit. - const accepted = Object.keys(result.accepted).length; - if (result.cancelled || !accepted) process.exitCode = 1; -} - -function status(argv: readonly string[]) { - const values = flags(argv, ["run-dir"]); - console.log(describeRun(required(values, "run-dir"))); -} - -function cancel(argv: readonly string[]) { - const values = flags(argv, ["run-dir", "reason"]); - const reason = required(values, "reason"); - const stored = cancelRun(required(values, "run-dir"), reason); - console.log(`cancellation recorded: ${stored}`); -} - -/** S4: run both dispatch orders over one spec and print what was measured. */ -async function compare(argv: readonly string[], repository: string) { - const [specPath, ...rest] = argv; - if (!specPath || specPath.startsWith("--")) throw new Error(USAGE); - const values = flags(rest, ["out", "runs"]); - const output = required(values, "out"); - const runs = values.runs === undefined ? 1 : Number(values.runs); - if (!Number.isSafeInteger(runs) || runs < 1) throw new Error("--runs must be a positive integer"); - const spec = parseRoundSpec(JSON.parse(readFileSync(specPath, "utf8"))); - const comparison = await compareModes({ spec, repository, outputDirectory: output, times: runs }); - for (const line of describe(comparison)) console.log(line); - console.log(`arms: ${comparison.arms.length}, worker ${comparison.worker}`); - if (!comparison.qualityParity) process.exitCode = 1; -} - -const [command, ...rest] = process.argv.slice(2); -const repository = process.cwd(); -try { - if (command === "submit") await submit(rest, repository); - else if (command === "status") status(rest); - else if (command === "cancel") cancel(rest); - else if (command === "compare") await compare(rest, repository); - else throw new Error(USAGE); -} catch (error) { - console.error(`round-cli: ${error instanceof Error ? error.message : String(error)}`); - process.exitCode = 2; -} diff --git a/evals/ooo-execution/round-client.ts b/evals/ooo-execution/round-client.ts new file mode 100644 index 00000000..cafb68f0 --- /dev/null +++ b/evals/ooo-execution/round-client.ts @@ -0,0 +1,116 @@ +/** + * The evidence drivers' thin adapter: they reach the round's board through the daemon that serves + * it, never by opening the database themselves. + * + * The design fixes one writer per store. A round's store is served by its host (an `NmgService` on + * the daemon transport), and the drivers - separate processes - are clients of that daemon. A driver + * that opened `/store.sqlite` directly would be a second writer, and a lifecycle write on a + * managed entry would bypass the run's coordinated transition; the store's fence refuses such a + * write, so the direct path cannot be the documented one either. + * + * Two pessimistic rules live here, both about the same failure: a process that serves an endpoint + * cannot answer a call to it while it is blocked, so "the answer will come" is an assumption this + * module refuses to make. + * + * - The call has a bound. A client gives up with a message naming the bound and the reason to + * suspect, instead of the transport's own five-minute headers timeout - which is what a blocked + * host produces, and what made this failure take 305 seconds to read the first time. + * - A client never calls the endpoint it serves. The lease records the serving pid, so this is a + * fact rather than a guess; a host that wants to write its own store calls the round's entry + * in-process (see `round-host.ts`), which is the design's shape for an offline host. + * + * The daemon is resolved from the store's own lease (`.server.json`), which is how the product + * discovers a daemon too. There is deliberately no fallback: when nothing serves the store this + * refuses by name, because "open the file instead" is the defect this module exists to remove. + */ +import { httpCall, TimeoutError } from "../../src/cli/http-client.ts"; +import { readServerState, serverStatePath, type ServerState } from "../../src/cli/lifecycle.ts"; +import type { + NmgMethodResult, + NmgTaskBoardParams, + NmgTaskRunParams, +} from "../../src/cli/protocol.ts"; + +/** + * How long a round's board call may take. + * + * A board verb is a local SQLite write, so seconds are already generous; the point of the bound is + * that a blocked host turns into a named failure in seconds rather than an opaque one in minutes. + */ +export const ROUND_CALL_TIMEOUT_MS = 30_000; + +/** + * The daemon serving this store, or a refusal naming what is missing. + * + * `readServerState` is a read of the lease file, not a spawn: a research round's store is served by + * the round host, and a driver must not start a second daemon on it. + */ +export function roundDaemon(databasePath: string): ServerState { + const state = readServerState(serverStatePath(databasePath)); + if (!state || state.transport !== "http" || !state.host || !state.port || !state.token) { + throw new Error( + `no daemon is serving ${databasePath}: start the round host for that store and pass --daemon ` + + "with the store it serves (a driver does not open a database of its own)", + ); + } + if (state.pid === process.pid) { + throw new Error( + `this process is the one serving ${databasePath}; a host calls the round's entry in-process ` + + "instead of over HTTP, because it cannot answer itself while it is blocked", + ); + } + return state; +} + +/** + * One board write or read, as the daemon's protocol defines it. + * + * The result is the protocol's own union rather than the store's return type, so a driver narrows by + * `action` and gets the same fields a daemon client gets - there is no second shape for the harness. + */ +export async function boardCall( + state: ServerState, + params: NmgTaskBoardParams, + options: { timeoutMs?: number } = {}, +): Promise { + return (await call( + state, + "taskBoard", + params, + options.timeoutMs, + )) as NmgMethodResult["taskBoard"]; +} + +/** + * One transition of a run, as the daemon's protocol defines it. + * + * The round's own record is written through this and not through the store: registering the run, + * freezing its plan and adopting the entries it carries are the runner's acts, and they belong to the + * daemon that owns the store for the same reason the board verbs do. + */ +export async function runCall( + state: ServerState, + params: NmgTaskRunParams, + options: { timeoutMs?: number } = {}, +): Promise { + return (await call(state, "taskRun", params, options.timeoutMs)) as NmgMethodResult["taskRun"]; +} + +async function call( + state: ServerState, + method: "taskBoard" | "taskRun", + params: unknown, + timeoutMs = ROUND_CALL_TIMEOUT_MS, +): Promise { + try { + return await httpCall(state, method, params, { timeoutMs }); + } catch (error) { + if (!(error instanceof TimeoutError)) throw error; + throw new Error( + `${error.message}: ${state.host}:${state.port} is served by pid ${state.pid}, and a host that is ` + + "blocked - a synchronous wait in that process - cannot answer, so check the host rather than " + + "retrying", + { cause: error }, + ); + } +} diff --git a/evals/ooo-execution/round-compare.test.ts b/evals/ooo-execution/round-compare.test.ts deleted file mode 100644 index 726165aa..00000000 --- a/evals/ooo-execution/round-compare.test.ts +++ /dev/null @@ -1,169 +0,0 @@ -// S4: the comparison must be honest about what it measured, so the test pins the two things -// that make it meaningful — the arms are the same round (quality parity), and the control -// really does not overlap the wait. -import assert from "node:assert/strict"; -import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import test from "node:test"; -import type { FrozenPatchWork } from "../../src/integration/ooo-patch.ts"; -import { compareModes, describe, armLog } from "./round-compare.ts"; -import { parseRoundSpec, type RoundSpec } from "./round-spec.ts"; - -const BASELINE = ["evals/ooo-execution/cancellation.test.ts"]; -const CASE_TOKEN = "cancelling a round kills"; - -function specObject(): Record { - return { - revision: "HEAD", - baseline: BASELINE, - checks: [ - { label: "protocol-regression", command: process.execPath, args: ["-e", "process.exit(0)"] }, - ], - a: { instruction: "repair what the check exposes", editable: BASELINE }, - b: { instruction: "add the missing regressions", editable: BASELINE }, - worker: { kind: "replay", log: "unused-in-this-test" }, - noChangeCases: { A: [{ name: "check-identity", token: CASE_TOKEN }] }, - admitted: { - A: ["no-change-needed", "cannot-complete"], - B: ["cannot-complete"], - C: ["promote-candidate", "cannot-complete"], - }, - budget: { perFile: 40_000, output: 40_000 }, - limits: { turns: 2, reads: 2, timeoutMs: 60_000 }, - }; -} - -/** The same answers for both arms: the comparison is about scheduling, so the inputs to the - * scheduler must not differ between them. */ -function answers(task: string, frozen: FrozenPatchWork): string { - if (task === "A") - return JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "no-change-needed", - summary: "the check passes, so there is nothing to repair", - evidence: "cited from the frozen suite", - citations: [ - { - case: "check-identity", - test: "cancelling a round kills its running check and leaves no accepted work", - }, - ], - }); - if (task === "B") - return JSON.stringify({ - digest: frozen.digest, - files: [ - { - path: BASELINE[0]!, - content: `${frozen.work.files[BASELINE[0]!]}\ntest("compare added", () => {});\n`, - }, - ], - }); - return JSON.stringify({ - digest: frozen.digest, - kind: "conclusion", - conclusion: "promote-candidate", - summary: "combined candidate verified by the host", - evidence: "composed-check accept", - citations: [], - }); -} - -test( - "the comparison runs both modes over the same plan and reports what it measured", - { timeout: 180_000 }, - async (t) => { - const directory = mkdtempSync(join(tmpdir(), "nmg-ooo-compare-")); - t.after(() => - rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }), - ); - const spec: RoundSpec = parseRoundSpec(specObject()); - const comparison = await compareModes({ - spec, - repository: process.cwd(), - outputDirectory: directory, - times: 1, - workerFor: async () => async (task, frozen) => answers(task, frozen), - }); - - // Both arms are the same round: same frozen work, same verdicts, same acceptances. Without - // that, the timings would be measuring two different rounds. - assert.equal(comparison.qualityParity, true, JSON.stringify(comparison.differences)); - for (const entry of comparison.arms) - assert.deepEqual(entry.verdicts, { B: "accepted", A: "accepted", C: "accepted" }); - assert.equal(comparison.arms.length, 2); - const ooo = comparison.arms.find((entry) => entry.mode === "ooo")!; - const sequential = comparison.arms.find((entry) => entry.mode === "sequential")!; - assert.equal(sequential.hiddenWaitMs, 0, "the control has nothing to overlap"); - assert.ok(ooo.hostChecks >= 1 && sequential.hostChecks >= 1); - - // The reported lines state both sides and do not declare a winner on their own. - const lines = describe(comparison); - assert.ok(lines.some((line) => line.startsWith("ooo: wall "))); - assert.ok(lines.some((line) => line.startsWith("sequential: wall "))); - assert.ok(lines.some((line) => line.includes("out-of-order minus sequential wall clock"))); - assert.ok( - !lines.some((line) => /significant|proven|better design/i.test(line)), - "the report must not claim a result the data does not carry", - ); - - // A reader of the CI transcript gets the measurements, not only the assertions that they - // exist: `--test-reporter=tap` then carries both arms' numbers out of the run. When the - // on-demand workflow asks for artifact output, the same bytes it asserted on are copied out - // of the test's temporary directory, so the uploaded compare.json is not a re-derivation. - // The accepted bodies are long and are already in the copied arm logs; a transcript is for - // reading, so only the measuring lines are printed. - for (const line of lines) - if (!line.includes(": accepted ")) console.log(`[ooo-compare] ${line}`); - const artifactDirectory = process.env["NMG_OOO_ARTIFACT_DIR"]; - if (artifactDirectory) { - mkdirSync(artifactDirectory, { recursive: true }); - copyFileSync(join(directory, "compare.json"), join(artifactDirectory, "compare.json")); - for (const entry of comparison.arms) - copyFileSync( - join(directory, `${entry.mode}-${entry.run}`, "round.jsonl"), - join(artifactDirectory, `${entry.mode}-${entry.run}.jsonl`), - ); - } - - // Every arm keeps its own store, log and record: a failure sample has to be pointable-at. - for (const entry of comparison.arms) { - const { terminal } = armLog(directory, entry.mode, entry.run); - assert.ok(terminal, `${entry.mode}-${entry.run} has no terminal event`); - assert.deepEqual(terminal.verdicts, entry.verdicts); - } - const written = JSON.parse(readFileSync(join(directory, "compare.json"), "utf8")) as { - arms: unknown[]; - }; - assert.equal(written.arms.length, 2); - }, -); - -test("a comparison whose arms disagree about quality says so instead of comparing times", async (t) => { - const directory = mkdtempSync(join(tmpdir(), "nmg-ooo-compare-skew-")); - t.after(() => - rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }), - ); - const spec: RoundSpec = parseRoundSpec(specObject()); - const comparison = await compareModes({ - spec, - repository: process.cwd(), - outputDirectory: directory, - times: 1, - // The out-of-order arm answers; the control refuses to complete. The verdicts differ, so - // the harness must refuse to treat the timings as comparable. - workerFor: async (mode) => - mode === "ooo" - ? async (task, frozen) => answers(task, frozen) - : async () => { - throw new Error("this arm could not answer"); - }, - }); - assert.equal(comparison.qualityParity, false); - assert.ok( - comparison.differences.some((line) => line.includes("not the same round")), - JSON.stringify(comparison.differences), - ); -}); diff --git a/evals/ooo-execution/round-compare.ts b/evals/ooo-execution/round-compare.ts deleted file mode 100644 index 36a2b393..00000000 --- a/evals/ooo-execution/round-compare.ts +++ /dev/null @@ -1,209 +0,0 @@ -// S4: compare ordered execution with the out-of-order round on the same frozen plan. -// -// The comparison reports what it measured and does not presume a winner. Quality, wall time, -// tokens, host cost, failures and human intervention are all recorded, because a scheduling -// change can trade one against another: this design's claim is about acceptance semantics, and -// the honest question for a comparison is whether it is slower. -// -// Two kinds of comparison, and the difference matters: -// - `replay` worker: the same recorded answers drive both modes, so the token columns are -// equal by construction and what is being measured is *scheduling* — wall clock, host -// checks, how much of the wait the out-of-order task covered. -// - `pi` worker: real model calls in both modes, which is the only way to compare tokens and -// cost. That spends money in both arms, so it is an explicit operator decision. -import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; -import { type CycleResult, type CycleWorker, runCycle } from "../../src/integration/ooo-cycle.ts"; -import { readRoundLog, terminalEvent } from "../../src/integration/ooo-round-log.ts"; -import { cycleOptionsFor, readBaseline, specDigest, specWorker } from "./round-runner.ts"; -import type { RoundSpec } from "./round-spec.ts"; - -export type Mode = "ooo" | "sequential"; - -export interface Arm { - mode: Mode; - run: number; - wallMs: number; - /** Host verification: the cost this design adds, and the term that dominates every round. */ - hostMs: number; - hostChecks: number; - /** The part of the external wait the independent task's own work covered (its claim-to-return - * window, not its later verification): zero for the control. */ - hiddenWaitMs: number; - tokens: number; - cacheRead: number; - checks: number; - verdicts: Record; - accepted: Record; - killed: number; - survived: number; - reopens: number; - failures: number; - cancelled?: string; -} - -export interface Comparison { - specDigest: string; - worker: string; - runs: number; - arms: Arm[]; - /** True when every arm accepted the same tasks. A comparison with different verdicts is - * not a comparison: it is two different rounds, and the harness says so. */ - qualityParity: boolean; - differences: string[]; -} - -function tokensOf(result: CycleResult): { tokens: number; cacheRead: number; checks: number } { - let tokens = 0; - let cacheRead = 0; - let checks = 0; - for (const metrics of Object.values(result.measurements.workers)) { - tokens += metrics.tokens ?? 0; - cacheRead += metrics.cacheRead ?? 0; - checks += metrics.checks ?? 0; - } - return { tokens, cacheRead, checks }; -} - -function arm(mode: Mode, run: number, result: CycleResult, wallMs: number): Arm { - const { tokens, cacheRead, checks } = tokensOf(result); - const killed = Object.values(result.killed).flat().length; - const survived = Object.values(result.survived).flat().length; - return { - mode, - run, - wallMs, - hostMs: result.measurements.hostMs, - hostChecks: result.measurements.hostChecks, - hiddenWaitMs: result.measurements.hiddenWaitMs, - tokens, - cacheRead, - checks, - verdicts: result.verdicts, - accepted: result.accepted, - killed, - survived, - reopens: result.measurements.reopens.length, - failures: result.rejections.length, - ...(result.cancelled ? { cancelled: result.cancelled } : {}), - }; -} - -/** Runs both modes `times` times over one spec and reports the measured differences. - * - * Each arm gets its own run directory, so every arm keeps its own store, log and record: a - * failure sample is evidence, and the comparison has to be able to point at it. */ -export async function compareModes(options: { - spec: RoundSpec; - repository: string; - outputDirectory: string; - times: number; - /** Injected by a test; the CLI resolves it from the spec like `submit` does. */ - workerFor?: (mode: Mode, runDirectory: string) => Promise; -}): Promise { - const { revision, baseline } = readBaseline(options.spec, options.repository); - const arms: Arm[] = []; - const modes: Mode[] = ["ooo", "sequential"]; - for (let run = 1; run <= options.times; run += 1) - for (const mode of modes) { - const runDirectory = join(options.outputDirectory, `${mode}-${run}`); - mkdirSync(runDirectory, { recursive: true }); - const worker = options.workerFor - ? await options.workerFor(mode, runDirectory) - : await specWorker( - options.spec, - { repository: options.repository, revision, baseline }, - { - live: false, - }, - ); - const started = Date.now(); - const result = await runCycle( - cycleOptionsFor({ - spec: options.spec, - repository: options.repository, - revision, - baseline, - worker, - runDirectory, - mode, - }), - ); - arms.push(arm(mode, run, result, Date.now() - started)); - } - // Compared by content, not by insertion order: the two modes accept in different orders, and - // a parity check that called that a quality difference would be measuring the wrong thing. - const canonical = (values: Record) => - JSON.stringify(Object.fromEntries(Object.entries(values).sort())); - const outcomes = new Set( - arms.map((entry) => `${canonical(entry.accepted)}|${canonical(entry.verdicts)}`), - ); - const comparison: Comparison = { - specDigest: specDigest(options.spec), - worker: options.spec.worker.kind, - runs: options.times, - arms, - qualityParity: outcomes.size === 1, - differences: [], - }; - comparison.differences = describe(comparison); - mkdirSync(options.outputDirectory, { recursive: true }); - writeFileSync( - join(options.outputDirectory, "compare.json"), - JSON.stringify(comparison, null, 2), - "utf8", - ); - return comparison; -} - -/** The measured differences in words. Deliberately states both sides: a comparison that only - * reports the winner is not evidence. */ -export function describe(comparison: Comparison): string[] { - const lines: string[] = []; - for (const mode of ["ooo", "sequential"] as const) { - const arms = comparison.arms.filter((entry) => entry.mode === mode); - if (!arms.length) continue; - const sum = (pick: (entry: Arm) => number) => - arms.reduce((total, entry) => total + pick(entry), 0); - const average = (pick: (entry: Arm) => number) => Math.round(sum(pick) / arms.length); - // Quality first: a comparison whose arms disagree about quality is not a comparison. - const verdicts = arms.map((entry) => JSON.stringify(entry.verdicts)); - const accepted = arms.map((entry) => JSON.stringify(entry.accepted)); - const unique = (values: string[]) => [...new Set(values)]; - lines.push(`${mode}: verdicts ${unique(verdicts).join(" | ")}`); - lines.push(`${mode}: accepted ${unique(accepted).join(" | ")}`); - lines.push( - `${mode}: wall ${average((entry) => entry.wallMs)} ms, host ${sum((entry) => entry.hostMs)} ms ` + - `over ${arms.reduce((total, entry) => total + entry.hostChecks, 0)} check(s), ` + - `hidden wait ${average((entry) => entry.hiddenWaitMs)} ms, tokens ${sum((entry) => entry.tokens)}, ` + - `cache read ${sum((entry) => entry.cacheRead)}, worker checks ${sum((entry) => entry.checks)}, ` + - `killed ${sum((entry) => entry.killed)}, survived ${sum((entry) => entry.survived)}, ` + - `failures ${sum((entry) => entry.failures)}, reopens ${sum((entry) => entry.reopens)}`, - ); - } - if (!comparison.qualityParity) - lines.push( - "quality differs between modes: the arms are not the same round and the timings are not comparable", - ); - const ooo = comparison.arms.filter((entry) => entry.mode === "ooo"); - const sequential = comparison.arms.filter((entry) => entry.mode === "sequential"); - if (ooo.length && sequential.length) { - const oooWall = ooo.reduce((total, entry) => total + entry.wallMs, 0) / ooo.length; - const seqWall = - sequential.reduce((total, entry) => total + entry.wallMs, 0) / sequential.length; - const delta = Math.round(oooWall - seqWall); - lines.push( - `out-of-order minus sequential wall clock: ${delta} ms ` + - `(${delta <= 0 ? "faster" : "slower"}) — reported, not treated as a result on its own`, - ); - } - return lines; -} - -/** The recorded log of one arm, for a reader who wants the timeline behind a number. */ -export function armLog(outputDirectory: string, mode: Mode, run: number) { - const events = readRoundLog( - readFileSync(join(outputDirectory, `${mode}-${run}`, "round.jsonl"), "utf8"), - ); - return { events, terminal: terminalEvent(events) }; -} diff --git a/evals/ooo-execution/round-host.ts b/evals/ooo-execution/round-host.ts new file mode 100644 index 00000000..2b97dcd9 --- /dev/null +++ b/evals/ooo-execution/round-host.ts @@ -0,0 +1,105 @@ +/** + * The host half of a round's board: it serves an existing round store over the daemon transport, so + * the drivers - separate processes - reach it as clients instead of opening the file. + * + * This is the same composition the product daemon uses (`NmgService` + `serveHttp` + the store's + * lease), and it is deliberately the *only* writer of that store while it runs: a round that both + * served its store and wrote it from another connection would be the two-writer shape the design + * rules out. + * + * Two ways to run it, and the difference matters: + * + * - `serveRoundStore` hosts in this process, for a caller that also *calls* the round entry + * in-process (`host.call(...)`) - the design's offline-host shape. Such a process must not call + * its own endpoint over HTTP: `round-client.ts` refuses that by the lease's pid, because a blocked + * host cannot answer itself. + * - `--store ` hosts as its own process, which is what a client that wants the wire needs + * (the test that proves the drivers work does exactly this). Its idle timeout is a backstop, so a + * host whose test died still exits instead of holding the store and the lease forever. + * + * `close()` asks the served endpoint to shut down, waits for it, and then closes the service. + */ +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; + +import { httpCall } from "../../src/cli/http-client.ts"; +import { serveHttp } from "../../src/cli/http-server.ts"; +import { + acquireServerLease, + readServerState, + serverStatePath, + type ServerState, +} from "../../src/cli/lifecycle.ts"; +import { NmgService } from "../../src/cli/service.ts"; +import type { NmgMethod, NmgMethodResult } from "../../src/cli/protocol.ts"; + +export interface RoundHost { + databasePath: string; + /** The endpoint as published on the store's lease; throws until it is published. */ + state(): ServerState; + /** This host's own call path: the round entry in-process, never the endpoint it serves. */ + call(method: M, params?: unknown): Promise; + /** Resolves when the endpoint stops - its idle timeout, or a `shutdown` call. */ + closed: Promise; + close(): Promise; +} + +const PUBLISH_TIMEOUT_MS = 10_000; + +export async function serveRoundStore( + databasePath: string, + options: { idleTimeoutMs?: number } = {}, +): Promise { + const service = new NmgService({ databasePath, environment: {} }); + const lease = acquireServerLease(databasePath); + const served = serveHttp(service, lease, { idleTimeoutMs: options.idleTimeoutMs ?? 0 }); + const state = () => { + const current = readServerState(serverStatePath(databasePath)); + if (!current?.port) { + throw new Error(`the round host has not published an endpoint for ${databasePath} yet`); + } + return current; + }; + + // serveHttp publishes the endpoint after it listens, so a caller that raced it would see a lease + // without a port. Wait for the port rather than guess how long listen takes. + const deadline = Date.now() + PUBLISH_TIMEOUT_MS; + while (!readServerState(serverStatePath(databasePath))?.port && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + + return { + databasePath, + state, + call: (method, params) => service.invoke(method, params), + closed: served, + close: async () => { + try { + await httpCall(state(), "shutdown"); + } catch { + // A host whose endpoint is already gone still has to release the service and the lease. + } + await served; + service.close(); + }, + }; +} + +const invokedPath = process.argv[1] ? resolve(process.argv[1]) : ""; +if (invokedPath === fileURLToPath(import.meta.url)) { + const { values } = parseArgs({ + options: { store: { type: "string" }, "idle-ms": { type: "string" } }, + }); + if (!values.store) throw new Error("--store is required: the round store this host serves"); + // A host run as its own process is what a client that wants the wire needs. The idle timeout is the + // backstop: a host whose owner died still exits rather than holding the store and its lease. + const idleMs = Number(values["idle-ms"] ?? 120_000); + const host = await serveRoundStore(values.store, { idleTimeoutMs: idleMs }); + const stop = () => void host.close(); + process.once("SIGINT", stop); + process.once("SIGTERM", stop); + process.stdout.write(`[host] serving ${host.databasePath} on pid ${process.pid}\n`); + await host.closed; + await host.close(); +} diff --git a/evals/ooo-execution/round-runner.ts b/evals/ooo-execution/round-runner.ts deleted file mode 100644 index d67260d3..00000000 --- a/evals/ooo-execution/round-runner.ts +++ /dev/null @@ -1,306 +0,0 @@ -// S3: run a round described by a spec file, in a run directory that carries everything a -// later process needs to query or cancel it. -// -// The run directory is the round's durable surface. `submit` / `status` / `cancel` are the -// three entry points the bootstrap design asks for, and they map onto durable-agent shapes -// that already exist elsewhere (submit a run, query its state, cancel it) instead of onto a -// new hand-written round script. -import { createHash } from "node:crypto"; -import { execFileSync } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { isAbsolute, join } from "node:path"; -import { BoardAdmission, type ProbePlan } from "../../src/integration/ooo-board.ts"; -import { verifyCandidate } from "../../src/integration/ooo-candidate.ts"; -import { - runCycle, - type CycleOptions, - type CycleResult, - type CycleWorker, -} from "../../src/integration/ooo-cycle.ts"; -import { - RoundLog, - readRoundLog, - recordedWorker, - terminalEvent, -} from "../../src/integration/ooo-round-log.ts"; -import type { RoundSpec, SpecWorker } from "./round-spec.ts"; - -/** The plan the round runs under, written into the run directory so a *different* process can - * open the same store and cancel the same round. */ -export const ROUND_PLAN: ProbePlan = [ - ["A", "", [], "isolated-artifact", "protocol-regression", null], - ["B", "", [], "isolated-artifact", null, null], - ["C", "", ["A", "B"], "isolated-artifact", null, null], -]; - -export interface RunRecord { - specDigest: string; - startedAt: string; - finishedAt?: string; - worker: SpecWorker; - revision: string; - verdicts?: Record; - accepted?: Record; - cancelled?: string; - measurements?: CycleResult["measurements"]; - composed?: { verdict: string; files: string[] }; - rejections?: CycleResult["rejections"]; -} - -/** `runCycle` owns `/store.sqlite`; the runner reads the same file, never a copy. */ -export const storePath = (runDir: string) => join(runDir, "store.sqlite"); -export const logPath = (runDir: string) => join(runDir, "round.jsonl"); -export const recordPath = (runDir: string) => join(runDir, "run.json"); - -export function specDigest(spec: RoundSpec): string { - return createHash("sha256").update(JSON.stringify(spec)).digest("hex"); -} - -export function readRecord(runDir: string): RunRecord | null { - if (!existsSync(recordPath(runDir))) return null; - return JSON.parse(readFileSync(recordPath(runDir), "utf8")) as RunRecord; -} - -/** The store the running round owns, opened by another process for `status` / `cancel`. - * Reading never creates one: a directory with no store has no round to report. */ -export function openRoundStore(runDir: string): BoardAdmission { - if (!existsSync(storePath(runDir))) throw new Error(`no round store in ${runDir}`); - return new BoardAdmission(storePath(runDir), ROUND_PLAN, {}); -} - -/** The round's own store, created if this is the round's first moment. The runner needs it - * before `runCycle` starts, because that is the channel an operator's `cancel` arrives on. */ -function ensureRoundStore(runDir: string): BoardAdmission { - mkdirSync(runDir, { recursive: true }); - return new BoardAdmission(storePath(runDir), ROUND_PLAN, {}); -} - -export interface RoundInputs { - repository: string; - revision: string; - baseline: Readonly>; -} - -/** Resolves a spec's worker against the round's own inputs. - * - * `pi` needs an explicit authorization at the call site: a spec file names a provider, but - * the provider boundary is not something a spec gets to cross on its own. */ -export async function specWorker( - spec: RoundSpec, - inputs: RoundInputs, - options: { live: boolean }, -): Promise { - const worker: SpecWorker = spec.worker; - if (worker.kind === "replay") { - // A recorded log may be archived outside the repository, so an absolute path is taken as - // given; a relative one is resolved against the repository like every other spec path. - const path = isAbsolute(worker.log) ? worker.log : join(inputs.repository, worker.log); - return recordedWorker(readRoundLog(readFileSync(path, "utf8"))); - } - if (!options.live) - throw new Error( - "refusing a live model round without --live: the spec names the provider, the operator " + - "authorizes the spend", - ); - const { executePiPatch } = await import("../../.pi/extensions/nmg/ooo-execution.ts"); - const checkTool = { - label: "run_check", - maxRuns: spec.checkRuns ?? 4, - run: async (files: { path: string; content: string }[]) => { - const result = await verifyCandidate({ - repository: inputs.repository, - revision: inputs.revision, - files: { - ...inputs.baseline, - ...Object.fromEntries(files.map((file) => [file.path, file.content])), - }, - checks: [...spec.checks], - }); - const failed = result.outcomes.find((outcome) => outcome.status !== "passed"); - return { - verdict: result.verdict, - log: - result.outcomes.map((outcome) => `${outcome.label}=${outcome.status}`).join(", ") + - (failed?.log ? ` | ${failed.log.slice(-2_000)}` : ""), - }; - }, - }; - return async (_taskId, frozen) => { - const run = await executePiPatch(frozen, worker.provider, worker.model, { - check: checkTool, - }); - const metrics = { tokens: run.tokens, turns: run.turns, checks: run.checks }; - return run.pushback - ? { artifact: "", pushback: run.pushback, metrics } - : { artifact: run.artifact, metrics }; - }; -} - -/** Reads the round's frozen baseline from the repository, at the spec's revision. */ -export function readBaseline( - spec: RoundSpec, - repository: string, -): { revision: string; baseline: Record } { - const revision = - spec.revision === "HEAD" - ? execFileSync("git", ["rev-parse", "HEAD"], { cwd: repository, encoding: "utf8" }).trim() - : spec.revision; - return { - revision, - baseline: Object.fromEntries( - spec.baseline.map((path) => [path, readFileSync(join(repository, path), "utf8")]), - ), - }; -} - -/** The round options a spec defines. One home for the mapping, so `submit` and the S4 - * comparison cannot drift into running subtly different rounds from the same spec. */ -export function cycleOptionsFor(options: { - spec: RoundSpec; - repository: string; - revision: string; - baseline: Readonly>; - worker: CycleWorker; - runDirectory: string; - mode?: "ooo" | "sequential"; - signal?: AbortSignal; - watchCancellation?: () => string | null; -}): CycleOptions { - const { spec } = options; - return { - repository: options.repository, - revision: options.revision, - baseline: options.baseline, - checks: spec.checks, - worker: options.worker, - mode: options.mode ?? "ooo", - aInstruction: spec.a.instruction, - bInstruction: spec.b.instruction, - aEditable: spec.a.editable, - bEditable: spec.b.editable, - budget: spec.budget ?? { perFile: 24_000, output: 48_000 }, - limits: spec.limits ?? { turns: 10, reads: 6, timeoutMs: 240_000 }, - roundLog: new RoundLog(logPath(options.runDirectory)), - databaseDir: options.runDirectory, - ...(options.watchCancellation ? { watchCancellation: options.watchCancellation } : {}), - ...(options.signal ? { signal: options.signal } : {}), - ...(spec.noChangeCases ? { noChangeCases: spec.noChangeCases } : {}), - ...(spec.mutations ? { mutations: spec.mutations } : {}), - ...(spec.visible ? { visible: spec.visible } : {}), - ...(spec.admitted ? { admitted: spec.admitted } : {}), - ...(spec.requires ? { requires: spec.requires } : {}), - ...(spec.maxReopens === undefined ? {} : { maxReopens: spec.maxReopens }), - }; -} - -/** Runs one round from its spec, writing the log and a durable record as it goes. The - * running round polls the store, so `cancelRun` from another process reaches it. */ -export async function runSpecifiedRound(options: { - spec: RoundSpec; - runDir: string; - repository: string; - /** Resolved once by the caller (or injected by a test). */ - worker: CycleWorker; - revision: string; - baseline: Readonly>; - signal?: AbortSignal; -}): Promise { - const existing = readRecord(options.runDir); - if (existing?.finishedAt) - throw new Error(`${options.runDir} already holds a finished round; use a new run directory`); - mkdirSync(options.runDir, { recursive: true }); - const started: RunRecord = { - specDigest: specDigest(options.spec), - startedAt: new Date().toISOString(), - worker: options.spec.worker, - revision: options.revision, - }; - writeFileSync(recordPath(options.runDir), JSON.stringify(started, null, 2), "utf8"); - const store = ensureRoundStore(options.runDir); - try { - const result = await runCycle( - cycleOptionsFor({ - spec: options.spec, - repository: options.repository, - revision: options.revision, - baseline: options.baseline, - worker: options.worker, - runDirectory: options.runDir, - watchCancellation: () => store.cancelled(), - ...(options.signal ? { signal: options.signal } : {}), - }), - ); - writeFileSync( - recordPath(options.runDir), - JSON.stringify( - { - ...started, - finishedAt: new Date().toISOString(), - verdicts: result.verdicts, - accepted: result.accepted, - composed: { verdict: result.composed.verdict, files: [...result.composed.files] }, - measurements: result.measurements, - rejections: result.rejections, - ...(result.cancelled ? { cancelled: result.cancelled } : {}), - } satisfies RunRecord, - null, - 2, - ), - "utf8", - ); - return result; - } finally { - store.close(); - } -} - -/** `status` for a run directory: the durable record, plus the store's own view, so a round - * cancelled by another process is reported even before its process writes anything. */ -export function describeRun(runDir: string): string { - const record = readRecord(runDir); - const lines: string[] = [`run directory: ${runDir}`]; - if (!record) lines.push("state: no run recorded here"); - else { - lines.push(`worker: ${record.worker.kind}`); - lines.push(`revision: ${record.revision.slice(0, 12)}`); - lines.push(`started: ${record.startedAt}`); - lines.push(record.finishedAt ? `finished: ${record.finishedAt}` : "state: not finished"); - if (record.cancelled) lines.push(`cancelled: ${record.cancelled}`); - if (record.verdicts) lines.push(`verdicts: ${JSON.stringify(record.verdicts)}`); - if (record.accepted) lines.push(`accepted: ${JSON.stringify(record.accepted)}`); - if (record.measurements) - lines.push( - `measurements: host ${record.measurements.hostMs} ms over ` + - `${record.measurements.hostChecks} check(s), hidden wait ` + - `${record.measurements.hiddenWaitMs} ms, reopens ${record.measurements.reopens.length}`, - ); - } - if (existsSync(logPath(runDir))) { - const events = readRoundLog(readFileSync(logPath(runDir), "utf8")); - lines.push(`log events: ${events.length}`); - const terminal = terminalEvent(events); - if (terminal) lines.push(`log terminal: ${JSON.stringify(terminal.verdicts)}`); - } - if (existsSync(storePath(runDir))) { - const store = openRoundStore(runDir); - try { - lines.push(`coordinator cancelled: ${store.cancelled() ?? "no"}`); - lines.push(`coordinator accepted: ${JSON.stringify(store.accepted())}`); - } finally { - store.close(); - } - } - return lines.join("\n"); -} - -/** Records the operator's cancellation in the round's own store. The running round polls that - * store, so this reaches a round this process does not own. */ -export function cancelRun(runDir: string, reason: string): string { - const store = openRoundStore(runDir); - try { - store.cancel(reason); - return store.cancelled() ?? reason; - } finally { - store.close(); - } -} diff --git a/evals/ooo-execution/round-spec.ts b/evals/ooo-execution/round-spec.ts deleted file mode 100644 index 37f54994..00000000 --- a/evals/ooo-execution/round-spec.ts +++ /dev/null @@ -1,260 +0,0 @@ -// S3: a round is described by data, not by editing a research script. The spec is the round's -// declared input; the runner freezes it, and everything the round hands a worker is -// digest-bound, so a spec that changes the verifier or the baseline is a different round and a -// replay refuses it by name (round-log.ts compareFrozen). -// -// Parsing is fail-closed: an unknown key, a missing required key, an absolute or escaping -// path, or a worker kind that is not supported is a named error, never a silently ignored -// field. A spec that is half-understood would run a round nobody declared. -import type { CandidateCheck } from "../../src/integration/ooo-candidate.ts"; -import type { ConclusionKind } from "../../src/integration/ooo-patch.ts"; -import type { CaseRule, Requirement } from "../../src/integration/ooo-cycle.ts"; -import type { Mutation } from "../../src/integration/ooo-mutation.ts"; - -export type SpecWorker = - | { kind: "pi"; provider: string; model: string } - /** Replays a recorded round: no model call, the same host checks. */ - | { kind: "replay"; log: string }; - -export interface RoundSpec { - revision: string; - baseline: readonly string[]; - checks: readonly CandidateCheck[]; - a: { instruction: string; editable: readonly string[]; visible?: readonly string[] }; - b: { instruction: string; editable: readonly string[]; visible?: readonly string[] }; - worker: SpecWorker; - noChangeCases?: Partial>; - mutations?: Partial>; - visible?: Partial>; - admitted?: Partial>; - requires?: Partial>; - budget?: { perFile: number; output: number }; - limits?: { turns: number; reads: number; timeoutMs: number }; - maxReopens?: number; - checkRuns?: number; -} - -const KEYS = [ - "revision", - "baseline", - "checks", - "a", - "b", - "worker", - "noChangeCases", - "mutations", - "visible", - "admitted", - "requires", - "budget", - "limits", - "maxReopens", - "checkRuns", -]; - -/** Repository-relative, no escape: a spec may not name a path outside the frozen tree. */ -function relativePath(value: unknown, field: string): string { - if (typeof value !== "string" || !value.trim()) throw new Error(`${field} must be a path`); - const path = value.trim(); - if ( - path.startsWith("/") || - /^[a-zA-Z]:/.test(path) || - path.includes("\\") || - path.split("/").some((part) => !part || part === "." || part === "..") - ) - throw new Error(`${field} must be a repository-relative path without escapes: ${path}`); - return path; -} - -function paths(value: unknown, field: string, required: boolean): string[] { - if (value === undefined && !required) return []; - if (!Array.isArray(value) || !value.length) throw new Error(`${field} must be a non-empty array`); - return value.map((item, index) => relativePath(item, `${field}[${index}]`)); -} - -function stringField(value: unknown, field: string): string { - if (typeof value !== "string" || !value.trim()) throw new Error(`${field} must be a string`); - return value; -} - -function positive(value: unknown, field: string): number { - if (!Number.isSafeInteger(value) || Number(value) <= 0) - throw new Error(`${field} must be a positive integer`); - return Number(value); -} - -function object(value: unknown, field: string): Record { - if (typeof value !== "object" || value === null || Array.isArray(value)) - throw new Error(`${field} must be an object`); - return value as Record; -} - -function rejectUnknown(value: Record, allowed: string[], field: string): void { - for (const key of Object.keys(value)) - if (!allowed.includes(key)) throw new Error(`${field} has an unknown key: ${key}`); -} - -function parseWorker(value: unknown): SpecWorker { - const worker = object(value, "worker"); - const kind = stringField(worker.kind, "worker.kind"); - if (kind === "pi") { - rejectUnknown(worker, ["kind", "provider", "model"], "worker"); - return { - kind: "pi", - provider: stringField(worker.provider, "worker.provider"), - model: stringField(worker.model, "worker.model"), - }; - } - if (kind === "replay") { - rejectUnknown(worker, ["kind", "log"], "worker"); - // A recorded log is a reference to an artifact, not part of the frozen envelope, so it may - // live outside the repository (a temp directory, an archived run). The paths that *are* - // frozen stay repository-relative, because those have to mean the same thing on any machine. - return { kind: "replay", log: stringField(worker.log, "worker.log") }; - } - throw new Error(`worker.kind must be "pi" or "replay", got ${kind}`); -} - -function parseTask(value: unknown, field: string) { - const task = object(value, field); - rejectUnknown(task, ["instruction", "editable", "visible"], field); - const visible = task.visible === undefined ? null : paths(task.visible, `${field}.visible`, true); - return { - instruction: stringField(task.instruction, `${field}.instruction`), - editable: paths(task.editable, `${field}.editable`, true), - ...(visible ? { visible } : {}), - }; -} - -function parseChecks(value: unknown): CandidateCheck[] { - if (!Array.isArray(value) || !value.length) throw new Error("checks must be a non-empty array"); - return value.map((item, index) => { - const check = object(item, `checks[${index}]`); - rejectUnknown(check, ["label", "command", "args"], `checks[${index}]`); - const args = check.args; - if (!Array.isArray(args) || args.some((arg) => typeof arg !== "string")) - throw new Error(`checks[${index}].args must be an array of strings`); - return { - label: stringField(check.label, `checks[${index}].label`), - command: stringField(check.command, `checks[${index}].command`), - args: args as string[], - }; - }); -} - -function perTask( - value: unknown, - field: string, - parse: (item: unknown, key: string) => T, -): Partial> { - const source = object(value, field); - const result: Record = {}; - for (const [key, item] of Object.entries(source)) { - if (!["A", "B", "C"].includes(key)) throw new Error(`${field} has an unknown task: ${key}`); - result[key] = parse(item, `${field}.${key}`); - } - return result as Partial>; -} - -function parseCases(value: unknown, key: string): readonly CaseRule[] { - if (!Array.isArray(value) || !value.length) throw new Error(`${key} must be a non-empty array`); - return value.map((item, index) => { - const rule = object(item, `${key}[${index}]`); - rejectUnknown(rule, ["name", "token"], `${key}[${index}]`); - return { - name: stringField(rule.name, `${key}[${index}].name`), - token: stringField(rule.token, `${key}[${index}].token`), - }; - }); -} - -function parseMutations(value: unknown, key: string): readonly Mutation[] { - if (!Array.isArray(value) || !value.length) throw new Error(`${key} must be a non-empty array`); - return value.map((item, index) => { - const mutation = object(item, `${key}[${index}]`); - rejectUnknown(mutation, ["id", "path", "from", "to"], `${key}[${index}]`); - return { - id: stringField(mutation.id, `${key}[${index}].id`), - path: relativePath(mutation.path, `${key}[${index}].path`), - from: stringField(mutation.from, `${key}[${index}].from`), - to: typeof mutation.to === "string" ? mutation.to : "", - }; - }); -} - -function parseAdmitted(value: unknown, key: string): readonly string[] { - if (!Array.isArray(value) || !value.length) throw new Error(`${key} must be a non-empty array`); - return value.map((item, index) => stringField(item, `${key}[${index}]`)); -} - -function parseRequirements(value: unknown, key: string): readonly Requirement[] { - if (!Array.isArray(value) || !value.length) throw new Error(`${key} must be a non-empty array`); - return value.map((item, index) => { - const requirement = object(item, `${key}[${index}]`); - rejectUnknown(requirement, ["kind", "task", "id", "token"], `${key}[${index}]`); - if (requirement.id === undefined && requirement.token === undefined) - throw new Error(`${key}[${index}] needs an id or a token`); - return { - kind: stringField(requirement.kind, `${key}[${index}].kind`), - task: stringField(requirement.task, `${key}[${index}].task`), - ...(requirement.id === undefined ? {} : { id: String(requirement.id) }), - ...(requirement.token === undefined ? {} : { token: String(requirement.token) }), - } as Requirement; - }); -} - -/** Parses a round spec, refusing anything it does not fully understand. */ -export function parseRoundSpec(value: unknown): RoundSpec { - const spec = object(value, "spec"); - rejectUnknown(spec, KEYS, "spec"); - const budget = spec.budget === undefined ? null : object(spec.budget, "budget"); - if (budget) rejectUnknown(budget, ["perFile", "output"], "budget"); - const limits = spec.limits === undefined ? null : object(spec.limits, "limits"); - if (limits) rejectUnknown(limits, ["turns", "reads", "timeoutMs"], "limits"); - return { - revision: spec.revision === undefined ? "HEAD" : stringField(spec.revision, "revision"), - baseline: paths(spec.baseline, "baseline", true), - checks: parseChecks(spec.checks), - a: parseTask(spec.a, "a"), - b: parseTask(spec.b, "b"), - worker: parseWorker(spec.worker), - ...(spec.noChangeCases === undefined - ? {} - : { noChangeCases: perTask(spec.noChangeCases, "noChangeCases", parseCases) }), - ...(spec.mutations === undefined - ? {} - : { mutations: perTask(spec.mutations, "mutations", parseMutations) }), - ...(spec.visible === undefined - ? {} - : { - visible: perTask(spec.visible, "visible", (item, key) => paths(item, key, true)), - }), - ...(spec.admitted === undefined - ? {} - : { admitted: perTask(spec.admitted, "admitted", parseAdmitted) }), - ...(spec.requires === undefined - ? {} - : { requires: perTask(spec.requires, "requires", parseRequirements) }), - ...(budget - ? { - budget: { - perFile: positive(budget.perFile, "budget.perFile"), - output: positive(budget.output, "budget.output"), - }, - } - : {}), - ...(limits - ? { - limits: { - turns: positive(limits.turns, "limits.turns"), - reads: positive(limits.reads, "limits.reads"), - timeoutMs: positive(limits.timeoutMs, "limits.timeoutMs"), - }, - } - : {}), - ...(spec.maxReopens === undefined - ? {} - : { maxReopens: positive(spec.maxReopens, "maxReopens") }), - ...(spec.checkRuns === undefined ? {} : { checkRuns: positive(spec.checkRuns, "checkRuns") }), - } as RoundSpec; -} diff --git a/evals/ooo-execution/speculation-pilot.ts b/evals/ooo-execution/speculation-pilot.ts new file mode 100644 index 00000000..a36dced4 --- /dev/null +++ b/evals/ooo-execution/speculation-pilot.ts @@ -0,0 +1,344 @@ +// E arm (bounded speculation), the first fidelity instrument for the design's lifecycle: one declared +// fact - whether this round needs the unit at all - one candidate prepared ahead of it, and the shared +// layer's own three outcomes deciding what happens to that candidate. +// +// The arm compares two things for each value of the fact, because the design asks for latency, extra +// cost and quality separately rather than as one "gain": +// +// baseline : the fact is decided first; if it is true, the unit runs then. +// speculation : the unit is prepared before the fact; when the fact holds, the prepared candidate is +// published (the host still verifies it - that is the quality term), and when it does +// not, `speculationOutcome` discards the candidate and its branch session is closed. +// +// `--live` is required and `PI_PROVIDER`/`PI_MODEL` must be named: the operator authorizes the spend. +// A published candidate is verified by running the unit's own frozen check against it, in a copy of the +// fixture directory, so the quality term is a real check result and not the model's own claim. +import { cpSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; + +import { patchCandidate, preparePatchWork } from "../../src/integration/ooo-patch.ts"; +import { + speculationOutcome, + type ResolvedPredicate, + type SpeculationCandidate, +} from "../../src/integration/ooo-execution.ts"; +import { + createPiSessionRunner, + patchSessionInput, +} from "../../.pi/extensions/nmg/ooo-execution.ts"; + +/** A named provider and model, refused by name rather than defaulted: the operator names the spend. */ +function required(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`Set ${name} explicitly`); + return value; +} + +if (!process.argv.includes("--live")) + throw new Error("pass --live explicitly: this calls the configured model"); +const provider = required("PI_PROVIDER"); +const model = required("PI_MODEL"); +const reps = Number(process.env.E_REPS ?? 2); +const outDir = `.temp/e-arm`; + +const FIXTURE = "evals/ooo-execution/fixtures/report"; +const TARGET = `${FIXTURE}/beta.ts`; +const INSTRUCTION = + "Implement betaSection in this directory so that beta.test.ts passes. It numbers the rows from one " + + 'in the order given, and describes itself with id "beta" and title "Beta". The interface file is ' + + "frozen: do not change its shape, and do not edit any other file."; + +/** The fact this arm guesses, and its evidence: the host's own answer, not the worker's. */ +const PREDICATE = "this-round-needs-beta"; + +mkdirSync(outDir, { recursive: true }); +/** Everything this run produced, kept whether or not the run goes well. Marked cleanable rather than + * thrown away: scratch is fine, forgetting is not. */ +const runId = new Date().toISOString().replace(/[:.]/g, "-"); +const evidenceDir = `${outDir}/evidence/${runId}`; +mkdirSync(evidenceDir, { recursive: true }); +writeFileSync( + `${evidenceDir}/CLEANABLE-scratch-${runId.slice(0, 10)}.md`, + [ + "# Scratch - cleanable", + "", + "This directory holds one paid run's own evidence: every attempt's artifact bytes, the candidate", + "tree each check ran in, and the check's output.", + "", + "It is scratch and may be deleted once a record quotes its numbers - the run's report is copied", + "into `docs/experiments/execution/archive/ooo-arms-2026-09-19/speculation-earm/` when it is", + "quoted. It is *not* deleted at the end of the run: that is what made an earlier run's quality", + "failures unexplainable.", + ].join("\n") + "\n", +); +const digestOf = (value: string) => createHash("sha256").update(value).digest("hex").slice(0, 16); +const interfaceDigest = digestOf(readFileSync(`${FIXTURE}/interface.ts`, "utf8")); + +/** The frozen work for one attempt, always under a fresh ticket: a discarded branch's ticket is never + * the real path's ticket, which is the design's "真实路径在新票据下重新执行". */ +function frozenFor(label: string) { + return preparePatchWork({ + taskId: `e-arm:${label}:${randomUUID()}`, + attempt: 1, + instruction: INSTRUCTION, + files: { [TARGET]: readFileSync(TARGET, "utf8") }, + editable: [TARGET], + // Fixed for every condition, and above the host default of three turns: a real attempt routinely + // needs four, which is the same envelope fact the D arm recorded. + limits: { turns: 6, reads: 3, timeoutMs: 120_000 }, + }); +} + +/** The quality term: run the unit's own frozen check against the candidate, in a copy of the fixture so + * the shared tree is never written to. Returns the check's own verdict. */ +function verify( + frozen: ReturnType, + artifact: string, + label: string, +): { + ok: boolean; + kind: string; + detail: string; + candidatePath?: string; + checkMs?: number; +} { + // A candidate that does not even parse as this work's envelope is a failed attempt with a reason, + // which is the same rule the adapter applies: the pilot records it instead of crashing on it. + const parsed = JSON.parse(artifact) as { kind?: string }; + if (parsed.kind === "conclusion") + // A conclusion is a legitimate artifact for a task whose rule admits one, and this unit's check + // cannot pass without files - which is also what the board would decide. Reported as its own kind + // rather than as a reader's complaint, which is what the first run of this arm got wrong. + return { + ok: false, + kind: "conclusion", + detail: "a conclusion carries no files: the unit's check cannot pass on it", + }; + let candidate: Readonly>; + try { + candidate = patchCandidate(frozen, artifact); + } catch (error) { + let keys: string; + try { + keys = Object.keys(JSON.parse(artifact)).join(","); + } catch { + keys = "not json"; + } + return { + ok: false, + kind: "unreadable", + detail: `${(error as Error).message} (artifact keys: ${keys})`, + }; + } + const text = candidate[TARGET]; + if (text === undefined) + return { ok: false, kind: "patch", detail: "the artifact names no editable file" }; + const dir = `${evidenceDir}/candidate-${label}-${randomUUID().slice(0, 8)}`; + const checkStartedAt = Date.now(); + const checkMs = () => Date.now() - checkStartedAt; + cpSync(FIXTURE, dir, { recursive: true }); + writeFileSync(`${dir}/beta.ts`, text); + try { + execFileSync( + process.execPath, + ["--experimental-strip-types", "--test", `${dir}/beta.test.ts`], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + return { + ok: true, + kind: "patch", + detail: "the unit's own check passed", + candidatePath: dir, + checkMs: checkMs(), + }; + } catch (error) { + const failure = error as { stdout?: string; stderr?: string }; + return { + ok: false, + kind: "patch", + candidatePath: dir, + checkMs: checkMs(), + detail: `${failure.stdout ?? ""}${failure.stderr ?? ""}`.slice(-2000), + }; + } finally { + // The candidate tree is kept on purpose: a failed check is the evidence for why the quality term is + // false, and the first run of this arm deleted it and could not say what went wrong. + } +} + +async function attempt(label: string): Promise<{ + frozen: ReturnType; + frozenDigest: string; + interfaceDigest: string; + kind: string | undefined; + tokens: number; + turns: number; + workMs: number; + artifact?: string; + artifactPath?: string; + failure?: string; +}> { + const frozen = frozenFor(label); + const startedAt = Date.now(); + const runner = await createPiSessionRunner({ + provider, + modelId: model, + patchMode: true, + first: patchSessionInput(frozen), + }); + try { + const run = await runner.runUnit(patchSessionInput(frozen)); + const artifactPath = `${evidenceDir}/${label}-artifact.json`; + if (run.artifact) writeFileSync(artifactPath, run.artifact); + const kind = run.artifact + ? ((JSON.parse(run.artifact) as { kind?: string; files?: unknown[] }).kind ?? + (Array.isArray((JSON.parse(run.artifact) as { files?: unknown[] }).files) + ? "patch" + : "unknown")) + : undefined; + return { + frozen, + frozenDigest: frozen.digest, + interfaceDigest, + kind, + tokens: run.tokens, + turns: run.turns, + workMs: Date.now() - startedAt, + ...(run.artifact ? { artifact: run.artifact, artifactPath } : { failure: "no artifact" }), + }; + } catch (error) { + // One attempt that misses its bounded contract is a recorded outcome of this arm, not a reason to + // abandon the run: the design asks what a wrong guess costs, and a failed attempt is part of that. + return { + frozen, + frozenDigest: frozen.digest, + interfaceDigest, + kind: undefined, + tokens: 0, + turns: 0, + workMs: Date.now() - startedAt, + failure: (error as Error).message.slice(0, 200), + }; + } finally { + runner.dispose(); + } +} + +const runs: Record[] = []; +for (const factHolds of [true, false]) { + for (let rep = 1; rep <= reps; rep += 1) { + // baseline: the fact first, so nothing is prepared for a round that does not need the unit. + const baseline = factHolds ? await attempt(`baseline-${rep}`) : undefined; + const baselineChecked = + baseline?.artifact && baseline.frozen + ? verify(baseline.frozen, baseline.artifact, `baseline-${rep}`) + : null; + const baselineRow = { + arm: "baseline", + fact: factHolds, + rep, + runId, + tokens: baseline?.tokens ?? 0, + turns: baseline?.turns ?? 0, + workMs: baseline?.workMs ?? 0, + postFactMs: baseline?.workMs ?? 0, + ...(baseline + ? { + frozenDigest: baseline.frozenDigest, + interfaceDigest: baseline.interfaceDigest, + artifactKind: baseline.kind ?? null, + ...(baseline.artifactPath ? { artifactPath: baseline.artifactPath } : {}), + } + : {}), + quality: baselineChecked?.ok ?? null, + qualityKind: baselineChecked?.kind ?? null, + ...(baselineChecked?.candidatePath ? { candidatePath: baselineChecked.candidatePath } : {}), + ...(baselineChecked?.checkMs !== undefined ? { checkMs: baselineChecked.checkMs } : {}), + ...(baselineChecked && !baselineChecked.ok ? { qualityDetail: baselineChecked.detail } : {}), + ...(baseline?.failure ? { failure: baseline.failure } : {}), + }; + + // speculation: the candidate first, then the shared rule decides what the fact means for it. + const prepared = await attempt(`speculation-${rep}`); + const assumption: SpeculationCandidate = { + taskId: `speculation-${rep}`, + assumptions: [{ predicateId: PREDICATE, version: "v1", expected: "true" }], + speculativeSuccessors: [], + irreversibleOperations: [], + }; + const evidence: ResolvedPredicate[] = [ + { predicateId: PREDICATE, version: "v1", value: String(factHolds), authoritative: true }, + ]; + const decision = speculationOutcome(assumption, evidence); + const published = decision.outcome === "publish" && prepared.artifact !== undefined; + const verifyStartedAt = Date.now(); + const quality = published + ? verify(prepared.frozen, prepared.artifact!, `speculation-${rep}`) + : null; + // A published candidate still has to cross the host boundary: latency saved is the work, not the check. + const postFactMs = published ? Date.now() - verifyStartedAt : 0; + const speculationRow = { + arm: "speculation", + fact: factHolds, + rep, + runId, + outcome: decision.outcome, + sessionReusable: decision.sessionReusable, + ticketId: assumption.taskId, + tokens: prepared.tokens, + turns: prepared.turns, + workMs: prepared.workMs, + frozenDigest: prepared.frozenDigest, + interfaceDigest: prepared.interfaceDigest, + artifactKind: prepared.kind ?? null, + ...(prepared.artifactPath ? { artifactPath: prepared.artifactPath } : {}), + // What is left after the fact is decided: a published candidate still has to be verified, and a + // discarded one has nothing left to do because the round does not need the unit. + postFactMs, + quality: quality?.ok ?? null, + qualityKind: quality?.kind ?? null, + ...(quality?.candidatePath ? { candidatePath: quality.candidatePath } : {}), + ...(quality?.checkMs !== undefined ? { checkMs: quality.checkMs } : {}), + ...(quality && !quality.ok ? { qualityDetail: quality.detail } : {}), + ...(prepared.failure ? { failure: prepared.failure } : {}), + }; + for (const row of [baselineRow, speculationRow]) { + runs.push(row); + writeFileSync(`${evidenceDir}/row-${runs.length}.json`, `${JSON.stringify(row)}\n`); + console.log(JSON.stringify(row)); + } + } +} + +const summarise = (arm: string, fact: boolean) => { + const rows = runs.filter((row) => row.arm === arm && row.fact === fact); + const total = (key: string) => + rows.reduce((sum, row) => sum + Number((row as Record)[key] ?? 0), 0); + return { + arm, + fact, + runs: rows.length, + tokens: total("tokens"), + workMs: total("workMs"), + postFactMs: total("postFactMs"), + quality: rows.map((row) => row.quality), + }; +}; +const aggregate = [true, false].flatMap((fact) => [ + summarise("baseline", fact), + summarise("speculation", fact), +]); +writeFileSync( + `${evidenceDir}/aggregate.json`, + `${JSON.stringify({ provider, model, aggregate }, null, 2)}\n`, +); +try { + cpSync(evidenceDir, `${outDir}/published`, { recursive: true }); +} catch { + // The copy is a convenience for the record, not part of the measurement. +} +console.log(`evidence: ${evidenceDir}`); +console.log("AGGREGATE"); +console.log(JSON.stringify(aggregate, null, 2)); diff --git a/package.json b/package.json index d7d506e6..6258b030 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,6 @@ "files": [ ".pi/extensions/nmg/controller-shadow.ts", ".pi/extensions/nmg/index.ts", - ".pi/extensions/nmg/ooo-round.ts", "bin/nmg-rcp.mjs", "bin/nmg.mjs", "dist/", @@ -126,7 +125,6 @@ "agent:context:check": "node --experimental-strip-types tools/repo-context.ts --check", "agent:verify": "node --experimental-strip-types tools/agent-verify.ts", "ci:uncovered-tests": "node --experimental-strip-types tools/ci-uncovered-tests.ts --check", - "ooo:round": "node --experimental-strip-types evals/ooo-execution/round-cli.ts", "complexity:gate": "node --experimental-strip-types tools/complexity-gate.ts", "mutation:teeth": "node --experimental-strip-types tools/mutation-teeth.ts", "verify:packages": "node --experimental-strip-types tools/verify-packages.ts", diff --git a/skills/repo-development/SKILL.md b/skills/repo-development/SKILL.md index 051e349e..4121bfe9 100644 --- a/skills/repo-development/SKILL.md +++ b/skills/repo-development/SKILL.md @@ -26,6 +26,8 @@ Complexity must be kept within a limit of 15. Automatically remove only generated, cached, or explicitly expiring material; propose reviewable candidates before deleting source, history, public interfaces, issues, or pull requests. +- Keep the repository's spending bounded: never invoke live LLM, embedding, or + full benchmark workloads unless the task explicitly calls for them. - Keep one authoritative writer for each fact. Observe or reference GitHub, Contracts, repository state, verification receipts, and NMG memory through their owning interfaces rather than mirroring them into a competing store. @@ -38,7 +40,7 @@ Complexity must be kept within a limit of 15. that silently swallows another Agent's working tree is a coordination failure, not a merge. When the change lands or is abandoned, remove the dedicated worktree and its branch (`git worktree remove ` then `git branch -D - `) so used-up worktrees do not accumulate. +`) so used-up worktrees do not accumulate. ## Before editing @@ -104,18 +106,72 @@ or remove it when its exit criteria are met. In a shared dirty worktree, pass `-- ` so unrelated changes stay outside the plan. Use `--include-advisory` only when research or chaos cost is intentional. -4. Use `npm run test:research` only for research adapters; use `npm run test:chaos` for explicit lifecycle +4. Run long checks detached and collect them before the commit. The cheap decisive + checks — LSP diagnostics on touched files, format, lint, `complexity:gate`, + `docs:check` — stay synchronous; the expensive three — full `mutation:teeth`, + `test:product`, the `evals/**` suites — are launched as soon as the code they measure + is final, while the documentation and the cheap lane are written: + + ```bash + { echo "started=$(date -Is)"; echo "snapshot=$(git rev-parse HEAD)"; } > .temp/x.state + (npm run X > .temp/x.log 2>&1; echo "exit=$?" >> .temp/x.state) & + echo "pid=$!" >> .temp/x.state + ``` + + Each launch is its own statement, header in the foreground: `A && B &` backgrounds the + whole list, which truncates the state file after its `pid=` line — the one state that + cannot be read. + + A launch returns immediately and is never followed by a wait — no `sleep`, no poll loop, + no peeking to see whether the state is worth reading yet: the collector reads it at the + next natural checkpoint while the time goes to the rest of the change. With nothing else + to do, run the check synchronously instead. + + **Read a detached run's state as three outcomes, not two.** _Finished:_ the state file + carries `exit=`, and that code is the result. _Still running:_ no `exit=` yet and + the recorded pid is alive (`kill -0 ` from a later shell); elapsed time is not a + state, so a long runtime is never read as a failure, and telling working from wedged uses + the check's own progress rather than the clock. What that progress *is* depends on the + check, and the two are not interchangeable: an npm test lane streams its TAP (one line per + test), while `mutation:teeth` writes its summary only at the end — its live signal is the + lock file's `target`, which moves as it takes each target, plus the target file's mtime as + it substitutes and restores. Measuring progress by the wrong one reads a working sweep as + wedged. + _Died:_ no `exit=` and the pid is gone, so nothing wrote a code — the run was killed, and + it may have left a mutant in the tree; the lock says which target, and `git diff` it first. + + A detached run records the tree it measured, and the collector compares that with the + current one: a code change means re-run. Never edit or stage a file a mutation run is + rewriting — a running run shows its current target as modified in `git status`, holding a + live mutant, and anything staged then is the mutant, not the work — and scoped `--targets=` + runs during a change and one full run before a push answer different questions. The measured + costs and traps are in + [the decision](../../docs/decisions/implemented/2026-09-18-detached-long-checks.md). + + **A running sweep makes the tree unreadable, not only unwritable.** Between its substitution + and its restore the target file **is** the mutant, so `lint`, `complexity:gate` and any other + check run in that window report on the mutant — and a check that *passes* there is evidence + about code that never existed. The sweep now says so in the tree (`.temp/mutation-lock.json`, + `tools/mutation-lock.ts`), `npm run agent:verify` refuses while it is held rather than + reporting on a mutant, and `npm run agent:context` prints it. Never re-derive that from memory: + the whole failure class is [post-mortem 0003](../../docs/postmortem/0003-checks-read-a-live-mutant.md). + +5. Use `npm run test:research` only for research adapters; use `npm run test:chaos` for explicit lifecycle fault testing. Neither substitutes for product tests. -5. For CI, packaging, or generated-output changes, validate from a clean checkout +6. For CI, packaging, or generated-output changes (see + [builds and generated artifacts](references/builds.md)), validate from a clean checkout or use `--require-clean` in an equivalent clean tree. CI automatically runs the named `verify:*` package contracts on push and pull request. -6. Commit one coherent change with only owned files. Leave unrelated user or Agent work untouched. +7. Commit one coherent change with only owned files. Leave unrelated user or Agent work untouched. Commit messages follow the repository's conventional style (`type(scope): summary` + a body that says what changed and why, one change per commit). A commit is a proposal, not a proof: the verification evidence (targeted test + `agent:verify`) is what makes it hold, so do not claim a - check passed in the message unless it ran. -7. When opening a pull request, read `.github/pull_request_template.md` and + check passed in the message unless it ran. Evidence has no third state: a + failure that cannot be reproduced is recorded with its reproduction attempt and rate, + or left open — never as "flaky", which is a label that closes a question nobody + answered ([post-mortem 0004](../../docs/postmortem/0004-flaky-was-a-clock-boundary.md)). +8. When opening a pull request, read `.github/pull_request_template.md` and follow it as the PR prompt: fill the three description blocks (What / Why / Changes) from the change plus `未验证项`, which names the surface the change did not exercise — a route that did not run, a platform or environment that was not @@ -127,85 +183,17 @@ or remove it when its exit criteria are met. CI enforces, and it catches locally what a CI round-trip would cost. Draft PRs and CI status are owned by the forge; the template checklist is the submitter's own pre-flight, not a substitute for `All checks passed`. -8. Resolve the in-flight goal after the task is completed or deliberately +9. Resolve the in-flight goal after the task is completed or deliberately abandoned. The board records that work is active, not a step-by-step history; Git and verification evidence remain the source of actual implementation state. -## Repository Control Plane beyond agent:verify - -`npm run agent:verify` auto-discovers the contract that uniquely covers the -current scope and runs the equivalent reconcile — that is the default for -ordinary changes (see [`ci-cd-and-quality.md` §7.11](../../docs/design/ci-cd-and-quality.md)). -Use the standalone `nmg-rcp` CLI (`node bin/nmg-rcp.mjs`, contract path first) -only in the scenarios `agent:verify` does not cover: - -- **Check CI state without opening the browser:** `nmg-rcp forge-status --pr ` - reads the forge's status-check rollup (`checks[]` with name/conclusion). Use - it before claiming "checks pass" or deciding a PR is mergeable. -- **Review what a reconcile would do before running it:** - `nmg-rcp plan ` (and `nmg-rcp compile ` when the contract - itself changed). -- **Inspect verification evidence:** `nmg-rcp receipt-list` / - `nmg-rcp receipt-verify ` / `nmg-rcp receipt-scan` — receipts live - under `.rcp/receipts/` and are append-only. -- **Retry after a failed reconcile, or run an explicit workspace-ready pass:** - `nmg-rcp reconcile --apply --workspace-ready [--recover-attempt]`. -- **Bind a PR or create a draft PR through the forge provider:** - `nmg-rcp forge-bind --pr ` / `nmg-rcp forge-create --base main --head `. - -`--apply` never runs by default; reconcile plans unless `--apply` is explicit. -When a Contract's status or verification drift from the design doc, update the -owning document (this SKILL, `ci-cd-and-quality.md`, the RCP decision) in the -same change — an improved tool that stays undocumented is a tool agents will -not reach for. - -**Do not recursively scan oversized directories by default.** This is an -Agent operating rule for searches, inventories, size estimation, and repository -observation—not a claim that the runtime enforces a size-based rejection. -Avoid known large dataset, benchmark, virtual-environment, and generated trees -unless the task explicitly requires them. Start with named files or narrow -paths; do not walk an entire large tree merely to estimate whether it is large. -When a large-tree scan is genuinely needed, obtain explicit authorization for -the paths and bound the scan to that scope. A broad wildcard alone is not a -substitute for that authorization. - -For RCP, choose narrow contract includes before observing. Directory pruning -only avoids include-unreachable subtrees; it is not a size guard, and broad -patterns may still reach large trees. Never silently omit in-scope files to -reduce cost, since that would change observation/digest semantics. Runtime -observation behavior is owned by -[`ci-cd-and-quality.md` §7.2](../../docs/design/ci-cd-and-quality.md#72-真相域). - -## Builds and generated artifacts - -Configure the formatting hook once per clone: `git config core.hooksPath .githooks`. -The pre-commit hook runs Prettier on staged `.ts` files. - -Regenerable outputs are **not** tracked (see the rejected decision -[Track build artifacts in version control](../../docs/decisions/rejected/2026-09-02-track-build-artifacts-in-git.md)): - -- `dist/` (root tsc build), `dsh/dsh-nmg/lib/` (tsdown), and - `src/prompts/nmg-prompts.generated.ts` (from `nmg-prompts.yaml`) are - gitignored; the tree stays clean only if you never `git add` them. -- A change to `src/` that feeds a generated output is verified by - regeneration, not by committing the output. - -Reproduce locally, in this order: - -1. Root package: `npm ci` (or `npm install` when adding a dependency), then - `npm run build` — regenerates `src/prompts/nmg-prompts.generated.ts` and - `dist/`. -2. Subpackages with their own lockfile (currently `dsh/dsh-nmg`, pnpm): - `cd dsh/dsh-nmg && pnpm install --frozen-lockfile && pnpm run build` — - regenerates `lib/`. `npm run verify:packages` runs every subpackage from a - frozen lockfile automatically. - Before starting a linked DSH web profile, also follow the adapter's - [startup prerequisites](../../dsh/dsh-nmg/README.md#启动前构建). -3. `npm run check:lock` fails when the root `package-lock.json` drifted from - `package.json`; fix with `npm install --package-lock-only`. - -When a change touches a subpackage's `src/`, `package.json`, or its lockfile, -`npm run agent:verify` covers it through `verify:static` → -`verify:packages`/`check:lock`. - -Never invoke live LLM, embedding, or full benchmark workloads unless the task explicitly calls for them. +## When to read the manual + +Everything above is the path an ordinary change walks. Two subjects are only needed +for a specific kind of change: + +- For RCP operations `agent:verify` does not cover (forge status, plan/compile, + receipts, an explicit reconcile, PR binding), and for the rule against scanning + oversized trees by default: [control plane](references/control-plane.md) +- For building this repository, regenerating outputs, subpackage installs, or + lockfile drift: [builds and generated artifacts](references/builds.md) diff --git a/skills/repo-development/references/builds.md b/skills/repo-development/references/builds.md new file mode 100644 index 00000000..4614de4b --- /dev/null +++ b/skills/repo-development/references/builds.md @@ -0,0 +1,31 @@ +# Builds and generated artifacts + +Configure the formatting hook once per clone: `git config core.hooksPath .githooks`. +The pre-commit hook runs Prettier on staged `.ts` files. + +Regenerable outputs are **not** tracked (see the rejected decision +[Track build artifacts in version control](../../../docs/decisions/rejected/2026-09-02-track-build-artifacts-in-git.md)): + +- `dist/` (root tsc build), `dsh/dsh-nmg/lib/` (tsdown), and + `src/prompts/nmg-prompts.generated.ts` (from `nmg-prompts.yaml`) are + gitignored; the tree stays clean only if you never `git add` them. +- A change to `src/` that feeds a generated output is verified by + regeneration, not by committing the output. + +Reproduce locally, in this order: + +1. Root package: `npm ci` (or `npm install` when adding a dependency), then + `npm run build` — regenerates `src/prompts/nmg-prompts.generated.ts` and + `dist/`. +2. Subpackages with their own lockfile (currently `dsh/dsh-nmg`, pnpm): + `cd dsh/dsh-nmg && pnpm install --frozen-lockfile && pnpm run build` — + regenerates `lib/`. `npm run verify:packages` runs every subpackage from a + frozen lockfile automatically. + Before starting a linked DSH web profile, also follow the adapter's + [startup prerequisites](../../../dsh/dsh-nmg/README.md#启动前构建). +3. `npm run check:lock` fails when the root `package-lock.json` drifted from + `package.json`; fix with `npm install --package-lock-only`. + +When a change touches a subpackage's `src/`, `package.json`, or its lockfile, +`npm run agent:verify` covers it through `verify:static` → +`verify:packages`/`check:lock`. diff --git a/skills/repo-development/references/control-plane.md b/skills/repo-development/references/control-plane.md new file mode 100644 index 00000000..039063ee --- /dev/null +++ b/skills/repo-development/references/control-plane.md @@ -0,0 +1,45 @@ +# Repository Control Plane beyond agent:verify + +`npm run agent:verify` auto-discovers the contract that uniquely covers the +current scope and runs the equivalent reconcile — that is the default for +ordinary changes (see [`ci-cd-and-quality.md` §7.11](../../../docs/design/ci-cd-and-quality.md)). +Use the standalone `nmg-rcp` CLI (`node bin/nmg-rcp.mjs`, contract path first) +only in the scenarios `agent:verify` does not cover: + +- **Check CI state without opening the browser:** `nmg-rcp forge-status --pr ` + reads the forge's status-check rollup (`checks[]` with name/conclusion). Use + it before claiming "checks pass" or deciding a PR is mergeable. +- **Review what a reconcile would do before running it:** + `nmg-rcp plan ` (and `nmg-rcp compile ` when the contract + itself changed). +- **Inspect verification evidence:** `nmg-rcp receipt-list` / + `nmg-rcp receipt-verify ` / `nmg-rcp receipt-scan` — receipts live + under `.rcp/receipts/` and are append-only. +- **Retry after a failed reconcile, or run an explicit workspace-ready pass:** + `nmg-rcp reconcile --apply --workspace-ready [--recover-attempt]`. +- **Bind a PR or create a draft PR through the forge provider:** + `nmg-rcp forge-bind --pr ` / `nmg-rcp forge-create --base main --head `. + +`--apply` never runs by default; reconcile plans unless `--apply` is explicit. +When a Contract's status or verification drift from the design doc, update the +owning document (the Skill entry or this reference, `ci-cd-and-quality.md`, the +RCP decision) in the +same change — an improved tool that stays undocumented is a tool agents will +not reach for. + +**Do not recursively scan oversized directories by default.** This is an +Agent operating rule for searches, inventories, size estimation, and repository +observation—not a claim that the runtime enforces a size-based rejection. +Avoid known large dataset, benchmark, virtual-environment, and generated trees +unless the task explicitly requires them. Start with named files or narrow +paths; do not walk an entire large tree merely to estimate whether it is large. +When a large-tree scan is genuinely needed, obtain explicit authorization for +the paths and bound the scan to that scope. A broad wildcard alone is not a +substitute for that authorization. + +For RCP, choose narrow contract includes before observing. Directory pruning +only avoids include-unreachable subtrees; it is not a size guard, and broad +patterns may still reach large trees. Never silently omit in-scope files to +reduce cost, since that would change observation/digest semantics. Runtime +observation behavior is owned by +[`ci-cd-and-quality.md` §7.2](../../../docs/design/ci-cd-and-quality.md#72-真相域). diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 8af44aa2..60027fd5 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -31,6 +31,7 @@ import type { NmgSplitNodeParams, NmgSyncStgParams, NmgTaskBoardParams, + NmgTaskRunParams, NmgTopologyProposalParams, } from "./protocol.ts"; @@ -790,6 +791,38 @@ export const NMG_CLI_COMMANDS: readonly CliCommandSpec[] = [ }) as unknown as NmgTaskBoardParams; }, }, + { + // The run surface's operator-facing half. A run registers, freezes its plan and adopts its + // entries from the runner that owns it (the daemon client is one client of those transitions, + // not the only one); asking what a run holds and stopping it are the two a person does, which is + // the same subset the board's own commands expose. + method: "taskRun", + words: ["run", "status"], + usageLine: "nmg run status RUN_ID [--json]", + options: [], + flags: [], + buildParams: (values): NmgTaskRunParams => ({ + action: "status", + runId: singlePositional(values, "run status"), + }), + }, + { + method: "taskRun", + words: ["run", "cancel"], + usageLine: "nmg run cancel RUN_ID [--task TASK_ID] [--reason TEXT] [--json]", + options: ["task", "reason"], + flags: [], + usageDetail: `Run cancel options: + --task ID Cancel that task of the run's plan instead of the whole run + --reason TEXT Why it was cancelled (recorded as the fact's payload)`, + buildParams: (values): NmgTaskRunParams => + compactObject({ + action: "cancel", + runId: singlePositional(values, "run cancel"), + taskId: firstOption(values, "task"), + reason: firstOption(values, "reason"), + }) as unknown as NmgTaskRunParams, + }, { method: "syncStg", words: ["stg", "sync"], diff --git a/src/cli/http-client.ts b/src/cli/http-client.ts index d93fa67f..7e5a603b 100644 --- a/src/cli/http-client.ts +++ b/src/cli/http-client.ts @@ -9,22 +9,49 @@ import type { NmgMethod } from "./protocol.ts"; * drags the core dependency tree into the Pi process. See * tests/cli/http-boundary.test.ts. */ +export interface HttpCallOptions { + /** + * How long to wait for an answer before giving up. + * + * Omitted by default, which leaves the platform's own behaviour (undici's ~300s headers timeout) + * in place: this is an opt-in bound, not a change of the product's default. A caller that can be + * held up by a process it does not control - a harness whose host may be blocked, or a client + * whose agent must not sit for five minutes - passes one, and gets a failure that names the bound + * instead of the transport's. + */ + timeoutMs?: number; +} + export async function httpCall( state: ServerState, method: NmgMethod, params: unknown = {}, + options: HttpCallOptions = {}, ): Promise { if (state.transport !== "http" || !state.host || !state.port || !state.token) { throw new Error("NMG daemon state does not contain an HTTP endpoint"); } - const response = await fetch(`http://${state.host}:${state.port}/`, { - method: "POST", - headers: { - "content-type": "application/json", - authorization: `Bearer ${state.token}`, - }, - body: JSON.stringify({ jsonrpc: "2.0", method, params, id: 1 }), - }); + let response: Response; + try { + response = await fetch(`http://${state.host}:${state.port}/`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${state.token}`, + }, + body: JSON.stringify({ jsonrpc: "2.0", method, params, id: 1 }), + ...(options.timeoutMs === undefined + ? {} + : { signal: AbortSignal.timeout(options.timeoutMs) }), + }); + } catch (error) { + // A bound that fired is a fact about this call, not a transport quirk to report as one: name it + // here so the caller's message can say what the wait was for. + if (options.timeoutMs !== undefined && isTimeout(error)) { + throw new TimeoutError(`nmg ${method} did not answer within ${options.timeoutMs}ms`); + } + throw error; + } const text = await response.text(); if (!response.ok) { throw new Error(text || `nmg ${method} failed (${response.status})`); @@ -38,3 +65,19 @@ export async function httpCall( } return parsed.result; } + +/** A call that was given a bound and reached it. Distinct from a transport failure on purpose. */ +export class TimeoutError extends Error { + constructor(message: string) { + super(message); + this.name = "TimeoutError"; + } +} + +function isTimeout(error: unknown): boolean { + if (error instanceof TimeoutError) return true; + const name = (error as { name?: unknown } | null)?.name; + if (name === "TimeoutError" || name === "AbortError") return true; + const cause = (error as { cause?: { name?: unknown } } | null)?.cause; + return cause?.name === "TimeoutError" || cause?.name === "HeadersTimeoutError"; +} diff --git a/src/cli/protocol.ts b/src/cli/protocol.ts index 0ead9595..3a2e969c 100644 --- a/src/cli/protocol.ts +++ b/src/cli/protocol.ts @@ -53,6 +53,7 @@ import type { LabScope, } from "../integration/lab-capabilities.ts"; import type { ContextFeedbackLabels } from "../lab/context-reward.ts"; +import type { RunPlanTask, RunStatus } from "../integration/task-coordinator.ts"; // This is a compatibility epoch, not a feature revision. Additive RPCs, // optional fields, and capabilities remain within the same epoch and are @@ -140,6 +141,7 @@ const RPC_DESCRIPTOR_SOURCE = { syncStg: {}, stgPurgeSession: {}, taskBoard: {}, + taskRun: {}, chainCreate: {}, chainAdd: {}, chainRemove: {}, @@ -524,6 +526,14 @@ export interface NmgTaskBoardPutParams extends NmgTaskBoardBase { * agent name, never sessionId (session changes on reload). Omit = ordinary * broadcast to subscribers. */ to?: string; + /** Adopt this entry into a run's task in the same transition that creates it. The binding is a run + * fact, so creating the entry and binding it have to stand or fall together: an entry that exists + * without its binding would be an unmanaged hole a later direct write could move. + * + * A client gates this field on the daemon advertising the `taskRun` method (see protocol.ts's + * descriptor for it): an older same-epoch daemon would ignore it and create an entry that the + * caller believes the run manages. */ + adopt?: { runId: string; taskId: string; attempt?: number }; } export interface NmgTaskBoardReadParams extends NmgTaskBoardBase { @@ -729,6 +739,42 @@ export type NmgTaskBoardParams = | NmgTaskBoardRenameParams | NmgTaskBoardDiscoverParams; +/** The run surface: the transitions a run goes through, reachable by any process that can reach the + * daemon. Registering a run and freezing its plan are the run's own record; `bind` adopts a board + * entry that already exists, while `taskBoard put` with `adopt` creates and adopts in one + * transition. `status` is a read of the same record. */ +export type NmgTaskRunParams = + | { + action: "register"; + runId: string; + planDigest: string; + policy: string; + revision: string; + retention: string; + } + | { + action: "freeze"; + runId: string; + /** The array order is the plan order, so the caller cannot send an order that disagrees with + * the positions. */ + tasks: Array>; + } + | { + action: "bind"; + runId: string; + taskId: string; + boardTaskId: string; + entryId: string; + attempt?: number; + } + | { + action: "cancel"; + runId: string; + /** Omit to cancel the run; name a task to cancel that task. */ taskId?: string; + reason?: string; + } + | { action: "status"; runId: string }; + export interface NmgRetentionCandidatesParams { dormantAfterDays?: number; quarantineAfterDays?: number; @@ -921,8 +967,13 @@ export type NmgMethodResult = { stgPurgeSession: { purged: number; projectDir: string }; taskBoard: | { - action: - "put" | "resolve" | "claim" | "release" | "acknowledge" | "veto" | "deliver" | "judge"; + action: "put"; + entry: TaskBoardEntry; + /** Present when the put adopted the entry: the binding fact of that same transition. */ + bound?: { sequence: number; recorded: boolean }; + } + | { + action: "resolve" | "claim" | "release" | "acknowledge" | "veto" | "deliver" | "judge"; entry: TaskBoardEntry; } | { action: "read"; entries: TaskBoardEntry[]; nextCursor: string | null } @@ -957,6 +1008,12 @@ export type NmgMethodResult = { }>; }; shutdown: { shuttingDown: true }; + taskRun: + | { action: "register"; runId: string } + | { action: "freeze"; runId: string; frozen: number } + | { action: "bind"; sequence: number; recorded: boolean } + | { action: "cancel"; sequence: number; recorded: boolean } + | { action: "status"; status: RunStatus }; }; export class NmgProtocolError extends Error { diff --git a/src/cli/service.ts b/src/cli/service.ts index 59f51553..6d904654 100644 --- a/src/cli/service.ts +++ b/src/cli/service.ts @@ -82,6 +82,16 @@ import { } from "../core/relevance-gate.ts"; import { readRelevanceModel } from "../lab/relevance-model.ts"; import { normalizeRecallTriggers } from "../core/recall-triggers.ts"; +import { + bindRunEntry, + cancelRun, + coordinatedEntryWrite, + createBoundEntry, + freezeRunPlan, + registerRun, + taskRunStatus, + type RunPlanTaskInput, +} from "../integration/task-coordinator.ts"; import { searchMemoryContext } from "../integration/search.ts"; import { simhash64, simhashToHex, simhashFromHex, hammingDistance } from "../core/simhash.ts"; import { ControllerPolicyChannel } from "../integration/controller-channel.ts"; @@ -154,6 +164,7 @@ import { type NmgSyncStgParams, type NmgStgPurgeSessionParams, type NmgTaskBoardParams, + type NmgTaskRunParams, type NmgTopologyProposalParams, } from "./protocol.ts"; import { resolveNmgDataDir } from "./data-path.ts"; @@ -209,6 +220,10 @@ export class NmgService { readonly #tesseraBackfillRoots = new Set(); readonly #stgSyncTimes = new WeakMap>(); #shutdownRequested = false; + /** Set by close(). New work is refused after this, so a shutdown cannot race a fresh request. */ + #closing = false; + /** Calls accepted and not yet answered; see drain(). */ + #inFlight = 0; /** Lazily loaded learned relevance gate; undefined until first read, null when * no model is configured or it fails to load. */ #relevanceModel: RelevanceModelLike | null | undefined; @@ -305,194 +320,250 @@ export class NmgService { return this.#onlineLearner; } + /** A call the daemon has accepted and not yet answered. Counted here rather than inferred from + * the store, because it is the service, not the database, that a shutdown has to fence. */ + get inFlight(): number { + return this.#inFlight; + } + + /** Let the calls already accepted finish. Shutdown is a sequence - stop new work, then let the + * work in flight finish, then close once - and the middle step needs an await, which a + * synchronous close() does not have. So close() requires that this has happened, instead of + * pretending it can fence a call it cannot see. */ + async drain(timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + while (this.#inFlight > 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + if (this.#inFlight > 0) { + throw new NmgProtocolError( + "DRAINING", + `${this.#inFlight} calls were still in flight after ${timeoutMs}ms`, + ); + } + } + + /** The four topology actions share one parameter parse and three mutually exclusive branches; + * keeping them in the dispatch switch made that switch the largest decision point in the file. */ + #topologyProposal(params: unknown): unknown { + const parsed = parseTopologyProposalParams(params); + if (parsed.action === "list") { + return { + action: "list", + proposals: this.#getStore().topologyProposals(parsed.status), + }; + } + if (parsed.action === "assess") { + return { + action: "assess", + assessment: this.#getStore().assessAutomaticMergeProposal(parsed.proposalId, { + minimumObservations: parsed.minimumObservations, + minimumEstimatedGain: parsed.minimumEstimatedGain, + minimumEvidenceMemories: parsed.minimumEvidenceMemories, + }), + }; + } + if (parsed.action === "review") { + return { + action: "review", + proposal: this.#getStore().reviewTopologyProposal(parsed.proposalId, parsed.decision), + }; + } + return { + action: "actuate", + transform: this.#getStore().actuateAutomaticMergeProposal(parsed.proposalId), + }; + } + async invoke(method: M, params?: unknown): Promise { - switch (method) { - case "hello": - return this.#hello() as NmgMethodResult[M]; - case "status": - return this.#status() as NmgMethodResult[M]; - case "remember": - return this.#remember(parseRememberParams(params)) as NmgMethodResult[M]; - case "rememberBatch": - return this.#rememberBatch(parseRememberBatchParams(params)) as NmgMethodResult[M]; - case "resolveRemember": - return this.#resolveRemember(parseResolveRememberParams(params)) as NmgMethodResult[M]; - case "recordClaimOutcomes": - return this.#recordClaimOutcomes( - parseRecordClaimOutcomesParams(params), - ) as NmgMethodResult[M]; - case "recordFeedback": - return this.#recordFeedback(parseRecordFeedbackParams(params)) as NmgMethodResult[M]; - case "search": - return (await this.#search(parseSearchParams(params))) as NmgMethodResult[M]; - case "get": - return this.#get(parseGetParams(params)) as NmgMethodResult[M]; - case "recordActiveGraphAttribution": - return this.#recordActiveGraphAttribution( - parseRecordActiveGraphAttributionParams(params), - ) as NmgMethodResult[M]; - case "retentionCandidates": - return { - candidates: this.#getStore().retentionCandidates(parseRetentionCandidatesParams(params)), - } as NmgMethodResult[M]; - case "perfAggregates": - return this.#getStore().perfAggregates() as NmgMethodResult[M]; - case "pruneRetrievalTraces": - return { - pruned: this.#getStore().pruneRetrievalTraces(parsePerfPruneParams(params)), - } as NmgMethodResult[M]; - case "setStorageState": { - const parsed = parseSetStorageStateParams(params); - return { - memoryId: parsed.memoryId, - storageState: this.#getStore().setMemoryStorageState( - parsed.memoryId, - parsed.storageState, - parsed.recoveryDays, - ), - } as NmgMethodResult[M]; - } - case "deleteMemory": { - const parsed = parseDeleteMemoryParams(params); - const memory = this.#getStore().deleteMemory(parsed.memoryId); - return { deleted: memory !== null, memory } as NmgMethodResult[M]; - } - case "exportMemories": - return this.#getStore().exportMemories( - parseExportMemoriesParams(params), - ) as NmgMethodResult[M]; - case "mergeNodes": - return this.#getStore().mergeNodes(parseMergeNodesParams(params)) as NmgMethodResult[M]; - case "rollbackNodeTransform": - return this.#getStore().rollbackNodeTransform( - parseRollbackNodeTransformParams(params).transformId, - ) as NmgMethodResult[M]; - case "splitNode": - return this.#getStore().splitNode(parseSplitNodeParams(params)) as NmgMethodResult[M]; - case "topologyProposal": { - const parsed = parseTopologyProposalParams(params); - if (parsed.action === "list") { + if (this.#closing) + throw new NmgProtocolError("SHUTTING_DOWN", "the service is closing and takes no new work"); + // The counter has to be the method callers use and the switch has to run inside it: splitting + // them into two methods only hid the same switch behind a new name, which the complexity gate + // counts as new code. + this.#inFlight += 1; + try { + switch (method) { + case "hello": + return this.#hello() as NmgMethodResult[M]; + case "status": + return this.#status() as NmgMethodResult[M]; + case "remember": + return this.#remember(parseRememberParams(params)) as NmgMethodResult[M]; + case "rememberBatch": + return this.#rememberBatch(parseRememberBatchParams(params)) as NmgMethodResult[M]; + case "resolveRemember": + return this.#resolveRemember(parseResolveRememberParams(params)) as NmgMethodResult[M]; + case "recordClaimOutcomes": + return this.#recordClaimOutcomes( + parseRecordClaimOutcomesParams(params), + ) as NmgMethodResult[M]; + case "recordFeedback": + return this.#recordFeedback(parseRecordFeedbackParams(params)) as NmgMethodResult[M]; + case "search": + return (await this.#search(parseSearchParams(params))) as NmgMethodResult[M]; + case "get": + return this.#get(parseGetParams(params)) as NmgMethodResult[M]; + case "recordActiveGraphAttribution": + return this.#recordActiveGraphAttribution( + parseRecordActiveGraphAttributionParams(params), + ) as NmgMethodResult[M]; + case "retentionCandidates": return { - action: "list", - proposals: this.#getStore().topologyProposals(parsed.status), + candidates: this.#getStore().retentionCandidates( + parseRetentionCandidatesParams(params), + ), } as NmgMethodResult[M]; - } - if (parsed.action === "assess") { + case "perfAggregates": + return this.#getStore().perfAggregates() as NmgMethodResult[M]; + case "pruneRetrievalTraces": return { - action: "assess", - assessment: this.#getStore().assessAutomaticMergeProposal(parsed.proposalId, { - minimumObservations: parsed.minimumObservations, - minimumEstimatedGain: parsed.minimumEstimatedGain, - minimumEvidenceMemories: parsed.minimumEvidenceMemories, - }), + pruned: this.#getStore().pruneRetrievalTraces(parsePerfPruneParams(params)), } as NmgMethodResult[M]; - } - if (parsed.action === "review") { + case "setStorageState": { + const parsed = parseSetStorageStateParams(params); return { - action: "review", - proposal: this.#getStore().reviewTopologyProposal(parsed.proposalId, parsed.decision), + memoryId: parsed.memoryId, + storageState: this.#getStore().setMemoryStorageState( + parsed.memoryId, + parsed.storageState, + parsed.recoveryDays, + ), } as NmgMethodResult[M]; } - return { - action: "actuate", - transform: this.#getStore().actuateAutomaticMergeProposal(parsed.proposalId), - } as NmgMethodResult[M]; - } - case "memoryMaintenanceProposal": { - const parsed = parseMemoryMaintenanceProposalParams(params); - if (parsed.action === "list") { + case "deleteMemory": { + const parsed = parseDeleteMemoryParams(params); + const memory = this.#getStore().deleteMemory(parsed.memoryId); + return { deleted: memory !== null, memory } as NmgMethodResult[M]; + } + case "exportMemories": + return this.#getStore().exportMemories( + parseExportMemoriesParams(params), + ) as NmgMethodResult[M]; + case "mergeNodes": + return this.#getStore().mergeNodes(parseMergeNodesParams(params)) as NmgMethodResult[M]; + case "rollbackNodeTransform": + return this.#getStore().rollbackNodeTransform( + parseRollbackNodeTransformParams(params).transformId, + ) as NmgMethodResult[M]; + case "splitNode": + return this.#getStore().splitNode(parseSplitNodeParams(params)) as NmgMethodResult[M]; + case "topologyProposal": + return this.#topologyProposal(params) as NmgMethodResult[M]; + case "memoryMaintenanceProposal": { + const parsed = parseMemoryMaintenanceProposalParams(params); + if (parsed.action === "list") { + return { + action: "list", + proposals: this.#getStore().memoryMaintenanceProposals(parsed.status), + } as NmgMethodResult[M]; + } + if (parsed.action === "review") { + return { + action: "review", + proposal: this.#getStore().reviewMemoryMaintenanceProposal( + parsed.proposalId, + parsed.decision, + parsed.reason, + ), + } as NmgMethodResult[M]; + } return { - action: "list", - proposals: this.#getStore().memoryMaintenanceProposals(parsed.status), + action: "propose", + proposal: this.#getStore().createMemoryMaintenanceProposal({ + defectType: parsed.defectType, + action: parsed.maintenanceAction, + targetMemoryIds: parsed.targetMemoryIds, + evidenceMemoryIds: parsed.evidenceMemoryIds, + evidenceTraceIds: parsed.evidenceTraceIds, + proposedStatement: parsed.proposedStatement, + proposedScope: parsed.proposedScope, + policy: parsed.policy, + longHorizonScore: parsed.longHorizonScore, + evaluationKind: parsed.evaluationKind, + evaluationRef: parsed.evaluationRef, + }), } as NmgMethodResult[M]; } - if (parsed.action === "review") { + case "syncStg": { + const parsed = parseSyncStgParams(params); return { - action: "review", - proposal: this.#getStore().reviewMemoryMaintenanceProposal( - parsed.proposalId, - parsed.decision, - parsed.reason, + copied: copyLtgSubsetToStg( + this.#getStore(), + this.#getStgStore(parsed.projectDir, parsed.sessionId), + parsed, ), + projectDir: parsed.projectDir, } as NmgMethodResult[M]; } - return { - action: "propose", - proposal: this.#getStore().createMemoryMaintenanceProposal({ - defectType: parsed.defectType, - action: parsed.maintenanceAction, - targetMemoryIds: parsed.targetMemoryIds, - evidenceMemoryIds: parsed.evidenceMemoryIds, - evidenceTraceIds: parsed.evidenceTraceIds, - proposedStatement: parsed.proposedStatement, - proposedScope: parsed.proposedScope, - policy: parsed.policy, - longHorizonScore: parsed.longHorizonScore, - evaluationKind: parsed.evaluationKind, - evaluationRef: parsed.evaluationRef, - }), - } as NmgMethodResult[M]; - } - case "syncStg": { - const parsed = parseSyncStgParams(params); - return { - copied: copyLtgSubsetToStg( - this.#getStore(), + case "stgPurgeSession": { + const parsed = parseStgPurgeSessionParams(params); + const purged = purgeSessionFromStg( this.#getStgStore(parsed.projectDir, parsed.sessionId), - parsed, - ), - projectDir: parsed.projectDir, - } as NmgMethodResult[M]; - } - case "stgPurgeSession": { - const parsed = parseStgPurgeSessionParams(params); - const purged = purgeSessionFromStg( - this.#getStgStore(parsed.projectDir, parsed.sessionId), - parsed.sessionId, - ); - return { purged, projectDir: parsed.projectDir } as NmgMethodResult[M]; - } - case "taskBoard": { - const parsed = parseTaskBoardParams(params); - return taskBoardHandlers[parsed.action](this.#getStore(), parsed) as NmgMethodResult[M]; + parsed.sessionId, + ); + return { purged, projectDir: parsed.projectDir } as NmgMethodResult[M]; + } + case "taskBoard": { + const parsed = parseTaskBoardParams(params); + return taskBoardHandlers[parsed.action](this.#getStore(), parsed) as NmgMethodResult[M]; + } + case "taskRun": { + const parsed = parseTaskRunParams(params); + return taskRunHandlers[parsed.action](this.#getStore(), parsed) as NmgMethodResult[M]; + } + case "chainCreate": + return this.#chainCreate(parseChainCreateParams(params)) as NmgMethodResult[M]; + case "chainAdd": + return this.#chainAdd(parseChainAddParams(params)) as NmgMethodResult[M]; + case "chainRemove": + return this.#chainRemove(parseChainRemoveParams(params)) as NmgMethodResult[M]; + case "chainEdgeAdd": + return this.#chainEdgeAdd(parseChainEdgeAddParams(params)) as NmgMethodResult[M]; + case "chainEdgeRemove": + return this.#chainEdgeRemove(parseChainEdgeRemoveParams(params)) as NmgMethodResult[M]; + case "chainGet": + return this.#chainGet(parseChainGetParams(params)) as NmgMethodResult[M]; + case "chainList": + return this.#chainList(parseChainListParams(params)) as NmgMethodResult[M]; + case "lab": + return this.#lab(parseLabParams(params)) as NmgMethodResult[M]; + case "sessionActiveGraph": + return this.#sessionActiveGraph( + parseSessionActiveGraphParams(params), + ) as NmgMethodResult[M]; + case "shutdown": + this.#shutdownRequested = true; + return { shuttingDown: true } as NmgMethodResult[M]; + default: + throw new NmgProtocolError("METHOD_NOT_FOUND", `unknown method: ${String(method)}`); } - case "chainCreate": - return this.#chainCreate(parseChainCreateParams(params)) as NmgMethodResult[M]; - case "chainAdd": - return this.#chainAdd(parseChainAddParams(params)) as NmgMethodResult[M]; - case "chainRemove": - return this.#chainRemove(parseChainRemoveParams(params)) as NmgMethodResult[M]; - case "chainEdgeAdd": - return this.#chainEdgeAdd(parseChainEdgeAddParams(params)) as NmgMethodResult[M]; - case "chainEdgeRemove": - return this.#chainEdgeRemove(parseChainEdgeRemoveParams(params)) as NmgMethodResult[M]; - case "chainGet": - return this.#chainGet(parseChainGetParams(params)) as NmgMethodResult[M]; - case "chainList": - return this.#chainList(parseChainListParams(params)) as NmgMethodResult[M]; - case "lab": - return this.#lab(parseLabParams(params)) as NmgMethodResult[M]; - case "sessionActiveGraph": - return this.#sessionActiveGraph( - parseSessionActiveGraphParams(params), - ) as NmgMethodResult[M]; - case "shutdown": - this.#shutdownRequested = true; - return { shuttingDown: true } as NmgMethodResult[M]; - default: - throw new NmgProtocolError("METHOD_NOT_FOUND", `unknown method: ${String(method)}`); + } finally { + this.#inFlight -= 1; } } + /** Closes in the order the design asks for: stop taking new work, revoke the timers and signals + * that could start more of it, then close each store exactly once. A second call is a no-op, so + * a shutdown path that runs twice cannot close a store out from under a live reader. */ close(): void { + // A close that silently drops calls already accepted would lose writes that were answered as + // accepted; the sequence is drain() first, and this refuses rather than pretending. + if (this.#inFlight > 0) { + throw new NmgProtocolError( + "DRAINING", + `${this.#inFlight} calls were still in flight; drain before closing`, + ); + } + this.#closing = true; for (const job of this.#maintenanceJobs.values()) clearImmediate(job); this.#maintenanceJobs.clear(); this.#maintenanceSignals.clear(); - this.#store?.close(); + const stores = [this.#store, ...this.#stgStores.values()]; this.#store = undefined; - for (const store of this.#stgStores.values()) store.close(); this.#stgStores.clear(); this.#sessionActiveGraphs.clear(); + for (const store of stores) if (store) store.close(); } #hello(): NmgHelloResult { @@ -2763,6 +2834,7 @@ function parseTaskBoardParams(value: unknown): NmgTaskBoardParams { to: optionalString(params, "to"), ttlSeconds, expiresAt, + adopt: optionalAdoption(params.adopt), }; } if (TASK_BOARD_READ_STYLE.has(action)) { @@ -2831,6 +2903,103 @@ function parseEntryStyleTaskBoardParams( return null; } +/** The adoption request when the caller sent one, or undefined. The branch lives here rather than in + * the put expression so an optional field does not decide points for the whole board parser. */ +function optionalAdoption( + value: unknown, +): { runId: string; taskId: string; attempt?: number } | undefined { + return value === undefined ? undefined : parseAdoption(value); +} + +/** A board put's adoption request, or the shape it must have to be one. */ +function parseAdoption(value: unknown): { runId: string; taskId: string; attempt?: number } { + const params = objectParams(value); + return { + runId: requiredString(params, "runId"), + taskId: requiredString(params, "taskId"), + attempt: optionalInteger(params, "attempt", 1, 1_000_000), + }; +} + +/** Request bounds, not semantics: a plan's meaning is the coordinator's and the compiler's, this + * only keeps one call from being unbounded. */ +const TASK_RUN_PLAN_LIMIT = 200; +const TASK_RUN_DEPENDENCY_LIMIT = 64; + +/** + * The run surface's parameters. Every field is validated here so the coordinator and the store can + * treat their inputs as well-formed: a wire type that admits a value the layer below cannot hold is + * a validation gap, not a caller's mistake. + */ +function parseTaskRunParams(value: unknown): NmgTaskRunParams { + const params = objectParams(value); + const action = requiredEnum(params, "action", [ + "register", + "freeze", + "bind", + "cancel", + "status", + ] as const); + const runId = requiredString(params, "runId"); + if (action === "register") { + return { + action, + runId, + planDigest: requiredString(params, "planDigest"), + policy: requiredString(params, "policy"), + revision: requiredString(params, "revision"), + retention: requiredString(params, "retention"), + }; + } + if (action === "freeze") { + const tasks = params.tasks; + if (!Array.isArray(tasks) || tasks.length === 0 || tasks.length > TASK_RUN_PLAN_LIMIT) { + throw new NmgProtocolError( + "INVALID_PARAMS", + `tasks must be an array of 1..${TASK_RUN_PLAN_LIMIT} tasks`, + ); + } + return { action, runId, tasks: tasks.map((task) => parseRunPlanTask(task)) }; + } + if (action === "bind") { + return { + action, + runId, + taskId: requiredString(params, "taskId"), + boardTaskId: requiredString(params, "boardTaskId"), + entryId: requiredString(params, "entryId"), + attempt: optionalInteger(params, "attempt", 1, 1_000_000), + }; + } + if (action === "cancel") { + return { + action, + runId, + taskId: optionalString(params, "taskId"), + reason: optionalString(params, "reason"), + }; + } + return { action: "status", runId }; +} + +/** One task of a freeze request. It carries no position: the array order is the plan order, and + * turning that order into positions is the coordinator's, so a caller cannot contradict it. */ +function parseRunPlanTask(value: unknown): RunPlanTaskInput { + const params = objectParams(value); + return { + taskId: requiredString(params, "taskId"), + revision: requiredString(params, "revision"), + input: requiredString(params, "input"), + dependencies: requiredStringArray(params, "dependencies", 0, TASK_RUN_DEPENDENCY_LIMIT), + effect: requiredString(params, "effect"), + waitEvent: optionalString(params, "waitEvent"), + operation: optionalString(params, "operation"), + kind: optionalString(params, "kind"), + patchFiles: optionalStringArray(params, "patchFiles"), + patchEditable: optionalStringArray(params, "patchEditable"), + }; +} + type TaskBoardParamsOf = Extract< NmgTaskBoardParams, { action: A } @@ -2848,18 +3017,20 @@ const taskBoardHandlers: Record const p = parsed as TaskBoardParamsOf<"put">; const expiresAt = p.expiresAt ?? new Date(Date.now() + (p.ttlSeconds ?? 86_400) * 1_000).toISOString(); - return { - action: "put", - entry: store.putTaskBoardEntry({ - taskId: p.taskId, - agentId: p.agentId, - sourceSessionId: p.sourceSessionId, - kind: p.kind ?? "note", - content: p.content, - expiresAt, - to: p.to, - }), + const entry = { + taskId: p.taskId, + agentId: p.agentId, + sourceSessionId: p.sourceSessionId, + kind: p.kind ?? "note", + content: p.content, + expiresAt, + to: p.to, }; + // Adoption is part of creating the entry, not a second call after it: an entry that exists + // without its binding is exactly the unmanaged hole the run fence exists to close, and a crash + // between two calls would leave one. An unadopted put takes the path it always took. + if (!p.adopt) return { action: "put", entry: store.putTaskBoardEntry(entry) }; + return { action: "put", ...createBoundEntry(store, { entry, ...p.adopt }) }; }, read: (store, parsed) => { const p = parsed as TaskBoardParamsOf<"read">; @@ -2880,11 +3051,27 @@ const taskBoardHandlers: Record list: (store) => ({ action: "list", boards: store.listTaskBoards() }), claim: (store, parsed) => { const p = parsed as TaskBoardParamsOf<"claim">; - return { action: "claim", entry: store.claimTaskBoardEntry(p) }; + return { + action: "claim", + entry: coordinatedEntryWrite(store, { + verb: "claim", + entryId: p.entryId, + actorId: p.agentId, + apply: () => store.claimTaskBoardEntry(p), + }), + }; }, release: (store, parsed) => { const p = parsed as TaskBoardParamsOf<"release">; - return { action: "release", entry: store.releaseTaskBoardEntry(p) }; + return { + action: "release", + entry: coordinatedEntryWrite(store, { + verb: "release", + entryId: p.entryId, + actorId: p.agentId, + apply: () => store.releaseTaskBoardEntry(p), + }), + }; }, deliveryCheck: (store, parsed) => { const p = parsed as TaskBoardParamsOf<"deliveryCheck">; @@ -2908,7 +3095,17 @@ const taskBoardHandlers: Record }, acknowledge: (store, parsed) => { const p = parsed as TaskBoardParamsOf<"acknowledge">; - store.acknowledgeTaskBoardEntry({ entryId: p.entryId, agentId: p.agentId, reason: p.reason }); + coordinatedEntryWrite(store, { + verb: "acknowledge", + entryId: p.entryId, + actorId: p.agentId, + apply: () => + store.acknowledgeTaskBoardEntry({ + entryId: p.entryId, + agentId: p.agentId, + reason: p.reason, + }), + }); return { action: "acknowledge", entry: store.getTaskBoardEntryById(p.taskId, p.entryId)!, @@ -2916,15 +3113,39 @@ const taskBoardHandlers: Record }, veto: (store, parsed) => { const p = parsed as TaskBoardParamsOf<"veto">; - return { action: "veto", entry: store.vetoTaskBoardEntry(p) }; + return { + action: "veto", + entry: coordinatedEntryWrite(store, { + verb: "veto", + entryId: p.entryId, + actorId: p.agentId, + apply: () => store.vetoTaskBoardEntry(p), + }), + }; }, deliver: (store, parsed) => { const p = parsed as TaskBoardParamsOf<"deliver">; - return { action: "deliver", entry: store.deliverTaskBoardEntry(p) }; + return { + action: "deliver", + entry: coordinatedEntryWrite(store, { + verb: "deliver", + entryId: p.entryId, + actorId: p.agentId, + apply: () => store.deliverTaskBoardEntry(p), + }), + }; }, judge: (store, parsed) => { const p = parsed as TaskBoardParamsOf<"judge">; - return { action: "judge", entry: store.judgeTaskBoardEntry(p) }; + return { + action: "judge", + entry: coordinatedEntryWrite(store, { + verb: "judge", + entryId: p.entryId, + actorId: p.agentId, + apply: () => store.judgeTaskBoardEntry(p), + }), + }; }, unsubscribe: (store, parsed) => { const p = parsed as TaskBoardParamsOf<"unsubscribe">; @@ -2978,7 +3199,55 @@ const taskBoardHandlers: Record }, resolve: (store, parsed) => { const p = parsed as TaskBoardParamsOf<"resolve">; - return { action: "resolve", entry: store.resolveTaskBoardEntry(p) }; + return { + action: "resolve", + entry: coordinatedEntryWrite(store, { + verb: "resolve", + entryId: p.entryId, + actorId: p.agentId, + apply: () => store.resolveTaskBoardEntry(p), + }), + }; + }, +}; + +type TaskRunParamsOf = Extract< + NmgTaskRunParams, + { action: A } +>; +type TaskRunHandler = (store: NmgStore, parsed: NmgTaskRunParams) => NmgMethodResult["taskRun"]; + +/** Table-driven run-surface dispatch, for the same reason as `taskBoardHandlers`: the transition + * rules live in the coordinator, and every branch here is one call into it. */ +const taskRunHandlers: Record = { + register: (store, parsed) => { + const p = parsed as TaskRunParamsOf<"register">; + return { action: "register", ...registerRun(store, p) }; + }, + freeze: (store, parsed) => { + const p = parsed as TaskRunParamsOf<"freeze">; + return { action: "freeze", ...freezeRunPlan(store, { runId: p.runId, tasks: p.tasks }) }; + }, + bind: (store, parsed) => { + const p = parsed as TaskRunParamsOf<"bind">; + return { + action: "bind", + ...bindRunEntry(store, { + runId: p.runId, + taskId: p.taskId, + boardTaskId: p.boardTaskId, + entryId: p.entryId, + attempt: p.attempt, + }), + }; + }, + cancel: (store, parsed) => { + const p = parsed as TaskRunParamsOf<"cancel">; + return { action: "cancel", ...cancelRun(store, p) }; + }, + status: (store, parsed) => { + const p = parsed as TaskRunParamsOf<"status">; + return { action: "status", status: taskRunStatus(store, p.runId) }; }, }; diff --git a/src/core/store/base.ts b/src/core/store/base.ts index e69bea24..cb3560ed 100644 --- a/src/core/store/base.ts +++ b/src/core/store/base.ts @@ -37,6 +37,7 @@ import type { VectorEmbedder, } from "../types.ts"; import { TASK_BOARD_VERDICTS, WORLD_BOARD_ID } from "../types.ts"; +import { currentlyValid, notExpired } from "./clock.ts"; import { histogramAdd } from "../perf.ts"; import { Router } from "../router.ts"; import { cosineSimilarity, HashingVectorEmbedder } from "../vector.ts"; @@ -68,13 +69,52 @@ import { mapSearchResult, } from "./rows.ts"; +export interface TransactionPort { + /** Which transition this port belongs to; the store matches it against the open one. */ + readonly generation: number; +} + +/** A callback that would keep a transaction open across an await is refused where it is handed in. */ +function refuseThenable(value: unknown, message: string): void { + const thenable = + value !== null && + (typeof value === "object" || typeof value === "function") && + typeof (value as { then?: unknown }).then === "function"; + if (thenable) throw new Error(message); +} + export class NmgStoreBase { + /** True when this store came from the read-only factory rather than a shared connection. */ + protected readOnly = false; protected db: DatabaseSync; protected embedder: VectorEmbedder; protected router: Router; protected vectorCaches = new Map(); protected scopeWriteIndexes = new Map(); protected scopeWriteIndexEnabled: boolean; + /** The open write transaction, if any. A port is the only way to join it. */ + private openTransaction: { port: TransactionPort; rollbackOnly: boolean } | null = null; + /** The run whose coordinated write scope is open, if any: the store's own state, set only by + * `coordinateRunWrite` while its callback runs. */ + private coordinatedRun: string | null = null; + private transactionGeneration = 0; + /** Set when ROLLBACK itself failed: the connection's state is unknown, so it takes no more work. */ + private connectionQuarantined = false; + + /** A read-only factory neither creates the file nor migrates an old one. Both refusals are named + * here rather than surfacing as a driver-level "unable to open database file" with no reason, and + * the decisions live here so the constructor is not the file's largest decision point. */ + #refuseUnusableReadOnlyOpen(databasePath: string): void { + if (!this.readOnly) return; + if (!existsSync(databasePath)) + throw new Error("this store does not exist; a read-only open does not create one"); + } + + #refuseUnrecognisableReadOnlyOpen(): void { + if (!this.readOnly) return; + if (this.hasSchema()) return; + throw new Error("this store has no recognisable schema; a read-only open does not migrate it"); + } constructor( databasePath: string, @@ -82,21 +122,38 @@ export class NmgStoreBase { options: NmgStoreOptions = {}, ) { mkdirSync(dirname(databasePath), { recursive: true }); - this.db = new DatabaseSync(databasePath); + // A read-only open is a different factory, not a mode of the shared connection: it gets a + // handle that cannot write, and it neither migrates nor checkpoints. + this.readOnly = options.readOnly === true; + // A read-only factory neither creates the file nor migrates an old one, so both refusals happen + // here rather than surfacing as a driver-level "unable to open database file" with no reason. + this.#refuseUnusableReadOnlyOpen(databasePath); + this.db = new DatabaseSync(databasePath, readOnlyOpenOptions(this.readOnly)); this.embedder = embedder; this.router = new Router(embedder); this.scopeWriteIndexEnabled = options.scopeWriteIndex ?? false; try { - this.db.exec(` - PRAGMA foreign_keys = ON; - PRAGMA journal_mode = WAL; - PRAGMA synchronous = NORMAL; - PRAGMA cache_size = -64000; - PRAGMA temp_store = MEMORY; - PRAGMA mmap_size = 268435456; - PRAGMA busy_timeout = 5000; - `); - migrate(this.db); + // `journal_mode` rewrites the database header, so a read-only handle must not set it; the + // remaining pragmas are connection-local and harmless. + this.db.exec( + this.readOnly + ? `PRAGMA foreign_keys = ON; + PRAGMA cache_size = -64000; + PRAGMA temp_store = MEMORY; + PRAGMA mmap_size = 268435456; + PRAGMA busy_timeout = 5000;` + : `PRAGMA foreign_keys = ON; + PRAGMA journal_mode = WAL; + PRAGMA synchronous = NORMAL; + PRAGMA cache_size = -64000; + PRAGMA temp_store = MEMORY; + PRAGMA mmap_size = 268435456; + PRAGMA busy_timeout = 5000;`, + ); + if (!this.readOnly) migrate(this.db); + // An existing file with no recognisable schema is not an empty store: reporting it as one would + // hide an unknown format behind default empties. + this.#refuseUnrecognisableReadOnlyOpen(); // checkpoint-on-open: fold any -wal left behind by a force-exit shutdown // (where close() never ran) into the main DB and truncate it, so WAL can // never accumulate across restarts. SQLite auto-recovers WAL frames on @@ -104,7 +161,7 @@ export class NmgStoreBase { // checkpoints, so the common open path skips the blocking TRUNCATE // (stg-v2 review ③a; wal_checkpoint is synchronous disk I/O that stalls // the event loop when the WAL is large). - if (existsSync(`${databasePath}-wal`)) { + if (!this.readOnly && existsSync(`${databasePath}-wal`)) { try { if (statSync(`${databasePath}-wal`).size > 0) { this.db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); @@ -126,15 +183,25 @@ export class NmgStoreBase { } } + /** The core schema's marker table: present means this file was written by a version this code reads. */ + private hasSchema(): boolean { + return ( + this.db + .prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory_nodes'") + .get() !== undefined + ); + } + close(): void { // WAL checkpoint before close: without this the daemon's force-exit // shutdown leaves -wal files behind (v1 measured ~1.5G across 1681 // session STG stores). TRUNCATE folds WAL into the main DB then resets it. - try { - this.db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); - } catch { - // ignore — closing anyway - } + if (!this.readOnly) + try { + this.db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); + } catch { + // ignore — closing anyway + } this.db.close(); } @@ -195,21 +262,146 @@ export class NmgStoreBase { if (scopeJson) this.scopeWriteIndexes.delete(scopeJson); else this.scopeWriteIndexes.clear(); } - putTaskBoardEntry(input: { + /** + * Run one coordinated transition for a run: exactly one write transaction, and inside it the + * scope that lets the run's own managed entries be written. It is deliberately not a way to wrap + * a whole round or an asynchronous RPC in a database transaction - the callback is synchronous + * and the boundary closes with it - and it refuses a run this store cannot name, so a scope can + * never authorize writes for a run state that does not exist. + */ + coordinateRunWrite(runId: string, work: (port: TransactionPort) => T): T { + if (!this.taskRunManifestExists(runId)) + throw new Error( + `run ${runId} is not registered; a managed write needs the run it belongs to`, + ); + return this.writeTransaction((port) => { + const previous = this.coordinatedRun; + this.coordinatedRun = runId; + try { + return work(port); + } finally { + this.coordinatedRun = previous; + } + }); + } + + /** + * A board write on an entry a run manages belongs to that run's coordinated scope, and the + * scope is the store's own state rather than the caller's claim - so a direct call on a managed + * entry is refused instead of applied beside the run. An entry no run manages returns here + * without a transaction: the ordinary board path is unchanged, including its cost. + */ + private requireManagedWriteScope(entryId: string): void { + const binding = this.taskRunForEntry(entryId); + if (!binding) return; + if (this.coordinatedRun === binding.runId) return; + throw new Error( + `entry ${entryId} is managed by run ${binding.runId}: its lifecycle writes go through the run's coordinated transition, not the board verb directly`, + ); + } + + /** + * The store owns the transaction boundary: this is the only place that runs BEGIN/COMMIT. A + * caller already inside a transition must join it with the port this issued rather than open a + * second one, and a write entry reached inside a live transaction without that port is refused + * instead of guessed at — `openTransaction` is the store's own state, never the caller's claim. + */ + writeTransaction(callback: (port: TransactionPort) => T): T { + if (this.openTransaction) + throw new Error("a write transaction is already open: join it with the port it issued"); + if (this.connectionQuarantined) + throw new Error("this connection is quarantined after a failed rollback"); + this.db.exec("BEGIN IMMEDIATE"); + const port: TransactionPort = { generation: ++this.transactionGeneration }; + this.openTransaction = { port, rollbackOnly: false }; + let value: T; + try { + value = callback(port); + refuseThenable(value, "a write transaction callback must be synchronous"); + } catch (error) { + this.openTransaction = null; + this.rollback(); + // The original failure is what the caller needs to see, whether ROLLBACK worked or not. + throw error; + } + const state = this.openTransaction; + this.openTransaction = null; + if (state?.rollbackOnly) { + this.rollback(); + throw new Error("the transaction was marked rollback-only by a failing operation"); + } + try { + this.db.exec("COMMIT"); + } catch (error) { + this.rollback(); + throw error; + } + return value; + } + + /** + * Join the transition a port was issued for, synchronously and only while its callback runs. A + * port from another store, a port whose callback has returned, and a second BEGIN are all + * refused: nothing here infers authority from a flag or a depth counter. + */ + withPort(port: TransactionPort, work: () => T): T { + const state = this.openTransaction; + if (!state || state.port !== port) + throw new Error("this port is not the store's live transaction scope"); + try { + const value = work(); + refuseThenable(value, "work inside a transaction must be synchronous"); + return value; + } catch (error) { + // Even if the caller catches this, the work up to the failure already happened. Only the + // outermost decides whether anything commits, and it will not. + state.rollbackOnly = true; + throw error; + } + } + + private rollback(): void { + try { + this.db.exec("ROLLBACK"); + } catch { + this.connectionQuarantined = true; + } + } + + putTaskBoardEntry( + input: { + taskId: string; + agentId: string; + sourceSessionId?: string; + kind: TaskBoardKind; + content: string; + expiresAt: string; + /** Directed delivery: stable agent_name to wake for this entry. */ + to?: string; + }, + port?: TransactionPort, + ): TaskBoardEntry { + // Standalone and composed writes share this implementation: outside a transition this opens + // one, and inside one it joins the open transition instead of running a second BEGIN. + return port + ? this.withPort(port, () => this.insertTaskBoardEntry(input)) + : this.writeTransaction(() => this.insertTaskBoardEntry(input)); + } + + /** The write itself, owning no boundary: whichever transaction is open decides whether it lands. */ + private insertTaskBoardEntry(input: { taskId: string; agentId: string; sourceSessionId?: string; kind: TaskBoardKind; content: string; expiresAt: string; - /** Directed delivery: stable agent_name to wake for this entry. */ to?: string; }): TaskBoardEntry { const now = new Date().toISOString(); this.pruneExpiredTaskBoardEntries(now, input.taskId); let id: string; - this.db.exec("BEGIN IMMEDIATE"); - try { + { // Global monotonic counter (single row, never recycled). The id = // _ is time-sortable, insertion-ordered for // same-millisecond entries, and globally unique across all channels. @@ -261,10 +453,6 @@ export class NmgStoreBase { input.to ?? null, serialState, ); - this.db.exec("COMMIT"); - } catch (error) { - this.db.exec("ROLLBACK"); - throw error; } return this.taskBoardEntry(id)!; } @@ -413,6 +601,7 @@ export class NmgStoreBase { agentId: string; resolution?: string; }): TaskBoardEntry { + this.requireManagedWriteScope(input.entryId); const existing = this.taskBoardEntry(input.entryId); if (!existing || existing.taskId !== input.taskId) { throw new Error(`task board entry not found in task ${input.taskId}`); @@ -449,6 +638,7 @@ export class NmgStoreBase { agentId: string; reason?: string; }): TaskBoardEntry { + this.requireManagedWriteScope(input.entryId); const existing = this.taskBoardEntry(input.entryId); if (!existing || existing.taskId !== input.taskId) { throw new Error(`task board entry not found in task ${input.taskId}`); @@ -484,6 +674,7 @@ export class NmgStoreBase { summary?: string; now?: string; }): TaskBoardEntry { + this.requireManagedWriteScope(input.entryId); const now = input.now ?? new Date().toISOString(); const existing = this.taskBoardEntry(input.entryId); if (!existing || existing.taskId !== input.taskId) { @@ -539,6 +730,7 @@ export class NmgStoreBase { reason?: string; now?: string; }): TaskBoardEntry { + this.requireManagedWriteScope(input.entryId); const now = input.now ?? new Date().toISOString(); const existing = this.taskBoardEntry(input.entryId); if (!existing || existing.taskId !== input.taskId) { @@ -627,6 +819,7 @@ export class NmgStoreBase { leaseSeconds?: number; now?: string; }): TaskBoardEntry { + this.requireManagedWriteScope(input.entryId); const now = input.now ?? new Date().toISOString(); this.pruneExpiredTaskBoardEntries(now, input.taskId); const existing = this.taskBoardEntry(input.entryId); @@ -702,6 +895,7 @@ export class NmgStoreBase { entryId: string; agentId: string; }): TaskBoardEntry { + this.requireManagedWriteScope(input.entryId); const result = this.db .prepare( `UPDATE task_board_entries @@ -854,6 +1048,7 @@ export class NmgStoreBase { reason?: string; now?: string; }): void { + this.requireManagedWriteScope(input.entryId); this.db .prepare( `INSERT INTO task_board_acks (id, entry_id, agent_id, acknowledged_at, reason) @@ -1147,8 +1342,7 @@ export class NmgStoreBase { /** Remove a memory reference from a chain. */ removeMemoryFromChain(input: { chainId: string; memoryId: string }): boolean { - this.db.exec("BEGIN IMMEDIATE"); - try { + return this.writeTransaction(() => { this.db .prepare( `DELETE FROM memory_chain_edges @@ -1158,12 +1352,8 @@ export class NmgStoreBase { const result = this.db .prepare("DELETE FROM memory_chain_members WHERE chain_id = ? AND memory_id = ?") .run(input.chainId, input.memoryId); - this.db.exec("COMMIT"); return result.changes > 0; - } catch (error) { - this.db.exec("ROLLBACK"); - throw error; - } + }); } // ── memory-chain DAG edges (pointers) ── @@ -1362,6 +1552,7 @@ export class NmgStoreBase { retainedUntil?: string | null; now?: string; }): boolean { + this.requireManagedWriteScope(input.entryId); const { taskId, entryId, owner, reason } = input; if (!taskId || !entryId) throw new Error("retention requires a channel and an entry"); if (!owner.trim()) throw new Error("retention owner required"); @@ -1385,6 +1576,7 @@ export class NmgStoreBase { /** Release this owner's pin. Other owners' pins on the same entry are untouched, and * the entry becomes prunable only when the last of them is gone. */ releaseTaskBoardRetention(input: { taskId: string; entryId: string; owner: string }): boolean { + this.requireManagedWriteScope(input.entryId); if (!input.taskId || !input.entryId) throw new Error("retention requires a channel and an entry"); if (!input.owner.trim()) throw new Error("retention owner required"); @@ -1540,6 +1732,318 @@ export class NmgStoreBase { entry.ackedBy = this.taskBoardAckMap([entry.id]).get(entry.id) ?? []; return entry; } + + // ---- Task run records. The writable half of the run namespace: a run's frozen manifest, its + // frozen task plan, and the facts it appends. Each write takes an optional port, so a caller + // that already holds the store's transaction joins it instead of opening a second one - the + // board write and the run fact of one transition have to land together or not at all. Nothing + // here creates a table or a connection: the schema owns both (src/core/store/schema.ts). + + /** Register a run's frozen identity. A retry after a lost response is a no-op; a second + * registration that names a different plan or policy is refused, because a run cannot be + * re-opened onto a different plan without becoming a different run. */ + registerTaskRun( + input: { + runId: string; + planDigest: string; + policy: string; + revision: string; + retention: string; + }, + port?: TransactionPort, + ): void { + return port + ? this.withPort(port, () => this.insertTaskRunManifest(input)) + : this.writeTransaction(() => this.insertTaskRunManifest(input)); + } + + private insertTaskRunManifest(input: { + runId: string; + planDigest: string; + policy: string; + revision: string; + retention: string; + }): void { + const existing = this.db + .prepare("SELECT plan_digest, policy FROM task_run_manifest WHERE run_id = ?") + .get(input.runId) as Row | undefined; + if (existing) { + if ( + String(existing.plan_digest) !== input.planDigest || + String(existing.policy) !== input.policy + ) + throw new Error( + `run ${input.runId} already froze a different plan; a new plan is a new run, not an overwrite`, + ); + return; + } + this.db + .prepare( + "INSERT INTO task_run_manifest (run_id, plan_digest, policy, revision, retention, created_at) VALUES (?, ?, ?, ?, ?, ?)", + ) + .run( + input.runId, + input.planDigest, + input.policy, + input.revision, + input.retention, + new Date().toISOString(), + ); + } + + /** Freeze one task of a run's plan. Frozen means frozen: the same task id with a different + * input, position or operation is refused rather than replaced, since the plan is the thing + * every later decision is read against. */ + freezeTaskRunTask( + input: { + runId: string; + taskId: string; + position: number; + revision: string; + input: string; + dependencies: readonly string[]; + effect: string; + waitEvent?: string | null; + operation?: string; + kind?: string; + patchFiles?: readonly string[] | null; + patchEditable?: readonly string[] | null; + }, + port?: TransactionPort, + ): void { + return port + ? this.withPort(port, () => this.insertTaskRunTask(input)) + : this.writeTransaction(() => this.insertTaskRunTask(input)); + } + + private insertTaskRunTask(input: { + runId: string; + taskId: string; + position: number; + revision: string; + input: string; + dependencies: readonly string[]; + effect: string; + waitEvent?: string | null; + operation?: string; + kind?: string; + patchFiles?: readonly string[] | null; + patchEditable?: readonly string[] | null; + }): void { + if (!this.taskRunManifestExists(input.runId)) + throw new Error(`run ${input.runId} is not registered; a task cannot be frozen into it`); + const dependencies = JSON.stringify(input.dependencies); + const existing = this.db + .prepare( + "SELECT input, dependencies, position, operation FROM task_run_tasks WHERE run_id = ? AND task_id = ?", + ) + .get(input.runId, input.taskId) as Row | undefined; + if (existing) { + const same = + String(existing.input) === input.input && + String(existing.dependencies) === dependencies && + Number(existing.position) === input.position && + String(existing.operation) === (input.operation ?? ""); + if (!same) + throw new Error( + `run ${input.runId} already froze task ${input.taskId} with a different definition`, + ); + return; + } + this.db + .prepare( + "INSERT INTO task_run_tasks (run_id, task_id, position, revision, input, dependencies, effect, wait_event, operation, kind, patch_files, patch_editable) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .run( + input.runId, + input.taskId, + input.position, + input.revision, + input.input, + dependencies, + input.effect, + input.waitEvent ?? null, + input.operation ?? "", + input.kind ?? "snapshot", + input.patchFiles ? JSON.stringify(input.patchFiles) : null, + input.patchEditable ? JSON.stringify(input.patchEditable) : null, + ); + } + + /** Append one run fact, if it is not already there. The fact's own identity (run, kind, task, + * attempt) is the duplicate key, which is what makes a retry after a lost response append once + * instead of twice; `recorded: false` says the fact was already known, and is not a failure. */ + appendTaskRunFact( + input: { + runId: string; + kind: string; + taskId?: string; + attempt?: number; + entryId?: string | null; + payload?: string | null; + }, + port?: TransactionPort, + ): { sequence: number; recorded: boolean } { + return port + ? this.withPort(port, () => this.insertTaskRunFact(input)) + : this.writeTransaction(() => this.insertTaskRunFact(input)); + } + + private insertTaskRunFact(input: { + runId: string; + kind: string; + taskId?: string; + attempt?: number; + entryId?: string | null; + payload?: string | null; + }): { sequence: number; recorded: boolean } { + const taskId = input.taskId ?? ""; + const attempt = input.attempt ?? 0; + const known = this.db + .prepare( + "SELECT sequence FROM task_run_facts WHERE run_id = ? AND kind = ? AND task_id = ? AND attempt = ?", + ) + .get(input.runId, input.kind, taskId, attempt) as Row | undefined; + if (known) return { sequence: Number(known.sequence), recorded: false }; + if (!this.taskRunManifestExists(input.runId)) + throw new Error(`run ${input.runId} is not registered; a fact cannot be appended to it`); + const next = this.db + .prepare("SELECT COALESCE(MAX(sequence), 0) + 1 AS next FROM task_run_facts WHERE run_id = ?") + .get(input.runId) as Row; + const sequence = Number(next.next); + this.db + .prepare( + "INSERT INTO task_run_facts (run_id, sequence, kind, task_id, attempt, entry_id, payload, recorded_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ) + .run( + input.runId, + sequence, + input.kind, + taskId, + attempt, + input.entryId ?? null, + input.payload ?? null, + new Date().toISOString(), + ); + return { sequence, recorded: true }; + } + + /** The run's frozen identity, or null when this store holds no such run. A read: it registers + * nothing and appends nothing, which is what lets a query path use it. */ + taskRunManifest(runId: string): { + runId: string; + planDigest: string; + policy: string; + revision: string; + retention: string; + createdAt: string; + } | null { + const row = this.db.prepare("SELECT * FROM task_run_manifest WHERE run_id = ?").get(runId) as + Row | undefined; + if (!row) return null; + return { + runId: String(row.run_id), + planDigest: String(row.plan_digest), + policy: String(row.policy), + revision: String(row.revision), + retention: String(row.retention), + createdAt: String(row.created_at), + }; + } + + /** The tasks the run froze, in plan order. */ + taskRunTasks(runId: string): { + taskId: string; + position: number; + revision: string; + input: string; + dependencies: string[]; + effect: string; + waitEvent: string | null; + operation: string; + kind: string; + patchFiles: string[] | null; + patchEditable: string[] | null; + }[] { + const rows = this.db + .prepare("SELECT * FROM task_run_tasks WHERE run_id = ? ORDER BY position, task_id") + .all(runId) as Row[]; + return rows.map((row) => ({ + taskId: String(row.task_id), + position: Number(row.position), + revision: String(row.revision), + input: String(row.input), + dependencies: JSON.parse(String(row.dependencies)) as string[], + effect: String(row.effect), + waitEvent: row.wait_event === null ? null : String(row.wait_event), + operation: String(row.operation), + kind: String(row.kind), + patchFiles: + row.patch_files === null ? null : (JSON.parse(String(row.patch_files)) as string[]), + patchEditable: + row.patch_editable === null ? null : (JSON.parse(String(row.patch_editable)) as string[]), + })); + } + + /** The run's appended facts in sequence order. `through` is how a caller asks for the facts as of + * one point in the log, which is what a replay or a re-derivation needs. */ + taskRunFacts( + runId: string, + through?: number, + ): { + sequence: number; + kind: string; + taskId: string; + attempt: number; + entryId: string | null; + payload: string | null; + recordedAt: string; + }[] { + const rows = this.db + .prepare("SELECT * FROM task_run_facts WHERE run_id = ? AND sequence <= ? ORDER BY sequence") + .all(runId, through ?? Number.MAX_SAFE_INTEGER) as Row[]; + return rows.map((row) => ({ + sequence: Number(row.sequence), + kind: String(row.kind), + taskId: String(row.task_id), + attempt: Number(row.attempt), + entryId: row.entry_id === null ? null : String(row.entry_id), + payload: row.payload === null ? null : String(row.payload), + recordedAt: String(row.recorded_at), + })); + } + + /** Which run a board entry is bound to, or null when nothing here manages it. This is the one + * question a caller holding only an entry id can ask: an entry bound by two runs is refused + * rather than answered with one of them. */ + taskRunForEntry( + entryId: string, + ): { runId: string; kind: string; taskId: string; attempt: number } | null { + const rows = this.db + .prepare( + "SELECT run_id, kind, task_id, attempt FROM task_run_facts WHERE entry_id = ? ORDER BY sequence", + ) + .all(entryId) as Row[]; + if (rows.length === 0) return null; + const runs = new Set(rows.map((row) => String(row.run_id))); + if (runs.size > 1) + throw new Error( + `entry ${entryId} is bound by ${runs.size} runs; a managed entry belongs to one`, + ); + const first = rows[0]!; + return { + runId: String(first.run_id), + kind: String(first.kind), + taskId: String(first.task_id), + attempt: Number(first.attempt), + }; + } + + private taskRunManifestExists(runId: string): boolean { + return ( + this.db.prepare("SELECT 1 FROM task_run_manifest WHERE run_id = ?").get(runId) !== undefined + ); + } cascadeDerivedMemories(sourceMemoryId: string): void { const derivations = this.db .prepare("SELECT derived_memory_id FROM memory_derivations WHERE source_memory_id = ?") @@ -1679,8 +2183,9 @@ export class NmgStoreBase { updated_at = excluded.updated_at`, ); const now = new Date().toISOString(); - this.db.exec("BEGIN IMMEDIATE"); - try { + // The store owns the boundary. The cache refresh stays outside it, so a batch whose + // transaction did not commit cannot warm the cache either. + this.writeTransaction(() => { for (const item of embeddings) { upsert.run( item.nodeId, @@ -1691,15 +2196,11 @@ export class NmgStoreBase { now, ); } - this.db.exec("COMMIT"); - for (const item of embeddings) { - this.updateVectorCache("node", model, item.nodeId, item.vector); - } - return embeddings.length; - } catch (error) { - this.db.exec("ROLLBACK"); - throw error; + }); + for (const item of embeddings) { + this.updateVectorCache("node", model, item.nodeId, item.vector); } + return embeddings.length; } storedNodeEmbeddings(model: string, afterNodeId = "", limit = 256): ExternalNodeEmbedding[] { const rows = this.db @@ -1759,8 +2260,8 @@ export class NmgStoreBase { updated_at = excluded.updated_at`, ); const now = new Date().toISOString(); - this.db.exec("BEGIN IMMEDIATE"); - try { + // As above: the boundary is the store's, and the cache is warmed only after it commits. + this.writeTransaction(() => { for (const item of embeddings) { upsert.run( item.blockId, @@ -1771,15 +2272,11 @@ export class NmgStoreBase { now, ); } - this.db.exec("COMMIT"); - for (const item of embeddings) { - this.updateVectorCache("leaf", model, item.blockId, item.vector); - } - return embeddings.length; - } catch (error) { - this.db.exec("ROLLBACK"); - throw error; + }); + for (const item of embeddings) { + this.updateVectorCache("leaf", model, item.blockId, item.vector); } + return embeddings.length; } storedLeafEmbeddings(model: string, afterBlockId = "", limit = 256): ExternalLeafEmbedding[] { const rows = this.db @@ -1837,8 +2334,7 @@ export class NmgStoreBase { updated_at = excluded.updated_at`, ); const now = new Date().toISOString(); - this.db.exec("BEGIN IMMEDIATE"); - try { + this.writeTransaction(() => { for (const item of embeddings) { upsert.run( item.memoryId, @@ -1849,12 +2345,8 @@ export class NmgStoreBase { now, ); } - this.db.exec("COMMIT"); - return embeddings.length; - } catch (error) { - this.db.exec("ROLLBACK"); - throw error; - } + }); + return embeddings.length; } storedEmbeddings(model: string, afterMemoryId = "", limit = 256): ExternalEmbedding[] { const rows = this.db @@ -2505,11 +2997,8 @@ export class NmgStoreBase { OR (? IS NULL AND m.session_id IS NULL))) AND m.status IN ('active', 'disputed', 'superseded') AND (? = 1 OR m.status IN ('active', 'disputed')) - AND (m.expires_at IS NULL OR m.expires_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) - AND (? = 1 OR ( - (m.valid_from IS NULL OR m.valid_from <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) - AND (m.valid_until IS NULL OR m.valid_until > strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) - )) + AND ${notExpired("m")} + AND (? = 1 OR ${currentlyValid("m")}) ORDER BY m.tier ASC, m.importance DESC, m.created_at DESC LIMIT ?`, ) @@ -2598,3 +3087,9 @@ function mapTaskBoardEntry(row: Row): TaskBoardEntry { function optionalText(value: unknown): string | null { return value === null || value === undefined ? null : String(value); } + +/** The handle options for one open: a read-only factory gets a handle that cannot write, which is a + * different factory rather than a mode of the shared connection. */ +function readOnlyOpenOptions(readOnly: boolean): { readOnly?: true } { + return readOnly ? { readOnly: true } : {}; +} diff --git a/src/core/store/clock.ts b/src/core/store/clock.ts new file mode 100644 index 00000000..5087c77e --- /dev/null +++ b/src/core/store/clock.ts @@ -0,0 +1,41 @@ +/** + * The clock the current-value windows are compared against, in one place. + * + * A write stamps its validity and expiry from JavaScript (`new Date()`), and a read compares against + * SQLite's `now`. Those are two clock readers and they disagree at millisecond granularity: measured on + * this machine, a row stamped `...T02:58:38.468Z` was read back while SQLite's `now` said `...38.467Z`, + * so `valid_from <= now` was false and a memory written a moment earlier read as "not active" (about one + * run in 1500, and the reason a product suite failed intermittently under load). + * + * A current-value window therefore applies a named grace, and applies it so it can only **widen** the + * window, never narrow it: a just-written value is current, and a value that expired a clock tick ago + * still reads as current. The cost is bounded by the constant below and is the same on every read path. + */ +export const CLOCK_GRACE_MS = 50; + +/** SQLite's `now` with the grace applied: `later` moves the instant into the future, `earlier` into the + * past. Both are ISO-8601 UTC with milliseconds - the format every stored boundary uses - so the + * comparison stays lexicographic. The unit is `seconds` with the grace written as a fraction: SQLite's + * date functions have no `milliseconds` modifier, and an unknown one makes the whole expression NULL, + * which silently excludes every row rather than erroring. */ +export function clockNow(direction: "later" | "earlier" = "later"): string { + const modifier = direction === "later" ? "+" : "-"; + const seconds = (CLOCK_GRACE_MS / 1000).toFixed(3); + return `strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '${modifier}${seconds} seconds')`; +} + +/** The freshness half of a current-value window for one table alias: not yet expired. Widened into the + * past, so a row whose expiry was stamped a clock tick before the read is still current. */ +export function notExpired(alias: string): string { + return `(${alias}.expires_at IS NULL OR ${alias}.expires_at > ${clockNow("earlier")})`; +} + +/** The validity half: `valid_from` has already started (widened into the future) and `valid_until` has + * not passed (widened into the past). Both are applied together, so the window includes more and never + * less - a future-dated value stays excluded, which is what the comparison is for. */ +export function currentlyValid(alias: string): string { + return ( + `((${alias}.valid_from IS NULL OR ${alias}.valid_from <= ${clockNow("later")})` + + ` AND (${alias}.valid_until IS NULL OR ${alias}.valid_until > ${clockNow("earlier")}))` + ); +} diff --git a/src/core/store/retrieval.ts b/src/core/store/retrieval.ts index 5e916115..34877364 100644 --- a/src/core/store/retrieval.ts +++ b/src/core/store/retrieval.ts @@ -13,6 +13,7 @@ import type { Constructor } from "./store-ctor.ts"; import { randomUUID } from "node:crypto"; import { extractEventWindow } from "./advanced-query.ts"; import { applyRelevanceGate } from "../relevance-gate.ts"; +import { notExpired } from "./clock.ts"; import { applyLearnedGate, rerankByRelevance } from "../learned-gate.ts"; import { nowMs, PerfTimer, SECTION } from "../perf.ts"; import type { PerfSnapshot } from "../perf.ts"; @@ -2093,7 +2094,7 @@ export function withRetrieval(Base: TBase) { AND (? IS NULL OR m.source_actor = ?) AND (? = 1 OR m.status IN ('active', 'disputed', 'superseded')) AND ((? IS NOT NULL AND (m.session_id IS NULL OR m.session_id = ?)) OR (? IS NULL AND m.session_id IS NULL)) - AND (m.expires_at IS NULL OR m.expires_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + AND ${notExpired("m")} AND (? IS NULL OR m.event_time >= ?) AND (? IS NULL OR m.event_time <= ?) ${scopeClause} diff --git a/src/core/store/schema.ts b/src/core/store/schema.ts index c5c10f5d..6a62b205 100644 --- a/src/core/store/schema.ts +++ b/src/core/store/schema.ts @@ -603,6 +603,60 @@ export function migrate(db: DatabaseSync): void { ON task_board_entries(task_id, status, created_at, id); CREATE INDEX IF NOT EXISTS idx_task_board_expiry ON task_board_entries(expires_at); + + -- Task run records: what a run froze and what it appended, in the same database, + -- connection and transaction as the board. Named for what they store rather than + -- for the feature that first needed them, because a run log is general: a plan + -- can be executed by anything that can read a board. + -- + -- Three tables and no fourth: the manifest is immutable, the facts are appended, + -- and the per-task current view is derived from the two (src/integration) instead + -- of stored, so nothing here can compete with the board as a second truth. + CREATE TABLE IF NOT EXISTS task_run_manifest ( + run_id TEXT PRIMARY KEY, + plan_digest TEXT NOT NULL, + policy TEXT NOT NULL, + revision TEXT NOT NULL, + retention TEXT NOT NULL, + created_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS task_run_tasks ( + run_id TEXT NOT NULL, + task_id TEXT NOT NULL, + position INTEGER NOT NULL, + revision TEXT NOT NULL, + input TEXT NOT NULL, + dependencies TEXT NOT NULL, + effect TEXT NOT NULL, + wait_event TEXT, + operation TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL DEFAULT 'snapshot', + patch_files TEXT, + patch_editable TEXT, + PRIMARY KEY (run_id, task_id) + ); + + -- Appended run facts, keyed by run and sequence. The unique key is the fact's own + -- identity, so a caller that retries after a lost response appends once: run, + -- kind, task and attempt name the fact, and an empty task id is how a run-level + -- fact (a cancellation, a plan revision) avoids colliding with a task's. + CREATE TABLE IF NOT EXISTS task_run_facts ( + run_id TEXT NOT NULL, + sequence INTEGER NOT NULL, + kind TEXT NOT NULL, + task_id TEXT NOT NULL DEFAULT '', + attempt INTEGER NOT NULL DEFAULT 0, + entry_id TEXT, + payload TEXT, + recorded_at TEXT NOT NULL, + PRIMARY KEY (run_id, sequence), + UNIQUE (run_id, kind, task_id, attempt) + ); + CREATE INDEX IF NOT EXISTS idx_task_run_facts_task + ON task_run_facts(run_id, task_id, sequence); + CREATE INDEX IF NOT EXISTS idx_task_run_facts_entry + ON task_run_facts(run_id, entry_id); `); ensureMemoryColumns(db); ensureHistoryColumns(db); diff --git a/src/core/types.ts b/src/core/types.ts index 422ee228..7dd84fdd 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -409,6 +409,8 @@ export interface RememberResult { /** Optional process-local accelerators. They never change persisted semantics. */ export interface NmgStoreOptions { + /** Owner factory configuration for a read-only open: no migration, no writes, no close checkpoint. */ + readonly readOnly?: boolean; /** Lazily cache same-scope write candidates to avoid repeated tokenization and scans. */ scopeWriteIndex?: boolean; } diff --git a/src/integration/ooo-verifier.ts b/src/integration/check-runner.ts similarity index 100% rename from src/integration/ooo-verifier.ts rename to src/integration/check-runner.ts diff --git a/src/integration/ooo-check.ts b/src/integration/check-ticket.ts similarity index 100% rename from src/integration/ooo-check.ts rename to src/integration/check-ticket.ts diff --git a/src/integration/ooo-board.ts b/src/integration/ooo-board.ts index a388f754..2873bcae 100644 --- a/src/integration/ooo-board.ts +++ b/src/integration/ooo-board.ts @@ -1,7 +1,10 @@ import { createHash, randomUUID } from "node:crypto"; +import { existsSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; import { NmgStore } from "../../src/core/store.ts"; -import { acceptedFact } from "./task-semantics.ts"; -import { checkResultValid, sameCheck, type CheckTicket, type CheckResult } from "./ooo-check.ts"; +import type { TransactionPort } from "../../src/core/store/base.ts"; +import { acceptedFact, deriveStatus } from "./task-semantics.ts"; +import { checkResultValid, sameCheck, type CheckTicket, type CheckResult } from "./check-ticket.ts"; import { patchCandidate, patchSubmission, @@ -29,7 +32,21 @@ export interface PatchTaskSpec { limits?: PatchLimits; verify: (submission: PatchSubmission) => Promise<"accept" | "reject" | "undecidable">; } -import { nextTask, snapshotAnswer, type SnapshotWork } from "./ooo-execution.ts"; +import { + checkedSlots, + remainingSlots, + selectableTasks, + snapshotAnswer, + type SnapshotWork, +} from "./ooo-execution.ts"; +import { compileTaskUnits, dispatchTasks, type RecordedFacts } from "./task-semantics.ts"; +import { + orderCandidates, + revalidateSuggestion, + type AdviceOutcome, + type SuggestionSource, +} from "./task-advisers.ts"; +import type { Refusal } from "./task-semantics.ts"; export type ProbePlan = readonly (readonly [ string, @@ -45,7 +62,11 @@ const arithmeticPlan: ProbePlan = [ ["C", "sum", ["A", "B"], "isolated-artifact", null, "sum"], ]; -export const channel = "ooo-process-probe"; +/** A run's board channel. One channel per run, so two rounds sharing a store cannot see, + * claim, or block each other's entries — a channel is the board's only boundary. */ +export function roundChannel(runId: string): string { + return `ooo-probe:${runId}`; +} /** Identity of an accepted artifact: sha256 of the canonical commit the round binds * dependents to. The board carries this short identity and never the bytes — the value @@ -115,6 +136,184 @@ export interface BoardTicket { patch?: FrozenPatchTask & { digest: string }; } +/** Which run of a store to open. Omitted, a store with exactly one run continues it and a + * store with several refuses the ambiguity rather than guessing. */ +/** The accepted artifacts by task id, read off a connection the caller owns. */ +export function readAccepted(db: DatabaseSync, runId: string): Record { + const rows = db + .prepare( + "SELECT id, artifact, source_revision, observed_revision FROM ooo_probe_task_view " + + "WHERE run_id=? AND artifact IS NOT NULL ORDER BY id", + ) + .all(runId) as unknown as Row[]; + // The verdict is looked up on the board by the digest of THIS attempt's artifact — no + // pointer column, because acceptance is the board's fact and retention keeps the entry + // readable. A verdict about another digest never transfers, and a later rejection of + // this digest withdraws acceptance. + const verdictOf = db.prepare( + `SELECT verdict, judged_digest FROM task_board_entries + WHERE task_id = ? AND deliverable_digest = ? AND verdict IS NOT NULL + ORDER BY judged_at DESC LIMIT 1`, + ); + const cancelled = readCancelled(db, runId) !== null; + const accepted: Record = {}; + for (const row of rows) { + const commit = String(row.artifact); + const digest = artifactDigest(commit); + const recorded = verdictOf.get(roundChannel(runId), digest) as unknown as + { verdict: string | null; judged_digest: string | null } | undefined; + if ( + !acceptedFact({ + artifact: commit, + digest, + verdict: recorded?.verdict ?? null, + judgedDigest: recorded?.judged_digest ?? null, + currentRevision: row.source_revision === row.observed_revision, + cancelled, + }) + ) + continue; + accepted[String(row.id)] = commit; + } + return accepted; +} +/** The round's terminal reason, or null while it is still running. */ +export function readCancelled(db: DatabaseSync, runId: string): string | null { + const runs = db.prepare("SELECT cancel_reason FROM ooo_probe_runs WHERE run_id=?").get(runId) as + { cancel_reason?: string | null } | undefined; + return runs?.cancel_reason ?? null; +} +/** The owner's narrow query port: typed reads only, no write, no raw connection, no SQL and no close. + * Constructing it creates nothing; the offline host opens its own read-only handle and asks for the + * same port, so one implementation serves both paths. */ +export interface RoundQueryPort { + readonly cancelled: () => string | null; + readonly accepted: () => Record; +} + +/** The offline host's read-only path: a true read-only handle, no migration, no initialisation and no + * publish, and nothing created when the file, the schema or the run is missing. The caller owns the + * handle it gets back. */ +export function openRoundQuery( + databasePath: string, + runId?: string, +): { readonly port: RoundQueryPort; readonly close: () => void } { + if (!existsSync(databasePath)) + throw new Error("this round store does not exist; a read-only view does not create one"); + const db = new DatabaseSync(databasePath, { readOnly: true }); + // A refusal has to close the handle it just opened: an open handle keeps the file locked on Windows. + // An explicit annotation on the variable is what lets the compiler narrow after the call. + const refuse: (message: string) => never = (message) => { + try { + db.close(); + } catch { + // Best effort; the refusal is what the caller needs. + } + throw new Error(message); + }; + const table = (name: string) => + db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name) !== undefined; + if (!table("ooo_probe_runs")) + refuse("this store carries no round schema; a read-only view does not migrate it"); + // A store with several runs refuses to guess, exactly as the writable path does; naming the run is + // the caller's job, because picking one is an answer to somebody's evidence. + const runs = db + .prepare("SELECT run_id FROM ooo_probe_runs ORDER BY created_at") + .all() as unknown as { + run_id: string; + }[]; + if (runId !== undefined && !runs.some((run) => run.run_id === runId)) + refuse(`this store holds no run ${runId}`); + const resolved = runId ?? (runs.length === 1 ? runs[0]!.run_id : undefined); + if (resolved === undefined) + refuse( + runs.length === 0 + ? "this store holds no run to read" + : `this store holds ${runs.length} runs; name the one to read`, + ); + return { + port: { + cancelled: () => readCancelled(db, resolved), + accepted: () => readAccepted(db, resolved), + }, + close: () => db.close(), + }; +} + +/** + * Advisers are optional. When one is given, the scope its scores are attributed to must be given too: + * a suggestion nobody can attribute to a session and a projection version is not adoptable, and + * discovering that later would mean a decision was made with an unattributable score. + * + * This lives outside the constructor so the admission's own branch budget stays what it was; the check + * is about the options, not about building an admission. + */ +function admissionAdvice(options: BoardAdmissionOptions): { + advisers: readonly SuggestionSource[]; + scope: BoardAdmissionOptions["adviceScope"]; +} { + const advisers = options.advisers ?? []; + if (advisers.length > 0 && !options.adviceScope) + throw new Error( + "advisers need an advice scope: a score nobody can attribute to a session and a projection " + + "version is not a suggestion the shared layer can adopt", + ); + return { advisers, scope: options.adviceScope }; +} + +/** + * A run that declares more than one claim must name who each handoff is offered to. + * + * The store keeps one outstanding un-directed actionable entry per channel and queues the next one as + * `pending`, so a second slot's handoff would be refused at claim time. A directed entry is exempt from + * that queue, which is why the target is required rather than optional - and why declaring a second slot + * does not change the store's own serialization. + */ +function admissionSlots(options: BoardAdmissionOptions): { + slots: number; + handoffTarget: ((taskId: string) => string) | undefined; +} { + const slots = checkedSlots(options.slots ?? 1); + if (slots > 1 && !options.handoffTarget) + throw new Error( + "a run with more than one slot must name each handoff's target: the board queues a second " + + "un-directed actionable behind the first, so the claim for it would be refused", + ); + return { slots, handoffTarget: options.handoffTarget }; +} + +export interface BoardAdmissionOptions { + runId?: string; + /** + * Optional suggestion sources, ordered. Absent is the rule policy, not a degraded mode: the same + * code path answers with the rule order, and the outcome records that nothing was adopted. + * + * This is the seam the design fixes (shared semantics computes the legal set, an optional HA/MGR + * source may rank inside it, the shared policy orders, the claim re-checks). Passing a source here + * does not enable one: no HA or MGR implementation is wired by default, and nothing about their + * gates changes. See `src/integration/task-advisers.ts`. + */ + advisers?: readonly SuggestionSource[]; + /** The projection identities a suggestion is scored against. Required when advisers are given. */ + adviceScope?: { + sessionId: string; + branchId: string; + parametersVersion: string; + projectionVersion: string; + observationOrder?: readonly string[]; + initialState?: string; + }; + /** + * How many claims this run may hold at once - the slot count the C arm varies, default `1`, which is + * the rule the repository already had. A claim spends one, the ordered legal set is cut to what is + * left, and the claim licence names only that part. Declaring more does not relax anything else: no + * dependency, no wait rule, no acceptance predicate and no store serialization changes. + */ + slots?: number; + /** Who each handoff is offered to, by task. Required when `slots > 1` (see `admissionSlots`). */ + handoffTarget?: (taskId: string) => string; +} + /** Experiment-only authority. Uses the real board store, not a second queue. * Mutable execution state lives in SQLite; workers never open the database. */ export class BoardAdmission extends NmgStore { @@ -122,15 +321,38 @@ export class BoardAdmission extends NmgStore { afterVerify: () => Promise = async () => {}; afterCommit: () => Promise = async () => {}; readonly runId: string; + /** The board channel this run publishes to; never shared with another run. */ + readonly channel: string; private readonly patchTasks: Readonly>; + /** The declared plan, kept because the shared compiler needs it and the store only keeps the + * rows it was expanded into. */ + private readonly plan: ProbePlan; + private readonly advisers: readonly SuggestionSource[]; + private readonly adviceScope: BoardAdmissionOptions["adviceScope"]; + /** The declared claim budget and who a handoff is offered to; both are the run's, not a task's. */ + private readonly slots: number; + private readonly handoffTarget: ((taskId: string) => string) | undefined; + /** What the last `next()` decided and why, for the run record. Never a second source of truth: + * the decision itself is the returned task, and this only says how it was reached. */ + private lastAdvice: AdviceOutcome | null = null; constructor( database: string, plan: ProbePlan = arithmeticPlan, patchTasks: Readonly> = {}, + options: BoardAdmissionOptions = {}, ) { + // The options are checked before the store is opened: a misdeclared budget or an unattributable + // suggestion is refused without creating or opening a database for it. + const advice = admissionAdvice(options); + const budget = admissionSlots(options); super(database); this.patchTasks = patchTasks; + this.plan = plan; + this.advisers = advice.advisers; + this.adviceScope = advice.scope; + this.slots = budget.slots; + this.handoffTarget = budget.handoffTarget; // A snapshot task can never carry host patch definitions. The opposite // direction is checked at claim time, because a round may install a task's // frozen envelope after construction but before it becomes claimable. @@ -138,95 +360,227 @@ export class BoardAdmission extends NmgStore { if (operation !== null && Object.hasOwn(patchTasks, id)) throw new Error("patch tasks and snapshot operations are mutually exclusive"); this.db.exec(` - CREATE TABLE IF NOT EXISTS ooo_probe_meta (id INTEGER PRIMARY KEY CHECK(id=1), run_id TEXT NOT NULL, policy TEXT NOT NULL); - CREATE TABLE IF NOT EXISTS ooo_probe_checks ( - task_id TEXT PRIMARY KEY, ticket TEXT NOT NULL, terminal TEXT, cancelled INTEGER NOT NULL DEFAULT 0 - ); - CREATE TABLE IF NOT EXISTS ooo_probe_tasks ( - id TEXT PRIMARY KEY, revision TEXT NOT NULL, input TEXT NOT NULL, dependencies TEXT NOT NULL, - entry_id TEXT UNIQUE, attempt INTEGER NOT NULL DEFAULT 0, owner TEXT, claim_time TEXT, - input_digest TEXT, artifact TEXT, position INTEGER NOT NULL, effect TEXT NOT NULL, - source_revision TEXT NOT NULL, observed_revision TEXT NOT NULL, - wait_event TEXT, external_ready INTEGER NOT NULL, operation TEXT NOT NULL, - kind TEXT NOT NULL DEFAULT 'snapshot', patch_files TEXT, patch_editable TEXT + CREATE TABLE IF NOT EXISTS ooo_probe_runs ( + run_id TEXT PRIMARY KEY, policy TEXT NOT NULL, cancel_reason TEXT, cancelled_at TEXT, + created_at TEXT NOT NULL ); `); - // Additive migration: a round store outlives the process that created it, so a new - // column must be added rather than assumed. Guarded by PRAGMA table_info, so it is - // idempotent and an existing store keeps its state. - this.ensureColumns("ooo_probe_tasks", { - // Durable pointer to the entry whose verdict accepted this artifact. `entry_id` - // cannot serve that role: publishReady() clears it for tasks that are neither - // selected nor live (to release the board's serial slot), while the verdict must - // stay reachable. A pointer, not an authority — the verdict remains the authority. - accepted_entry_id: "TEXT", - }); - this.ensureColumns("ooo_probe_meta", { - cancel_reason: "TEXT", - cancelled_at: "TEXT", - }); + this.createCheckTable(); + this.createManifestTables(); + this.migrateToTaskTables(); + // The additive `accepted_entry_id` column this store used to carry is gone with the table + // that held it. It was a pointer to the entry whose verdict accepted an artifact, and the + // verdict is looked up by artifact digest now — a migrated store keeps the artifact plus the + // entries that decide it, and keeps no stale pointer that a reader could mistake for one. + const wanted = `${policy}:${digest(plan)}`; + const runs = this.db + .prepare("SELECT run_id, policy FROM ooo_probe_runs ORDER BY created_at") + .all() as unknown as { run_id: string; policy: string }[]; + // A store with several runs refuses to guess which one was meant: adopting the newest and + // starting another are both silent answers to somebody's evidence. + let runId = options.runId ?? (runs.length === 1 ? runs[0]!.run_id : null); + if (runId === null && runs.length === 0) runId = randomUUID(); + if (runId === null) + this.refuse(`this store holds ${runs.length} runs; name the one to open with { runId }`); + const recorded = runs.find((run) => run.run_id === runId); + if (recorded && recorded.policy !== wanted) + this.refuse("probe policy changed; use a new database"); + this.runId = runId; + this.channel = roundChannel(runId); this.transaction(() => { this.db - .prepare("INSERT OR IGNORE INTO ooo_probe_meta (id, run_id, policy) VALUES (1, ?, ?)") - .run(randomUUID(), `${policy}:${digest(plan)}`); - const meta = this.db.prepare("SELECT * FROM ooo_probe_meta WHERE id=1").get()!; - if (meta.policy !== `${policy}:${digest(plan)}`) - throw new Error("probe policy changed; use a new database"); + .prepare( + "INSERT OR IGNORE INTO ooo_probe_runs (run_id, policy, created_at) VALUES (?, ?, ?)", + ) + .run(runId, wanted, new Date(this.now).toISOString()); for (const [ position, [id, input, dependencies, effect, event, operation], ] of plan.entries()) { const spec = patchTasks[id]; + // What the run froze, written once: the manifest is never updated afterwards. Beside it + // the facts of this task start empty (attempts and the external-ready event are facts the + // board does not carry), and the derived row is not created at all — a task with no + // derived row reads as one nobody has claimed. this.db .prepare( - "INSERT OR IGNORE INTO ooo_probe_tasks (id, revision, input, dependencies, position, effect, source_revision, observed_revision, wait_event, external_ready, operation, kind, patch_files, patch_editable) VALUES (?, 'v1', ?, ?, ?, ?, 'input-v1', 'input-v1', ?, ?, ?, ?, ?, ?)", + "INSERT OR IGNORE INTO ooo_probe_manifest (run_id, id, revision, input, dependencies, position, effect, source_revision, wait_event, operation, kind, patch_files, patch_editable) VALUES (?, ?, 'v1', ?, ?, ?, ?, 'input-v1', ?, ?, ?, ?, ?)", ) .run( + runId, id, spec ? spec.instruction : input, JSON.stringify(dependencies), position, effect, event, - event ? 0 : 1, operation ?? "", spec ? "patch" : "snapshot", spec ? JSON.stringify(spec.files) : null, spec ? JSON.stringify(spec.editable) : null, ); + this.db + .prepare( + "INSERT OR IGNORE INTO ooo_probe_facts (run_id, id, attempt, external_ready) VALUES (?, ?, 0, ?)", + ) + .run(runId, id, event ? 0 : 1); } }); this.publishReady(); - this.runId = String( - this.db.prepare("SELECT run_id FROM ooo_probe_meta WHERE id=1").get()!.run_id, - ); } - private transaction(operation: () => T): T { - this.db.exec("BEGIN IMMEDIATE"); - try { - const result = operation(); - this.db.exec("COMMIT"); - return result; - } catch (error) { - this.db.exec("ROLLBACK"); - throw error; + /** + * Three tables, because a task row carried three kinds of fact and the difference decides what + * may be rebuilt: what the run froze (immutable), what it appended (facts the board does not + * have), and what is recomputable from the frozen manifest (the digest of the input this attempt + * was claimed against). The claim holder is *not* in the last group: the board stops reporting + * `claimedBy` once the round resolves the entry, so who claimed is a fact, and a rebuild that + * tried to re-derive it would quietly lose it. Reads go through `ooo_probe_task_view`, which is a + * projection and not storage. + */ + private createManifestTables() { + this.db.exec(` + CREATE TABLE IF NOT EXISTS ooo_probe_manifest ( + run_id TEXT NOT NULL, id TEXT NOT NULL, revision TEXT NOT NULL, input TEXT NOT NULL, + dependencies TEXT NOT NULL, position INTEGER NOT NULL, effect TEXT NOT NULL, + source_revision TEXT NOT NULL, wait_event TEXT, operation TEXT NOT NULL, + kind TEXT NOT NULL DEFAULT 'snapshot', patch_files TEXT, patch_editable TEXT, + PRIMARY KEY (run_id, id) + ); + CREATE TABLE IF NOT EXISTS ooo_probe_facts ( + run_id TEXT NOT NULL, id TEXT NOT NULL, attempt INTEGER NOT NULL DEFAULT 0, + artifact TEXT, entry_id TEXT, external_ready INTEGER NOT NULL DEFAULT 0, + observed_revision TEXT, owner TEXT, claim_time TEXT, + PRIMARY KEY (run_id, id), UNIQUE (run_id, entry_id) + ); + CREATE TABLE IF NOT EXISTS ooo_probe_derived ( + run_id TEXT NOT NULL, id TEXT NOT NULL, input_digest TEXT, PRIMARY KEY (run_id, id) + ); + DROP VIEW IF EXISTS ooo_probe_task_view; + CREATE VIEW ooo_probe_task_view AS + SELECT m.run_id, m.id, m.revision, m.input, m.dependencies, m.position, m.effect, + m.source_revision, m.wait_event, m.operation, m.kind, m.patch_files, m.patch_editable, + COALESCE(f.attempt, 0) AS attempt, f.artifact, f.entry_id, + COALESCE(f.external_ready, 0) AS external_ready, + COALESCE(f.observed_revision, m.source_revision) AS observed_revision, + f.owner, f.claim_time, d.input_digest + FROM ooo_probe_manifest m + LEFT JOIN ooo_probe_facts f ON f.run_id = m.run_id AND f.id = m.id + LEFT JOIN ooo_probe_derived d ON d.run_id = m.run_id AND d.id = m.id; + `); + } + + private createCheckTable(name = "ooo_probe_checks") { + this.db.exec(` + CREATE TABLE IF NOT EXISTS ${name} ( + run_id TEXT NOT NULL, task_id TEXT NOT NULL, ticket TEXT NOT NULL, terminal TEXT, + cancelled INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (run_id, task_id) + ); + `); + } + + /** + * Bring a store written before this split up to it: the old single task table is copied into the + * manifest, the facts and the derived cache, and the old table is dropped. Both earlier shapes + * are accepted — the pre-namespace one, which states its run in the single-row meta table, and + * the run-scoped one — because a round's rows are its evidence, and re-attributing or discarding + * them would be worse than migrating them. + */ + private migrateToTaskTables(): void { + const columns = (table: string) => + new Set( + (this.db.prepare(`PRAGMA table_info(${table})`).all() as unknown as { name: string }[]).map( + (column) => column.name, + ), + ); + const meta = columns("ooo_probe_meta"); + const tasks = columns("ooo_probe_tasks"); + const checks = columns("ooo_probe_checks"); + if (tasks.size === 0) { + if (meta.size > 0) throw new Error("legacy round store has meta but no task table"); + return; } + const scoped = tasks.has("run_id"); + // A pre-namespace store states its run in the single-row meta table. Without that row the + // rows cannot be attributed to a run, and inventing one would manufacture history. + const legacy = meta.size + ? (this.db.prepare("SELECT * FROM ooo_probe_meta WHERE id=1").get() as unknown as + | { run_id: string; policy: string; cancel_reason?: string; cancelled_at?: string } + | undefined) + : undefined; + if (meta.size > 0 && !legacy) throw new Error("legacy round store has no meta row"); + if (!scoped && !legacy) + throw new Error( + "legacy round store has rows but no run identity; refusing to attribute them", + ); + const runId = legacy?.run_id ?? ""; + const needsChecks = checks.size > 0 && !checks.has("run_id"); + this.transaction(() => { + if (legacy) + this.db + .prepare( + "INSERT OR IGNORE INTO ooo_probe_runs (run_id, policy, cancel_reason, cancelled_at, created_at) VALUES (?, ?, ?, ?, ?)", + ) + .run( + legacy.run_id, + legacy.policy, + legacy.cancel_reason ?? null, + legacy.cancelled_at ?? null, + new Date(this.now).toISOString(), + ); + this.db.exec("ALTER TABLE ooo_probe_tasks RENAME TO ooo_probe_tasks_pre_split"); + // Filling the missing run column first keeps the copy below a single statement per table + // instead of one variant per shape. + if (!scoped) { + this.db.exec("ALTER TABLE ooo_probe_tasks_pre_split ADD COLUMN run_id TEXT"); + this.db.prepare("UPDATE ooo_probe_tasks_pre_split SET run_id=?").run(runId); + } + this.createManifestTables(); + this.db.exec(` + INSERT INTO ooo_probe_manifest (run_id, id, revision, input, dependencies, position, + effect, source_revision, wait_event, operation, kind, patch_files, patch_editable) + SELECT run_id, id, revision, input, dependencies, position, effect, source_revision, + wait_event, operation, kind, patch_files, patch_editable + FROM ooo_probe_tasks_pre_split; + INSERT INTO ooo_probe_facts (run_id, id, attempt, artifact, entry_id, external_ready, + observed_revision, owner, claim_time) + SELECT run_id, id, attempt, artifact, entry_id, external_ready, observed_revision, + owner, claim_time + FROM ooo_probe_tasks_pre_split; + INSERT INTO ooo_probe_derived (run_id, id, input_digest) + SELECT run_id, id, input_digest + FROM ooo_probe_tasks_pre_split; + `); + this.db.exec("DROP TABLE ooo_probe_tasks_pre_split"); + if (meta.size > 0) this.db.exec("DROP TABLE ooo_probe_meta"); + if (needsChecks) { + this.db.exec("ALTER TABLE ooo_probe_checks RENAME TO ooo_probe_checks_legacy"); + this.createCheckTable("ooo_probe_checks_scoped"); + this.db + .prepare( + `INSERT INTO ooo_probe_checks_scoped (run_id, task_id, ticket, terminal, cancelled) + SELECT ?, task_id, ticket, terminal, cancelled FROM ooo_probe_checks_legacy`, + ) + .run(runId); + this.db.exec("DROP TABLE ooo_probe_checks_legacy"); + this.db.exec("ALTER TABLE ooo_probe_checks_scoped RENAME TO ooo_probe_checks"); + } + }); } - private ensureColumns(table: string, columns: Readonly>) { - const existing = new Set( - (this.db.prepare(`PRAGMA table_info(${table})`).all() as unknown as { name: string }[]).map( - (column) => column.name, - ), - ); - for (const [name, type] of Object.entries(columns)) - if (!existing.has(name)) this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${name} ${type}`); + /** + * One transition, on the store's boundary. The port is what lets a write that owns a boundary of + * its own (a board publication) join this one instead of opening a second BEGIN, and only the + * store decides whether the transition commits. + */ + private transaction(work: (port: TransactionPort) => T): T { + return this.writeTransaction(work); } private row(id: string): Row { - const row = this.db.prepare("SELECT * FROM ooo_probe_tasks WHERE id=?").get(id) as unknown as - Row | undefined; + const row = this.db + .prepare("SELECT * FROM ooo_probe_task_view WHERE run_id=? AND id=?") + .get(this.runId, id) as unknown as Row | undefined; if (!row) throw new Error("unknown task"); return row; } @@ -291,7 +645,17 @@ export class BoardAdmission extends NmgStore { ]); } - private publish(kind: "handoff" | "decision", content: string): string { + /** + * Publish a board entry, joining the transition this call belongs to when it is given a port. + * Without one it opens its own boundary, which is why a caller inside a transition must pass it: + * a second BEGIN is refused rather than nested. + */ + private publish( + kind: "handoff" | "decision", + content: string, + port?: TransactionPort, + to?: string, + ): string { // The real board put owns its transaction. Recover a post-put/pre-link crash by adopting // the existing publication instead of creating another message — but only while it is still // open: a *resolved* publication with the same content is a finished handoff, and adopting @@ -300,41 +664,50 @@ export class BoardAdmission extends NmgStore { .prepare( "SELECT id FROM task_board_entries WHERE task_id=? AND agent_id='coordinator' AND kind=? AND content=? AND status='open' ORDER BY id LIMIT 1", ) - .get(channel, kind, content); + .get(this.channel, kind, content); if (existing) return String(existing.id); - return this.putTaskBoardEntry({ - taskId: channel, - agentId: "coordinator", - kind, - content, - expiresAt: new Date(this.now + 86_400_000).toISOString(), - }).id; + return this.putTaskBoardEntry( + { + taskId: this.channel, + agentId: "coordinator", + kind, + content, + to, + expiresAt: new Date(this.now + 86_400_000).toISOString(), + }, + port, + ).id; } - private publishReady(): void { + private publishReady(port?: TransactionPort): void { // Pending publications are derived from durable rows: a tiny transactional // outbox, drained by this single daemon after commit and on restart/retry. const delivered = this.db - .prepare("SELECT * FROM ooo_probe_tasks WHERE artifact IS NOT NULL ORDER BY id") - .all() as unknown as Row[]; + .prepare( + "SELECT * FROM ooo_probe_task_view WHERE run_id=? AND artifact IS NOT NULL ORDER BY id", + ) + .all(this.runId) as unknown as Row[]; for (const row of delivered) this.publish( "decision", JSON.stringify({ id: row.id, attempt: row.attempt, artifact: row.artifact }), + port, ); - // A published handoff for a task that is no longer the selected one must give the board's - // serial slot back. The plan can move past it (a waiting task became ready first, which is - // exactly what ordered execution does), and an unclaimed, unselected handoff would then - // block every later claim in the round. Nothing is fenced here: no ticket exists for a task - // nobody claimed, so only the publication is retired. - const selected = this.next(); + // A published handoff for a task that is no longer startable must give the board's serial + // slot back. The plan can move past it (a waiting task became ready first, which is exactly + // what ordered execution does), and an unclaimed, unstartable handoff would then block later + // claims in the round. With more than one declared slot the startable set has more than one + // member, and each of them keeps its handoff while it is startable or live. + const startable = this.startable(); for (const row of this.db - .prepare("SELECT * FROM ooo_probe_tasks WHERE entry_id IS NOT NULL ORDER BY id") - .all() as unknown as Row[]) { - if (row.id === selected || this.live(row)) continue; + .prepare( + "SELECT * FROM ooo_probe_task_view WHERE run_id=? AND entry_id IS NOT NULL ORDER BY id", + ) + .all(this.runId) as unknown as Row[]) { + if (startable.includes(row.id) || this.live(row)) continue; try { this.resolveTaskBoardEntry({ - taskId: channel, + taskId: this.channel, entryId: row.entry_id!, agentId: "coordinator", resolution: "no longer the selected task", @@ -347,19 +720,22 @@ export class BoardAdmission extends NmgStore { // readable past this entry's own TTL. if (!this.delivered(row)) this.releaseTaskBoardRetention({ - taskId: channel, + taskId: this.channel, entryId: row.entry_id!, owner: RETENTION_OWNER, }); - this.db.prepare("UPDATE ooo_probe_tasks SET entry_id=NULL WHERE id=?").run(row.id); + this.db + .prepare("UPDATE ooo_probe_facts SET entry_id=NULL WHERE run_id=? AND id=?") + .run(this.runId, row.id); } const rows = this.db - .prepare("SELECT * FROM ooo_probe_tasks WHERE entry_id IS NULL ORDER BY id") - .all() as unknown as Row[]; + .prepare("SELECT * FROM ooo_probe_task_view WHERE run_id=? AND entry_id IS NULL ORDER BY id") + .all(this.runId) as unknown as Row[]; const accepted = this.acceptedArtifacts(); for (const row of rows) { - // Do not occupy the board's serial outstanding slot with a waiting task. - if (row.id !== selected) continue; + // Do not occupy the board's serial outstanding slot with a waiting task, and (at more than one + // slot) do not offer a handoff for a task the run may not start yet. + if (!startable.includes(row.id)) continue; if ((JSON.parse(row.dependencies) as string[]).some((id) => !Object.hasOwn(accepted, id))) continue; const entryId = this.publish( @@ -373,13 +749,20 @@ export class BoardAdmission extends NmgStore { attempt: row.attempt, input: row.input, }), + port, + // Un-directed at the default budget, which is the broadcast handoff the round has always + // published. A declared budget above 1 offers each one point-to-point, because the store + // queues a second un-directed actionable entry behind the first. + this.slots > 1 ? this.handoffTarget!(row.id) : undefined, ); - this.db.prepare("UPDATE ooo_probe_tasks SET entry_id=? WHERE id=?").run(entryId, row.id); + this.db + .prepare("UPDATE ooo_probe_facts SET entry_id=? WHERE run_id=? AND id=?") + .run(entryId, this.runId, row.id); // Pin what the round is about to reference. Without this the entry could be pruned // on its own TTL while the round still needs its verdict, and acceptance would // silently disappear from a round that is still running. this.retainTaskBoardEntry({ - taskId: channel, + taskId: this.channel, entryId, owner: RETENTION_OWNER, reason: `round ${this.runId ?? "initial"} handoff for ${row.id}`, @@ -398,14 +781,19 @@ export class BoardAdmission extends NmgStore { if ( this.db .prepare( - "SELECT 1 FROM ooo_probe_checks c JOIN ooo_probe_tasks t ON c.task_id=t.id WHERE t.wait_event=?", + "SELECT 1 FROM ooo_probe_checks c JOIN ooo_probe_task_view t ON c.task_id=t.id AND c.run_id=t.run_id WHERE t.run_id=? AND t.wait_event=?", ) - .get(event) + .get(this.runId, event) ) throw new Error("managed check requires bound terminal evidence"); const changed = this.db - .prepare("UPDATE ooo_probe_tasks SET external_ready=1 WHERE wait_event=?") - .run(event); + .prepare( + // `wait_event` is part of the frozen manifest, so the fact is written to the tasks whose + // manifest says they are waiting on this event. A view over three tables is not updatable, + // which is why this is a subquery rather than a single-table predicate. + "UPDATE ooo_probe_facts SET external_ready=1 WHERE run_id=? AND id IN (SELECT id FROM ooo_probe_manifest WHERE run_id=? AND wait_event=?)", + ) + .run(this.runId, this.runId, event); if (!changed.changes) throw new Error("unknown external event"); this.publishReady(); } @@ -420,8 +808,8 @@ export class BoardAdmission extends NmgStore { throw new Error("task is not waiting"); if (row.source_revision !== row.observed_revision) throw new Error("stale check input"); const previous = this.db - .prepare("SELECT ticket FROM ooo_probe_checks WHERE task_id=?") - .get(id); + .prepare("SELECT ticket FROM ooo_probe_checks WHERE run_id=? AND task_id=?") + .get(this.runId, id); const attempt = previous ? (JSON.parse(String(previous.ticket)) as CheckTicket).attempt + 1 : 1; @@ -435,6 +823,9 @@ export class BoardAdmission extends NmgStore { // host refused the recorded artifact for a digest it could no longer reproduce. // Its scope is the store, and one store is one round, so this stays unique where // it has to be. + // Deliberately NOT run-scoped: a check id is what a round log records, and a replay runs + // under a new runId while reproducing the same attempts. Uniqueness in the store is the + // (run_id, task_id) key, so two runs may share a check id without colliding. checkId: digest([id, attempt]).slice(0, 32), attempt, inputDigest: this.inputDigest(row), @@ -442,8 +833,10 @@ export class BoardAdmission extends NmgStore { expiresAt: this.now + 60_000, }); this.db - .prepare("INSERT OR REPLACE INTO ooo_probe_checks VALUES (?, ?, NULL, 0)") - .run(id, JSON.stringify(ticket)); + .prepare( + "INSERT OR REPLACE INTO ooo_probe_checks (run_id, task_id, ticket, terminal, cancelled) VALUES (?, ?, ?, NULL, 0)", + ) + .run(this.runId, id, JSON.stringify(ticket)); return ticket; }); } @@ -452,15 +845,17 @@ export class BoardAdmission extends NmgStore { return this.transaction(() => { const current = this.checkRecord(ticket); if (!current || current.terminal !== null || current.cancelled) return false; - this.db.prepare("UPDATE ooo_probe_checks SET cancelled=1 WHERE task_id=?").run(ticket.taskId); + this.db + .prepare("UPDATE ooo_probe_checks SET cancelled=1 WHERE run_id=? AND task_id=?") + .run(this.runId, ticket.taskId); return true; }); } private checkRecord(ticket: CheckTicket) { const record = this.db - .prepare("SELECT * FROM ooo_probe_checks WHERE task_id=?") - .get(ticket.taskId); + .prepare("SELECT * FROM ooo_probe_checks WHERE run_id=? AND task_id=?") + .get(this.runId, ticket.taskId); return record && sameCheck(ticket, JSON.parse(String(record.ticket)) as CheckTicket) ? record : null; @@ -482,11 +877,13 @@ export class BoardAdmission extends NmgStore { if (record.terminal !== null) return record.terminal === terminal ? "duplicate" : "rejected"; if (result.ticket.expiresAt <= this.now) return "stale"; this.db - .prepare("UPDATE ooo_probe_checks SET terminal=? WHERE task_id=?") - .run(terminal, row.id); + .prepare("UPDATE ooo_probe_checks SET terminal=? WHERE run_id=? AND task_id=?") + .run(terminal, this.runId, row.id); // Unknown completion is not evidence that the wait resolved. if (result.outcome !== "undecidable") - this.db.prepare("UPDATE ooo_probe_tasks SET external_ready=1 WHERE id=?").run(row.id); + this.db + .prepare("UPDATE ooo_probe_facts SET external_ready=1 WHERE run_id=? AND id=?") + .run(this.runId, row.id); return "accepted"; }); this.publishReady(); @@ -496,37 +893,153 @@ export class BoardAdmission extends NmgStore { observeRevision(id: string, revision: string): void { if (!revision.trim()) throw new Error("revision required"); this.row(id); - this.db.prepare("UPDATE ooo_probe_tasks SET observed_revision=? WHERE id=?").run(revision, id); + this.db + .prepare("UPDATE ooo_probe_facts SET observed_revision=? WHERE run_id=? AND id=?") + .run(revision, this.runId, id); this.publishReady(); } next(): string | null { + return this.candidates()[0] ?? null; + } + + /** + * The ordered legal set: every task the rules allow right now, in the order the shared policy + * would dispatch them. `next()` is its head, so a caller that starts more than one at a time + * cannot disagree with a caller that starts one - which is the difference between the two arms of + * the granularity comparison, and the reason this is exposed rather than re-derived by a driver. + * + * The rules are the shared semantics' and the ordering is the shared policy's; both live here, + * once. This is the whole legal set, not the part a run may start: `startable()` is that part. + */ + candidates(): readonly string[] { + return this.ordered().order; + } + + /** + * The part of the ordered candidate set this run may start now, which is the licence a claim is + * checked against: the ordered set cut to the budget that is left (`slots` minus the claims in + * flight). At the default one slot that is the head - the rule the claim always had - and the cut is + * applied after ordering, so an ordering step still decides which legal task comes first. + */ + startable(): readonly string[] { + const { order, room } = this.ordered(); + return order.slice(0, room); + } + + private ordered(): { order: readonly string[]; room: number } { // A cancelled round selects nothing: the successor is not "the next task", it is // the explicit terminal decision the caller asked for. - if (this.cancelled() !== null) return null; + if (this.cancelled() !== null) { + this.lastAdvice = null; + return { order: [], room: 0 }; + } const rows = this.db - .prepare("SELECT * FROM ooo_probe_tasks ORDER BY position") - .all() as unknown as Row[]; - const accepted = this.acceptedArtifacts(); - // A task whose artifact is delivered but no longer accepted is not selectable: the - // bytes still exist, so it cannot be claimed again either, and selecting it would - // publish a handoff nobody can claim. The coordinator recovers it with reopen(). - const schedulable = rows.filter( - (row) => !this.delivered(row) || Object.hasOwn(accepted, row.id), - ); - return nextTask( - schedulable.map((row) => ({ - id: row.id, - effect: row.effect, - sourceVersion: row.source_revision, - observedVersion: row.observed_revision, - dependencies: JSON.parse(row.dependencies) as string[], - accepted: Object.hasOwn(accepted, row.id), - claimed: this.live(row), - externalEvent: row.wait_event ?? undefined, - externalReady: row.external_ready === 1, - })), + .prepare("SELECT * FROM ooo_probe_task_view WHERE run_id=? ORDER BY position") + .all(this.runId) as unknown as Row[]; + // The store is the fact source and the shared semantics is the rule. This used to be a second + // selection implementation standing beside task-semantics.ts, which is how a plan the compiler + // refuses could still be scheduled. + const compiled = compileTaskUnits({ plan: this.plan, specs: this.patchTasks }); + if (!compiled.legal) { + // No silent degradation: a plan the compiler refuses is refused here, by name, with the + // refusals that say why - not scheduled on a hand-rolled reading of the same rows. + const first = compiled.refusals[0]; + throw new Error( + `plan refused by the shared semantics: ${compiled.refusals.length} refusal(s); ` + + `first is ${String(first?.task)}/${String(first?.field)}: ${String(first?.reason)}`, + ); + } + const dispatch = dispatchTasks(compiled.units, this.recordedFacts(rows)); + const legal = selectableTasks(dispatch, this.slots); + const room = remainingSlots(dispatch, this.slots); + if (this.advisers.length === 0) { + // No source: the rule policy, which is the legal set in its own order. This is the path a run + // without HA/MGR takes, and it is identical to the rule alone. + this.lastAdvice = null; + return { order: legal, room }; + } + const scope = this.adviceScope!; + const status = deriveStatus(compiled.units, this.recordedFacts(rows), this.slots); + const outcome = orderCandidates( + legal, + { + sessionId: scope.sessionId, + branchId: scope.branchId, + parametersVersion: scope.parametersVersion, + projectionVersion: scope.projectionVersion, + observationOrder: scope.observationOrder, + initialState: scope.initialState, + ready: status.ready, + accepted: status.accepted, + blocked: Object.fromEntries(status.blocked.map((entry) => [entry.id, entry.waitingFor])), + }, + this.advisers, ); + this.lastAdvice = outcome; + return { order: outcome.order, room }; + } + + /** + * What the last `next()` adopted, refused, fell back on or had to re-score. + * + * Kept for the run record: the decision is the returned task, and this says how a source's score + * entered it - including the case where nothing did. A source that proposed an action outside the + * legal set, claimed a different session's score, or could not name the inputs of its own score + * leaves evidence here rather than disappearing silently. + */ + lastAdviceOutcome(): AdviceOutcome | null { + return this.lastAdvice; + } + + /** + * The claim-side check the design names: an adopted ranking is not a write licence. + * + * The caller computes the legal set again at the moment it writes the claim, and this refuses a + * ranking whose task left it in between (accepted, cancelled, claimed by someone else). + */ + refuseStaleRanking(taskId: string, legalNow: readonly string[]): Refusal | null { + const adopted = this.lastAdvice?.adopted.find((entry) => entry.taskId === taskId); + return adopted ? revalidateSuggestion(adopted, legalNow) : null; + } + + /** What the store recorded, in the shape the shared compiler reads. Acceptance comes from the + * board's single acceptance reader, so the derived view cannot disagree with it about which + * artifacts were accepted - only about eligibility, which is the compiler's business. */ + private recordedFacts(rows: readonly Row[]): RecordedFacts { + const facts: { + artifacts: Record; + verdicts: Record< + string, + { digest: string; verdict: "accepted" | "rejected" | "undecidable" } + >; + revisions: Record; + sourceRevisions: Record; + externalReady: string[]; + claimed: string[]; + } = { + artifacts: {}, + verdicts: {}, + revisions: {}, + sourceRevisions: {}, + externalReady: [], + claimed: [], + }; + for (const row of rows) { + const artifact = this.acceptedArtifacts()[row.id] ?? row.artifact ?? null; + if (artifact !== null) { + const accepted = Object.hasOwn(this.acceptedArtifacts(), row.id); + facts.artifacts[row.id] = artifact; + // The verdict is bound to the artifact it judged: bytes that exist without an accepted + // verdict for those bytes are delivered, not accepted. + facts.verdicts[row.id] = { digest: artifact, verdict: accepted ? "accepted" : "rejected" }; + } + facts.revisions[row.id] = row.observed_revision; + facts.sourceRevisions[row.id] = row.source_revision; + if (row.external_ready === 1) facts.externalReady.push(row.id); + if (this.live(row)) facts.claimed.push(row.id); + } + return facts; } claim(id: string, agentId: string): BoardTicket { @@ -536,7 +1049,7 @@ export class BoardAdmission extends NmgStore { const row = this.claimableRow(id); const dependencies = this.inputs(row); const entry = this.claimTaskBoardEntry({ - taskId: channel, + taskId: this.channel, entryId: row.entry_id!, agentId, leaseSeconds: 60, @@ -555,11 +1068,15 @@ export class BoardAdmission extends NmgStore { if (!row.operation && !spec) throw new Error("patch task has no host spec"); const frozen = spec ? this.patchFrozen({ ...row, attempt }, attempt) : null; const inputDigest = frozen ? frozen.digest : this.inputDigest(row); + // The attempt, the holder and the claim time are facts: the board stops reporting its claim + // once the entry is resolved, so they cannot be re-derived. The input digest is the cache, + // recomputable from the frozen manifest at any time. this.db .prepare( - "UPDATE ooo_probe_tasks SET attempt=?, owner=?, claim_time=?, input_digest=? WHERE id=?", + "UPDATE ooo_probe_facts SET attempt=?, owner=?, claim_time=? WHERE run_id=? AND id=?", ) - .run(attempt, agentId, entry.claimedAt, inputDigest, id); + .run(attempt, agentId, entry.claimedAt, this.runId, id); + this.putInputDigest(id, inputDigest); return { runId: this.runId, taskId: id, @@ -595,7 +1112,10 @@ export class BoardAdmission extends NmgStore { throw new Error("unfulfilled dependencies"); if (!row.entry_id) throw new Error("no published handoff for this task"); if (this.live(row)) throw new Error("task already claimed"); - if (this.next() !== id) throw new Error("task not selected by narrow dispatch"); + // The licence is the startable part of the ordered set: the head at the default budget, and as + // many as the run has slots for when it declared more. The message keeps its name because a + // caller refuses this claim for the same reason either way - the run is not allowed to start it. + if (!this.startable().includes(id)) throw new Error("task not selected by narrow dispatch"); return row; } @@ -617,7 +1137,7 @@ export class BoardAdmission extends NmgStore { } private live(row: Row): boolean { - const entry = row.entry_id ? this.getTaskBoardEntryById(channel, row.entry_id) : null; + const entry = row.entry_id ? this.getTaskBoardEntryById(this.channel, row.entry_id) : null; return ( entry !== null && entry.status === "open" && @@ -655,6 +1175,23 @@ export class BoardAdmission extends NmgStore { } } + /** The last post-commit notification failure, or null. A commit that landed is not undone by a + * subscriber that could not be reached, so the two outcomes are reported separately: a caller + * that reads a failed notification as "the submission failed" submits the same work twice. */ + private notificationFailure: string | null = null; + + lastNotificationFailure(): string | null { + return this.notificationFailure; + } + + /** A host freezes its tasks from its own options, not from the store it opened, so it has to + * install them on that store explicitly - a factory that opens the store cannot know them. This + * used to happen by handing the constructor an object literal and then filling that same object + * in, which worked only while the caller also created the store. */ + installPatchTask(id: string, spec: PatchTaskSpec): void { + (this.patchTasks as Record)[id] = spec; + } + async submit(resultId: string): Promise { const parsed = this.readSubmission(resultId); if (typeof parsed === "string") return parsed; @@ -671,8 +1208,15 @@ export class BoardAdmission extends NmgStore { if (this.patchSpec(row) && !(await this.verifyCandidate(row, commit))) return "rejected"; await this.afterVerify(); const verdict = this.commitArtifact(ticket, commit); - await this.afterCommit(); - this.publishReady(); + // Past this line the round has the artifact: `verdict` is what happened. Everything below is + // notification, and failing to reach a subscriber is recorded rather than thrown, because the + // commit is not undone and a caller must not be told it was. + try { + await this.afterCommit(); + this.publishReady(); + } catch (error) { + this.notificationFailure = error instanceof Error ? error.message : String(error); + } return verdict; } @@ -681,10 +1225,10 @@ export class BoardAdmission extends NmgStore { private readSubmission( resultId: string, ): { ticket: BoardTicket; artifact: string; row: Row } | "rejected" | "stale" { - const result = this.getTaskBoardEntryById(channel, resultId); + const result = this.getTaskBoardEntryById(this.channel, resultId); if ( !result || - result.taskId !== channel || + result.taskId !== this.channel || result.kind !== "result" || Date.parse(result.expiresAt) <= this.now ) @@ -722,7 +1266,9 @@ export class BoardAdmission extends NmgStore { if (!this.bound(ticket, row)) return "stale"; if (this.delivered(row)) return row.artifact === commit ? "duplicate" : "rejected"; if (!this.live(row)) return "stale"; - this.db.prepare("UPDATE ooo_probe_tasks SET artifact=? WHERE id=?").run(commit, row.id); + this.db + .prepare("UPDATE ooo_probe_facts SET artifact=? WHERE run_id=? AND id=?") + .run(commit, this.runId, row.id); // Acceptance is recorded as protocol, not as a self-report. The artifact is // delivered against the claim it belongs to — the holder's own attempt, so // deliveredBy is the agent that produced it, recorded by the coordinator on @@ -734,7 +1280,7 @@ export class BoardAdmission extends NmgStore { // read wall-clock time inside a protocol write. const now = new Date(this.now).toISOString(); this.deliverTaskBoardEntry({ - taskId: channel, + taskId: this.channel, entryId: row.entry_id!, agentId: ticket.owner, digest: artifactDigest(commit), @@ -742,7 +1288,7 @@ export class BoardAdmission extends NmgStore { now, }); this.judgeTaskBoardEntry({ - taskId: channel, + taskId: this.channel, entryId: row.entry_id!, agentId: "coordinator", verdict: "accepted", @@ -756,7 +1302,7 @@ export class BoardAdmission extends NmgStore { // that entry readable past its own TTL — so the fact stays derivable instead of // being stored twice. this.resolveTaskBoardEntry({ - taskId: channel, + taskId: this.channel, entryId: row.entry_id!, agentId: "coordinator", resolution: "verified current attempt", @@ -783,7 +1329,7 @@ export class BoardAdmission extends NmgStore { private releaseRowRetention(row: Row): void { if (row.entry_id) this.releaseTaskBoardRetention({ - taskId: channel, + taskId: this.channel, entryId: row.entry_id, owner: RETENTION_OWNER, }); @@ -797,52 +1343,17 @@ export class BoardAdmission extends NmgStore { const digest = artifactDigest(commit); const entries = this.db .prepare("SELECT id FROM task_board_entries WHERE task_id = ? AND deliverable_digest = ?") - .all(channel, digest) as unknown as { id: string }[]; + .all(this.channel, digest) as unknown as { id: string }[]; for (const entry of entries) this.releaseTaskBoardRetention({ - taskId: channel, + taskId: this.channel, entryId: String(entry.id), owner: RETENTION_OWNER, }); } - private acceptedArtifacts(): Record { - const rows = this.db - .prepare( - "SELECT id, artifact, source_revision, observed_revision FROM ooo_probe_tasks " + - "WHERE artifact IS NOT NULL ORDER BY id", - ) - .all() as unknown as Row[]; - // The verdict is looked up on the board by the digest of THIS attempt's artifact — no - // pointer column, because acceptance is the board's fact and retention keeps the entry - // readable. A verdict about another digest never transfers, and a later rejection of - // this digest withdraws acceptance. - const verdictOf = this.db.prepare( - `SELECT verdict, judged_digest FROM task_board_entries - WHERE task_id = ? AND deliverable_digest = ? AND verdict IS NOT NULL - ORDER BY judged_at DESC LIMIT 1`, - ); - const cancelled = this.cancelled() !== null; - const accepted: Record = {}; - for (const row of rows) { - const commit = String(row.artifact); - const digest = artifactDigest(commit); - const recorded = verdictOf.get(channel, digest) as unknown as - { verdict: string | null; judged_digest: string | null } | undefined; - if ( - !acceptedFact({ - artifact: commit, - digest, - verdict: recorded?.verdict ?? null, - judgedDigest: recorded?.judged_digest ?? null, - currentRevision: row.source_revision === row.observed_revision, - cancelled, - }) - ) - continue; - accepted[String(row.id)] = commit; - } - return accepted; + acceptedArtifacts(): Record { + return readAccepted(this.db, this.runId); } /** The accepted artifacts, by task id — the values dependents bind to. */ @@ -870,13 +1381,15 @@ export class BoardAdmission extends NmgStore { const withdrawn: string[] = []; this.transaction(() => { for (const row of this.db - .prepare("SELECT * FROM ooo_probe_tasks ORDER BY position") - .all() as unknown as Row[]) + .prepare("SELECT * FROM ooo_probe_task_view WHERE run_id=? ORDER BY position") + .all(this.runId) as unknown as Row[]) this.fenceRow(row, withdrawn); - this.db.prepare("UPDATE ooo_probe_checks SET terminal='cancelled', cancelled=1").run(); this.db - .prepare("UPDATE ooo_probe_meta SET cancel_reason=?, cancelled_at=? WHERE id=1") - .run(reason.slice(0, 1_000), new Date(this.now).toISOString()); + .prepare("UPDATE ooo_probe_checks SET terminal='cancelled', cancelled=1 WHERE run_id=?") + .run(this.runId); + this.db + .prepare("UPDATE ooo_probe_runs SET cancel_reason=?, cancelled_at=? WHERE run_id=?") + .run(reason.slice(0, 1_000), new Date(this.now).toISOString(), this.runId); }); this.publish("decision", `cancel: ${reason}`.slice(0, 1_000)); return withdrawn; @@ -902,7 +1415,7 @@ export class BoardAdmission extends NmgStore { if (row.entry_id) try { this.resolveTaskBoardEntry({ - taskId: channel, + taskId: this.channel, entryId: row.entry_id, agentId: "coordinator", resolution, @@ -916,16 +1429,60 @@ export class BoardAdmission extends NmgStore { if (row.artifact !== null || row.owner !== null) dropped.push(row.id); this.db .prepare( - "UPDATE ooo_probe_tasks SET artifact=NULL, attempt=attempt+1, owner=NULL, claim_time=NULL, external_ready=0, entry_id=NULL WHERE id=?", + "UPDATE ooo_probe_facts SET artifact=NULL, attempt=attempt+1, owner=NULL, claim_time=NULL, external_ready=0, entry_id=NULL WHERE run_id=? AND id=?", ) - .run(row.id); + .run(this.runId, row.id); + this.putInputDigest(row.id, null); + } + + /** Warm the cache for one task. There is exactly one derived column, and this is its writer. */ + private putInputDigest(id: string, digest: string | null): void { + this.db + .prepare( + `INSERT INTO ooo_probe_derived (run_id, id, input_digest) VALUES (?, ?, ?) + ON CONFLICT(run_id, id) DO UPDATE SET input_digest=excluded.input_digest`, + ) + .run(this.runId, id, digest); + } + + /** + * Rebuild the cache from what decides it: the frozen manifest names the input digest of the + * attempt a task was claimed at. Nothing here is authoritative, so this is safe to call whenever + * the cache is suspect - and if it produces a different answer than the cache held, the cache was + * wrong, not the sources. + */ + refreshDerived(): number { + const rows = this.db + .prepare("SELECT * FROM ooo_probe_task_view WHERE run_id=? ORDER BY position") + .all(this.runId) as unknown as Row[]; + return this.transaction(() => { + let rebuilt = 0; + for (const row of rows) { + const frozen = + row.attempt >= 1 && this.patchTasks[row.id] !== undefined + ? this.patchFrozen(row, row.attempt) + : null; + // A task with no attempt has no digest to record: the ticket mints one at claim time. + this.putInputDigest( + row.id, + row.attempt >= 1 ? (frozen?.digest ?? this.inputDigest(row)) : null, + ); + rebuilt += 1; + } + return rebuilt; + }); + } + + /** Refusing a store must not leave its file handle behind: the caller is told to open a + * different database, and on Windows an open handle keeps that file locked. */ + private refuse(message: string): never { + this.db.close(); + throw new Error(message); } /** The round's terminal reason, or null while it is still running. */ cancelled(): string | null { - const meta = this.db.prepare("SELECT * FROM ooo_probe_meta WHERE id=1").get() as - { cancel_reason?: string | null } | undefined; - return meta?.cancel_reason ?? null; + return readCancelled(this.db, this.runId); } /** Explicit terminal decision for a check that never reported: otherwise a wait only @@ -936,8 +1493,10 @@ export class BoardAdmission extends NmgStore { const current = this.checkRecord(ticket); if (!current || current.terminal !== null) return false; this.db - .prepare("UPDATE ooo_probe_checks SET terminal='undecidable', cancelled=1 WHERE task_id=?") - .run(ticket.taskId); + .prepare( + "UPDATE ooo_probe_checks SET terminal='undecidable', cancelled=1 WHERE run_id=? AND task_id=?", + ) + .run(this.runId, ticket.taskId); return true; }); if (decided) @@ -949,8 +1508,8 @@ export class BoardAdmission extends NmgStore { const invalidated: string[] = []; this.transaction(() => { const rows = this.db - .prepare("SELECT * FROM ooo_probe_tasks ORDER BY position") - .all() as unknown as Row[]; + .prepare("SELECT * FROM ooo_probe_task_view WHERE run_id=? ORDER BY position") + .all(this.runId) as unknown as Row[]; const affected = new Set([id]); // Transitive dependents: an artifact built from a value that no longer exists // must not stay accepted. @@ -972,7 +1531,7 @@ export class BoardAdmission extends NmgStore { if (row.entry_id) try { this.resolveTaskBoardEntry({ - taskId: channel, + taskId: this.channel, entryId: row.entry_id, agentId: "coordinator", resolution: "invalidated by reopen", @@ -985,9 +1544,10 @@ export class BoardAdmission extends NmgStore { this.releaseRowRetention(row); this.db .prepare( - "UPDATE ooo_probe_tasks SET artifact=NULL, attempt=attempt+1, owner=NULL, claim_time=NULL, external_ready=0, entry_id=NULL WHERE id=?", + "UPDATE ooo_probe_facts SET artifact=NULL, attempt=attempt+1, owner=NULL, claim_time=NULL, external_ready=0, entry_id=NULL WHERE run_id=? AND id=?", ) - .run(taskId); + .run(this.runId, taskId); + this.putInputDigest(taskId, null); } }); this.publish("decision", `reopen ${invalidated.join(",")}: ${reason}`.slice(0, 1_000)); diff --git a/src/integration/ooo-cycle.ts b/src/integration/ooo-cycle.ts deleted file mode 100644 index 18d20f39..00000000 --- a/src/integration/ooo-cycle.ts +++ /dev/null @@ -1,1039 +0,0 @@ -import { mkdtempSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { BoardAdmission, type PatchTaskSpec, type ProbePlan } from "./ooo-board.ts"; -import { verifyCandidate, type CandidateCheck } from "./ooo-candidate.ts"; -import { - preparePatchWork, - patchSubmission, - type FrozenPatchTask, - type FrozenPatchWork, - type PatchSubmission, - type ConclusionKind, -} from "./ooo-patch.ts"; - -import { mutate, type Mutation } from "./ooo-mutation.ts"; -import { RoundLog, checksDigest, type RoundEvent, type RoundEventInput } from "./ooo-round-log.ts"; - -export type WorkerMetrics = { - tokens?: number; - turns?: number; - checks?: number; - cacheRead?: number; - cacheWrite?: number; -}; - -export type WorkerResult = - | string - | { - artifact: string; - metrics?: WorkerMetrics; - /** Reported mid-attempt: the input cannot satisfy a declared requirement, so the - * attempt ends without an artifact and the host decides about the dependency. */ - pushback?: { dependency: string; requirement: string; evidence: string }; - }; - -export type CycleWorker = ( - taskId: string, - frozen: FrozenPatchWork, - dependencies: Readonly>, -) => Promise; - -/** A declared precondition on a dependency's accepted artifact. The host evaluates - * these mechanically, so a downstream pushback is a checkable fact and not a mood. */ -export type Requirement = - | { kind: "verified"; task: string } - | { kind: "mutant-killed"; task: string; id: string } - | { kind: "test-title"; task: string; token: string }; - -export interface CheckOutcomeSummary { - verdict: "accept" | "reject" | "undecidable"; - outcomes: { label: string; status: string; log?: string }[]; -} - -export type CheckRunner = (options: { - repository: string; - files: Readonly>; - revision: string; - checks: readonly CandidateCheck[]; - /** Present when the round can be cancelled: a runner that spawns processes must kill - * their trees on abort rather than leave them running for a round nobody waits for. */ - signal?: AbortSignal; -}) => Promise; - -export type CaseRule = { - /** Host-frozen case name. A no-change conclusion must cite it by this exact name. */ - readonly name: string; - /** Host-frozen title token: a patch proves coverage for this case only by adding a - * test whose title contains it. Stated in the instruction, never inferred. */ - readonly token: string; -}; - -export interface CycleOptions { - repository: string; - revision: string; - /** Round-frozen baseline. Uncommitted OoO files are absent from HEAD, so every - * file a fixed check needs must appear here, not only the editable ones. */ - baseline: Readonly>; - checks: readonly CandidateCheck[]; - worker: CycleWorker; - aInstruction: string; - bInstruction: string; - aEditable: readonly string[]; - bEditable: readonly string[]; - budget: { perFile: number; output: number }; - limits: { - turns: number; - reads: number; - timeoutMs: number; - }; /** Overridable so the deterministic test does not create worktrees. */ - runChecks?: CheckRunner; - /** Operator cancellation. Aborting it stops dispatch, fences the round in the - * coordinator, and kills the process tree of every check still running, so a cancelled - * round leaves no orphan worker or check process behind. */ - signal?: AbortSignal; - /** Cancellation can also arrive as durable state written by *another* process (a CLI - * `cancel` on a round it does not own). The round polls this while it runs, and the - * decision is the same one `signal` expresses, so both share a single path. */ - watchCancellation?: () => string | null; - /** Dispatch order. `ooo` (default) runs the independent task while the waiting task's check - * is outstanding, which is the design under test; `sequential` waits for the check first and - * then runs the fixed plan in order. Same plan, inputs, checks and acceptance rules — only - * the order differs, which is what an S4 comparison needs. */ - mode?: "ooo" | "sequential"; - /** Independently reviewed, task-specific no-change claims, fixed before the round. - * Omission disables no-change acceptance; test names alone are not coverage proof. */ - noChangeCases?: Partial>; - /** Proven-gap mode: the host declares faults the frozen suite does not detect, and - * a patch is accepted only when the candidate suite passes intact and kills at - * least one of them. It replaces the title-token rule for that task, because a - * surviving mutant is stronger evidence than a title match. */ - mutations?: Partial>; - /** Readable subset per task. The whole baseline is re-sent on every model turn, so a - * task sees only what its acceptance rule can point at; omission shows everything. */ - visible?: Partial>; - /** Which conclusion kinds each task's acceptance rule admits. The tool schema is - * derived from this, so a task is never offered an answer the host must reject. */ - admitted?: Partial>; - /** Preconditions each task declares on its dependencies. A failed requirement is a - * dependency rejection: the host reopens the responsible task instead of letting a - * dependent work on an input that cannot satisfy it. */ - requires?: Partial>; - /** Bounded downstream pushback. Exhausting it stops the round with evidence rather - * than repairing a failing version forever. */ - maxReopens?: number; - /** Where the round records what happened. Omission keeps it in memory, which is what - * the deterministic tests use; a live round writes it next to its report so the round - * can be replayed without a model. */ - roundLog?: RoundLog; - databaseDir?: string; -} - -export interface CycleResult { - timeline: { at: string; step: string; detail?: string }[]; - verdicts: Record; - submissions: Record; - accepted: Record; - /** Bounded artifacts of rejected attempts, so a failed round stays diagnosable. */ - rejections: { task: string; attempt: number; artifact: string }[]; - /** Declared mutants the accepted candidate suite detected / still does not detect. */ - killed: Record; - survived: Record; - /** Cost and latency evidence for the round: what the host spent, what the worker - * spent, and how much of an unresolved wait the out-of-order task actually hid. */ - measurements: { - workers: Record; - hiddenWaitMs: number; - checkMs: number; - hostChecks: number; - hostMs: number; - reopens: string[]; - }; - composed: { verdict: string; files: string[] }; - /** The round's own record, in order. Replaying it needs no model call. */ - log: readonly RoundEvent[]; - /** Set when the operator cancelled the round: the reason, as the coordinator stored it. */ - cancelled?: string; -} - -const plan: ProbePlan = [ - ["A", "", [], "isolated-artifact", "protocol-regression", null], - ["B", "", [], "isolated-artifact", null, null], - ["C", "", ["A", "B"], "isolated-artifact", null, null], -]; - -/** Host driver for one A/B/C round: the check is real and concurrent with B, the - * bypass is the coordinator's decision, and every acceptance is host-verified. */ -/** The round's check verdict as the terminal evidence the board records. */ -/** Round inputs with their documented defaults applied once, so the orchestrator reads - * resolved values instead of repeating `??` at every use. */ -function cycleDefaults(options: CycleOptions) { - return { - runChecks: (options.runChecks ?? verifyCandidate) as CheckRunner, - noChangeCases: structuredClone(options.noChangeCases ?? {}), - declared: structuredClone(options.mutations ?? {}), - maxReopens: options.maxReopens ?? 1, - }; -} - -function checkOutcome( - verdict: CheckOutcomeSummary["verdict"], -): "passed" | "failed" | "undecidable" { - if (verdict === "accept") return "passed"; - return verdict === "reject" ? "failed" : "undecidable"; -} - -/** The check's own duration: issued-to-terminal, from the recorded timestamps. */ -function checkWindow(timeline: CycleResult["timeline"]): number { - const at = (step: string) => Date.parse(timeline.find((e) => e.step === step)?.at ?? "0"); - return at("check-finished") - at("check-issued") || 0; -} - -/** One line per check, in order: the evidence the coordinator and the worker both see. */ -function describeOutcomes(result: CheckOutcomeSummary): string { - return result.outcomes.map((item) => `${item.label}=${item.status}`).join(", "); -} - -/** Every check in the round must have reported a pass for a no-change claim to stand. */ -function allChecksPassed(result: CheckOutcomeSummary, checks: readonly CandidateCheck[]): boolean { - return checks.every(({ label }) => - result.outcomes.some((item) => item.label === label && item.status === "passed"), - ); -} - -/** The driver and the coordinator must agree on the frozen work, or the round is not - * running the same task. A bare "mismatch" cost two debugging rounds, so the error - * names the fields that differ. */ -function assertFrozenMatches( - frozen: FrozenPatchWork, - claimed: FrozenPatchTask & { digest: string }, -) { - if (frozen.digest === claimed.digest) return; - const own = frozen.work as unknown as Record; - const theirs = claimed as unknown as Record; - const differing = Object.keys(own).filter( - (key) => JSON.stringify(own[key]) !== JSON.stringify(theirs[key]), - ); - const detail = differing - .map((key) => `${key}=${JSON.stringify(own[key])} vs ${JSON.stringify(theirs[key])}`) - .join(" | "); - throw new Error( - `frozen digest mismatch: ${frozen.digest.slice(0, 12)} vs ${claimed.digest.slice(0, 12)}; ` + - `differing fields: ${differing.join(", ") || "none"} ${detail}`, - ); -} - -/** The coordinator stays the acceptance authority; this records why the shared - * contract rejected an artifact, which is otherwise invisible. */ -function logContractError( - log: (step: string, detail?: string) => void, - taskId: string, - frozen: FrozenPatchWork, - artifact: string, -) { - try { - patchSubmission(frozen, artifact); - } catch (error) { - log( - `contract-error:${taskId}`, - (error instanceof Error ? error.message : String(error)).slice(0, 300), - ); - } -} - -export async function runCycle(options: CycleOptions): Promise { - const roundLog = options.roundLog ?? new RoundLog(); - const trace = (event: RoundEventInput) => - roundLog.append({ ...event, at: new Date().toISOString() } as RoundEvent); - const defaults = cycleDefaults(options); - const runChecks = defaults.runChecks; - const directory = options.databaseDir ?? mkdtempSync(join(tmpdir(), "ooo-cycle-db-")); - const specs: Record = {}; - const gate = new BoardAdmission(join(directory, "store.sqlite"), plan, specs); - const timeline: CycleResult["timeline"] = []; - const verdicts: Record = {}; - const submissions: Record = {}; - const rejections: CycleResult["rejections"] = []; - const noChangeCases = defaults.noChangeCases; - const declared = defaults.declared; - const killed: Record = {}; - const survived: Record = {}; - const workers: CycleResult["measurements"]["workers"] = {}; - const reopens: string[] = []; - let lastPushback: { dependency: string; requirement: string; evidence: string } | undefined; - let premiseFailed = false; - /** Set when the operator cancels: the round stops dispatching, fences itself in the - * coordinator, and reports the reason instead of finishing as if it had run. */ - let cancelled: string | null = null; - /** A premise that could not be measured is not a premise that failed: the round must - * stop, but it must say which of the two happened. */ - let premiseUnmeasured = false; - /** Assigned when the round starts; the verifier awaits it so a false premise can - * never accept anything, while the work itself already runs beside the model call. */ - let matrix: Promise = Promise.resolve(); - let hostChecks = 0; - let hostMs = 0; - const hostCheck = async (files: Readonly>) => { - const started = Date.now(); - try { - return await runChecks({ - repository: options.repository, - revision: options.revision, - files, - checks: options.checks, - signal: own.signal, - }); - } finally { - hostChecks += 1; - hostMs += Date.now() - started; - } - }; - /** Coverage is resolved to a named test title, never to the worker's own words. - * This proves a title exists for the case; it does not prove the test asserts it. */ - const titles = (files: Readonly>) => - new Set( - Object.values(files).flatMap((source) => - [...source.matchAll(/test(?:\.\w+)?\(\s*["']([^"'\n]+)["']/g)].map((match) => match[1]), - ), - ); - const frozenTitles = titles(options.baseline); - const caseResolved = ( - rule: CaseRule, - available: ReadonlySet, - cited: ReadonlyMap, - ) => { - const title = cited.get(rule.name); - if (title) return frozenTitles.has(title); - // The token is host-frozen and stated in the instruction; an unstated token is - // not guessed, it fails closed. - const token = rule.token.trim().toLowerCase(); - if (!token) return false; - return [...available].some((candidate) => candidate.toLowerCase().includes(token)); - }; - const log = (step: string, detail?: string) => - timeline.push({ at: new Date().toISOString(), step, detail }); - /** In-flight checks are killed through this controller, which both cancellation channels - * abort: the operator's signal, and the store when another process asked for the stop. */ - const own = new AbortController(); - options.signal?.addEventListener("abort", () => own.abort(), { once: true }); - /** Operator cancellation, recorded once and fenced in the coordinator. - * - * The coordinator is the authority, so the round does not merely stop: it cancels the - * round it is running, which advances every attempt and retires every live claim and - * ticket. That is what makes a late artifact `stale` instead of accepted into a round - * nobody is waiting for. Returns true when the round must stop dispatching. */ - const stopped = () => { - if (cancelled === null) { - const reason = options.signal?.aborted - ? "operator cancelled the round" - : (options.watchCancellation?.() ?? null); - if (reason === null) return false; - gate.cancel(reason); - cancelled = gate.cancelled() ?? reason; - // A cancelled round must not keep a check running: this kills its process tree. - own.abort(); - log("cancelled", reason); - } - return true; - }; - /** A round waiting on a long check still has to notice a cancellation from another - * process, so the store is polled rather than only consulted at dispatch points. */ - const watcher = options.watchCancellation - ? setInterval(() => { - if (stopped()) clearInterval(watcher); - }, 250) - : undefined; - /** How much of the unresolved external wait the out-of-order task's OWN WORK covered. - * Computed from real timestamps: the task's claim-to-return window intersected with the - * check's issued-to-terminal window. Deliberately not claim-to-submission: submission comes - * after the host verified the candidate, and the verification window (the mutation matrix and - * the candidate check) outlasts the check itself, so counting it would report the whole check - * as hidden even when the task's own work lasted a fraction of it. */ - const hiddenWait = () => { - const at = (step: string) => - Date.parse(timeline.find((entry) => entry.step === step)?.at ?? ""); - const claim = timeline.find((entry) => entry.step === "claim:B")?.at; - const returned = timeline.filter((entry) => entry.step === "worker-returned:B").pop()?.at; - const issued = at("check-issued"); - const finished = at("check-finished"); - if (!claim || !returned || isNaN(issued) || isNaN(finished)) return 0; - return Math.max( - 0, - Math.min(Date.parse(returned), finished) - Math.max(Date.parse(claim), issued), - ); - }; - const check = (files: Readonly>) => hostCheck(files); - const record = async (result: CheckOutcomeSummary) => { - const failed = result.outcomes.find((item) => item.status !== "passed"); - const detail = - `${result.verdict}: ${describeOutcomes(result)}` + - (failed?.log ? ` | ${failed.log.slice(-600)}` : ""); - log("candidate-check", detail); - return result.verdict; - }; - /** Case rules resolve against a title the host can actually see; an unresolved case - * fails closed rather than being matched by the worker's own words. */ - const resolveCases = ( - task: string, - cases: readonly CaseRule[], - available: ReadonlySet, - cited: ReadonlyMap, - ): boolean => { - for (const rule of cases) - if (!caseResolved(rule, available, cited)) { - log( - cited.size ? "citation-unresolved" : "case-unresolved", - cited.size - ? `${task} ${rule.name} -> ${cited.get(rule.name) ?? "none"}` - : `${task} ${rule.name} -> no test title matching ${rule.token}`, - ); - return false; - } - return true; - }; - /** The retention rule of mutation-guided testing: a new test is worth keeping only if - * it passes on the intact implementation and fails on a declared mutant. */ - const runMutationProof = async ( - task: string, - candidate: Readonly>, - mutants: readonly Mutation[], - ) => { - for (const mutation of mutants) { - const run = await check(mutate(candidate, mutation)); - const killedIt = run.verdict === "reject"; - (killedIt ? (killed[task] ??= []) : (survived[task] ??= [])).push(mutation.id); - log(killedIt ? "mutant-killed" : "mutant-survived", `${task} ${mutation.id}`); - trace({ - kind: "mutant", - taskId: task, - id: mutation.id, - outcome: killedIt ? "killed" : "survived", - }); - } - }; - /** A patch candidate: the premise must hold, the declared cases must resolve, the - * candidate must pass intact, and it must close at least one declared gap. */ - const verifyPatch = async ( - task: "A" | "B", - submission: Extract, - ) => { - const mutants = declared[task] ?? []; - // The mutant premise gates acceptance, not the start of work: awaiting the - // concurrently proved matrix here keeps a false premise from accepting anything. - await matrix; - if (premiseFailed || premiseUnmeasured) { - log( - premiseFailed ? "premise-invalid" : "premise-unmeasured", - `${task}: ${premiseFailed ? "a declared mutant is already detected" : "the declared faults could not be measured"}`, - ); - return "reject"; - } - const candidate = { ...options.baseline, ...submission.files }; - // Proven-gap mode replaces the title-token rule for patches, rather than stacking - // with it: a surviving mutant is strictly stronger evidence than a title match, and - // a round that declares both would reject a candidate for the weaker rule. - const cases = mutants.length ? [] : (noChangeCases[task] ?? []); - if (!resolveCases(task, cases, titles(candidate), new Map())) return "reject"; - const intact = await check(candidate); - if (intact.verdict !== "accept") return record(intact); - await runMutationProof(task, candidate, mutants); - if (mutants.length && !killed[task]?.length) { - log("gap-not-closed", `${task}: no declared mutant killed`); - return "reject"; - } - return record(intact); - }; - /** The refusals a conclusion can hit before any check runs, as a verdict or null. */ - const conclusionRefusal = ( - task: "A" | "B", - submission: Extract, - mutants: readonly Mutation[], - ): "reject" | null => { - // A blocked report is not completed work: it must not unlock A or C. - if (submission.conclusion === "cannot-complete") { - log("blocked", `${task}: ${submission.summary.slice(0, 300)}`); - return "reject"; - } - if (submission.conclusion !== "no-change-needed") return "reject"; - // When the host has proved a surviving mutant, "no change needed" is false by the - // host's own evidence, so it cannot be accepted on any citation. - if (mutants.length) { - log("gap-proven", `${task}: ${mutants.length} declared mutant(s) survived the frozen suite`); - return "reject"; - } - if (!noChangeCases[task]?.length) return "reject"; - return null; - }; - /** No-change claims are only as good as a citation the host can resolve: an invented, - * misquoted, or missing citation fails closed. */ - const conclusionCasesResolved = (task: "A" | "B", submission: PatchSubmission) => { - const cases = noChangeCases[task] ?? []; - const cited = new Map( - submission.kind === "conclusion" ? submission.citations.map((i) => [i.case, i.test]) : [], - ); - const unknown = [...cited.keys()].find((name) => !cases.some((rule) => rule.name === name)); - if (unknown) { - log("citation-unresolved", `${task} unknown case ${unknown}`); - return false; - } - return resolveCases(task, cases, frozenTitles, cited); - }; - const verifyConclusion = async ( - task: "A" | "B", - submission: Extract, - ) => { - const refusal = conclusionRefusal(task, submission, declared[task] ?? []); - if (refusal) return refusal; - if (!conclusionCasesResolved(task, submission)) return "reject"; - const result = await check(options.baseline); - if (!options.checks.length || !allChecksPassed(result, options.checks)) return "undecidable"; - return record(result); - }; - const patchVerifier = (task: "A" | "B") => async (submission: PatchSubmission) => - submission.kind === "patch" - ? verifyPatch(task, submission) - : verifyConclusion(task, submission); - const composedVerifier = - (abFiles: Readonly>) => async (submission: PatchSubmission) => { - // C may only promote what the host itself can re-verify as a composed candidate. - if (submission.kind !== "conclusion" || submission.conclusion !== "promote-candidate") - return "reject"; - return record(await check({ ...options.baseline, ...abFiles })); - }; - - /** Host evaluation of a declared precondition. Only facts the coordinator has - * recorded are allowed, so a rejected dependency is never a matter of opinion. */ - const describe = (requirement: Requirement) => - requirement.kind === "verified" - ? `${requirement.task} verified` - : requirement.kind === "mutant-killed" - ? `${requirement.task} kills ${requirement.id}` - : `${requirement.task} adds a test titled *${requirement.token}*`; - const requirementMet = (requirement: Requirement) => { - const submission = submissions[requirement.task]; - if (requirement.kind === "verified") return submission !== undefined; - if (!submission) return false; - if (requirement.kind === "mutant-killed") - return (killed[requirement.task] ?? []).includes(requirement.id); - if (submission.kind !== "patch") return false; - const token = requirement.token.trim().toLowerCase(); - if (!token) return false; - return [...titles(submission.files)].some((title) => title.toLowerCase().includes(token)); - }; - const firstUnmet = (task: "A" | "B" | "C") => - (options.requires?.[task] ?? []).find((requirement) => !requirementMet(requirement)); - - /** A task whose instruction says "the host proved these faults survive" depends on that - * proof: if the premise is false the task as stated does not exist, and dispatching it - * anyway spends a full model call on work the host then refuses on its own evidence - * (measured: 204k tokens). The proof is seconds of host time, so it is awaited before the - * dispatch instead of being raced against the model. Returns the refusal reason, or null - * when the task may run. */ - const premiseRefusal = async (taskId: string): Promise => { - if (!(declared[taskId as "A" | "B"] ?? []).length) return null; - await matrix; - if (!premiseFailed && !premiseUnmeasured) return null; - const reason = premiseFailed - ? `${taskId} declares a fault the frozen suite already detects` - : `${taskId}'s declared fault could not be measured`; - // Nothing is claimed yet, but something *was* published: the round announced this task's - // handoff, and the board serializes actionable entries, so leaving it outstanding would - // block every later claim in the round. - gate.withdrawHandoff(taskId, reason); - log(premiseFailed ? "premise-invalid" : "premise-unmeasured", `${taskId}: not dispatched`); - trace({ - kind: "worker-failed", - taskId, - attempt: 0, - reason: `premise-invalid: ${reason}`, - }); - rejections.push({ task: taskId, attempt: 0, artifact: `${taskId} not dispatched: ${reason}` }); - verdicts[taskId] = "rejected"; - return reason; - }; - - /** One worker invocation only. A rejected claim is not artificially expired. */ - const runTask = async (taskId: string) => { - if (stopped()) { - verdicts[taskId] ??= "cancelled"; - return undefined; - } - // Selection belongs to the coordinator: an unreleased dependency or a live - // claim on another task means this task simply does not run in this round. - const selected = gate.next(); - if (selected !== taskId) { - log(`skip:${taskId}`, `selected=${selected ?? "none"}`); - return undefined; - } - if ((await premiseRefusal(taskId)) !== null) return "rejected"; - const ticket = gate.claim(taskId, `worker-${taskId}`); - const claimedAt = Date.now(); - if (!ticket.patch) throw new Error("round tasks must be patch tasks"); - // Rebuilt from the ticket, which carries the frozen work: the driver can then only - // disagree with the coordinator by constructing a different digest. - const frozen = preparePatchWork({ - taskId: ticket.patch.taskId, - attempt: ticket.attempt, - instruction: ticket.patch.instruction, - files: ticket.patch.files, - editable: ticket.patch.editable, - visible: ticket.patch.visible, - admittedConclusions: ticket.patch.admittedConclusions, - budget: ticket.patch.budget, - limits: ticket.patch.limits, - }); - assertFrozenMatches(frozen, ticket.patch); - log(`claim:${taskId}`, `attempt=${ticket.attempt} digest=${frozen.digest.slice(0, 12)}`); - trace({ - kind: "claim", - taskId, - attempt: ticket.attempt, - digest: frozen.digest, - owner: ticket.owner, - }); - // A worker that fails or returns truncated output produced no artifact: it is a - // failed attempt with recorded reason, not a crashed round and not a retry. - const produced = await callWorker(taskId, frozen, ticket.dependencies); - // The task's own work ends here. What follows (its verification) is host time, and the - // sequential arm pays it too, so it must not be counted as wait the task covered. - log(`worker-returned:${taskId}`, `attempt=${ticket.attempt}`); - if (produced.metrics) workers[taskId] = { ...produced.metrics, ms: Date.now() - claimedAt }; - if (produced.failure) { - log(`worker-failed:${taskId}`, produced.failure.slice(0, 400)); - trace({ - kind: "worker-failed", - taskId, - attempt: ticket.attempt, - reason: produced.failure.slice(0, 2_000), - }); - rejections.push({ - task: taskId, - attempt: ticket.attempt, - artifact: produced.failure.slice(0, 2_000), - }); - verdicts[taskId] = "rejected"; - return "rejected"; - } - const artifact = produced.artifact!; - trace({ - kind: "artifact", - taskId, - attempt: ticket.attempt, - artifact: artifact.slice(0, 256_000), - metrics: produced.metrics, - }); - if (produced.pushback) { - lastPushback = produced.pushback; - log( - `pushback:${taskId}`, - `${produced.pushback.dependency} ${produced.pushback.requirement}: ${produced.pushback.evidence.slice(0, 200)}`, - ); - trace({ - kind: "pushback", - taskId, - dependency: produced.pushback.dependency, - requirement: produced.pushback.requirement, - evidence: produced.pushback.evidence.slice(0, 2_000), - }); - verdicts[taskId] = "blocked-by-dependency"; - return "blocked-by-dependency"; - } - logContractError(log, taskId, frozen, artifact); - const entry = gate.putTaskBoardEntry({ - taskId: "ooo-process-probe", - agentId: ticket.owner, - kind: "result", - content: JSON.stringify({ ticket, artifact }), - expiresAt: new Date(gate.now + 86_400_000).toISOString(), - }); - const verdict = await gate.submit(entry.id); - log(`submit:${taskId}`, verdict); - trace({ kind: "verdict", taskId, attempt: ticket.attempt, verdict }); - verdicts[taskId] = verdict; - const accepted = verdict === "accepted" ? gate.accepted()[taskId] : undefined; - if (accepted) submissions[taskId] = JSON.parse(accepted) as PatchSubmission; - else - rejections.push({ - task: taskId, - attempt: ticket.attempt, - artifact: artifact.slice(0, 2_000), - }); - return verdict; - }; - - /** One worker invocation, with the failure of a worker call turned into a recorded - * attempt rather than a crashed round. */ - const callWorker = async ( - taskId: string, - frozen: FrozenPatchWork, - dependencies: Record, - ): Promise<{ - artifact?: string; - pushback?: { dependency: string; requirement: string; evidence: string }; - metrics?: WorkerMetrics; - failure?: string; - }> => { - try { - const produced = await options.worker(taskId, frozen, dependencies); - return typeof produced === "string" ? { artifact: produced } : produced; - } catch (error) { - return { failure: error instanceof Error ? error.message : String(error) }; - } - }; - - /** Installs a task's host spec. Kept in one place because a reopened task must be - * reinstallable with the consumer's evidence, and its digest must cover that text. */ - const casesTail = (task: "A" | "B") => - "\nHost-proven faults the frozen suite does not detect (each is an exact code change): " + - JSON.stringify( - (declared[task] ?? []).map(({ id, path, from, to }) => ({ id, path, from, to })), - ) + - "\nHost-frozen cases (case name and required title token): " + - JSON.stringify(noChangeCases[task] ?? []); - const installB = (tail: string) => { - specs.B = { - instruction: options.bInstruction + tail + casesTail("B"), - files: options.baseline, - editable: options.bEditable, - budget: options.budget, - limits: options.limits, - visible: options.visible?.B, - admittedConclusions: options.admitted?.B, - verify: patchVerifier("B"), - }; - }; - const installA = (tail: string) => { - specs.A = { - instruction: options.aInstruction + tail + casesTail("A"), - files: options.baseline, - editable: options.aEditable, - budget: options.budget, - limits: options.limits, - visible: options.visible?.A, - admittedConclusions: options.admitted?.A, - verify: patchVerifier("A"), - }; - }; - - /** A round that is already cancelled never starts: the explicit terminal state is the whole - * point of cancellation, and a round that threw `round cancelled` from its first coordinator - * call would leave the operator with an exception instead of a decision. */ - const cancelledResult = (): CycleResult => { - const verdicts = { A: "cancelled", B: "cancelled", C: "cancelled" }; - const result: CycleResult = { - timeline, - verdicts, - submissions: {}, - accepted: {}, - rejections: [], - killed: {}, - survived: {}, - measurements: { - workers: {}, - hiddenWaitMs: 0, - checkMs: 0, - hostChecks: 0, - hostMs: 0, - reopens: [], - }, - composed: { verdict: "undecidable", files: [] }, - log: [], - cancelled: cancelled ?? "cancelled before dispatch", - }; - trace({ - kind: "plan", - tasks: ["A", "B", "C"], - checks: options.checks.map((check) => check.label), - revision: options.revision, - checkDigest: checksDigest(options.checks), - mode: options.mode ?? "ooo", - }); - trace({ - kind: "terminal", - accepted: {}, - verdicts, - composed: { verdict: "undecidable", files: [] }, - cancelled: result.cancelled!, - }); - return { ...result, log: roundLog.recorded() }; - }; - - /** The ordered phase: the waiting task's check is issued first, the independent task may run - * while it is outstanding (`ooo`), and the fixed plan is then dispatched in order. Extracted - * because both dispatch orders live here, and one function should not carry both the ordering - * decision and every guard around it. */ - /** The verdicts of the two independent tasks: C's frozen envelope states them, and a reopen - * reassigns them, so they outlive the phase that produced them. */ - let a: string | undefined; - let b: string | undefined; - - const dispatchPhase = async () => { - installB(""); - const ticket = gate.issueCheck("A", "protocol-check-host"); - log("check-issued", ticket.checkId); - trace({ kind: "check-issued", taskId: "A", ticket }); - const background = check(options.baseline).then((result) => { - log("check-finished", result.verdict); - return result; - }); - // Sequential mode is the control: the check is awaited before any task is dispatched, so - // nothing overlaps the wait. The plan and every acceptance rule stay identical. - const outcome = (options.mode ?? "ooo") === "sequential" ? await background : null; - // In sequential mode nothing is dispatched before the check reports; otherwise B runs now, - // which is the out-of-order decision this design is about. - if (outcome) log("sequential-wait", "the check was awaited before any dispatch"); - else b = await runTask("B"); - const settled = outcome ?? (await background); - gate.submitCheck({ - ticket, - outcome: checkOutcome(settled.verdict), - log: describeOutcomes(settled).slice(0, 4_000), - }); - log("check-terminal", settled.verdict); - trace({ - kind: "check-result", - taskId: "A", - verdict: settled.verdict, - outcomes: settled.outcomes.map((item) => ({ label: item.label, status: item.status })), - }); - log("dispatch-after-bypass", gate.next() ?? "none"); - - installA( - `\n\nCheck ${ticket.checkId} finished with ${settled.verdict} ` + - `(${describeOutcomes(settled)}). ` + - "Fix a defect this check exposed, or return a no-change conclusion citing one listed case.", - ); - a = await runTask("A"); - b ??= await runTask("B"); - }; - - if (stopped()) { - // The gate is this function's connection to the round's store: the early path has to close - // it too, or a cancelled round leaves the database locked behind it. - const early = cancelledResult(); - gate.close(); - return early; - } - - try { - trace({ - kind: "plan", - tasks: ["A", "B", "C"], - checks: options.checks.map((check) => check.label), - revision: options.revision, - checkDigest: checksDigest(options.checks), - mode: options.mode ?? "ooo", - }); - // Class A/D: proving the declared mutants survive is required work that does not - // depend on B's model call, so it runs concurrently with it instead of before the - // round. The verifier awaits it, so a false premise still fails closed. - // - // Three outcomes, not two: an `undecidable` check measured nothing, and folding it into - // "already detected" turns a repository hiccup into a false premise — which is how a - // round once reported a surviving mutant as killed 76 ms after the previous one. - matrix = (async () => { - for (const [task, mutants] of Object.entries(declared)) - for (const mutation of mutants ?? []) { - const run = await check(mutate(options.baseline, mutation)); - if (run.verdict === "accept") { - log("mutant-survived-baseline", `${task} ${mutation.id}`); - trace({ kind: "mutant", taskId: task, id: mutation.id, outcome: "survived" }); - continue; - } - if (run.verdict === "reject") { - log("precondition-failed", `${task} ${mutation.id} is already detected`); - trace({ kind: "mutant", taskId: task, id: mutation.id, outcome: "killed" }); - premiseFailed = true; - continue; - } - log("premise-unmeasured", `${task} ${mutation.id} could not be measured`); - trace({ kind: "mutant", taskId: task, id: mutation.id, outcome: "unmeasured" }); - premiseUnmeasured = true; - } - })(); - // B has no dependency on A, so it runs while A waits on the real check. - // The declared gaps are part of the frozen instruction, so the digest binds them. - await dispatchPhase(); - - const changedFiles = () => { - const files: Record = {}; - // Only files the accepted work actually changed, so the composed candidate names - // what the round produced instead of dumping every frozen file. - for (const submission of [submissions.A, submissions.B]) - if (submission?.kind === "patch") - for (const [path, content] of Object.entries(submission.files)) - if (options.baseline[path] !== content) files[path] = content; - return files; - }; - const compose = async (): Promise<{ - files: Record; - result: CheckOutcomeSummary; - }> => { - const files = changedFiles(); - // A cancelled round does not keep verifying: the operator decided the round's work is - // not wanted, and its composition must not be promoted on a check nobody asked for. - if (stopped()) - return { - files: {}, - result: { - verdict: "undecidable" as const, - outcomes: [{ label: "cancelled", status: "undecidable" as const }], - }, - }; - const result = await check({ ...options.baseline, ...files }); - log("composed-check", result.verdict); - return { files, result }; - }; - let { files: abFiles, result: composed } = await compose(); - /** The round's report, assembled once the phase above has produced its outcome. */ - const buildResult = (): CycleResult => ({ - ...(cancelled ? { cancelled } : {}), - timeline, - verdicts, - submissions, - rejections, - killed, - survived, - measurements: { - workers, - hiddenWaitMs: hiddenWait(), - checkMs: checkWindow(timeline), - hostChecks, - hostMs, - reopens, - }, - accepted: gate.accepted(), - composed: { verdict: composed.verdict, files: Object.keys(abFiles) }, - log: roundLog.recorded(), - }); - - /** C's frozen envelope names the composed candidate it may promote, so it is - * installed immediately before every attempt; a reopen changes both. */ - const installC = () => { - specs.C = { - instruction: - "Act as the composition task for this round. A and B already ran; the host has its own composed " + - "check result. Your submission is accepted only as a promote-candidate conclusion: the host " + - "rejects any other kind, because the composed candidate is the round's product and nothing " + - "else can carry it. Return it only when the composed check passed and nothing in the " + - "submissions weakens verification; if it did not pass, say so instead and explain what is " + - "wrong, which is recorded as a blocked report rather than an accepted composition.\n" + - `A=${a}; B=${b}; composed check=${composed.verdict} ` + - `(${describeOutcomes(composed)}).`, - files: { ...options.baseline, ...abFiles }, - editable: options.aEditable, - budget: options.budget, - limits: options.limits, - visible: options.visible?.C, - admittedConclusions: options.admitted?.C, - verify: composedVerifier(abFiles), - }; - }; - - /* Bounded downstream pushback: a dependent's declared precondition that the host - * can prove unmet reopens the responsible task with fresh input and fresh check - * evidence, which fences everything bound to the invalidated value. Exhausting the - * budget stops the round with evidence instead of repairing in place forever. */ - const maxReopens = defaults.maxReopens; - let blocked: string | null = null; - const reopenDependency = async (task: "A" | "B", requirement: string, tail: string) => { - if (reopens.length >= maxReopens) { - log("reopen-exhausted", `${task}: ${requirement}`); - blocked = `${task}: ${requirement}`; - return false; - } - const invalidated = gate.reopen(task, requirement); - reopens.push(task); - trace({ kind: "reopen", taskId: task, requirement, invalidated }); - log("reopen", `${task} invalidated=${invalidated.join(",") || "none"}`); - if (task === "A") { - // Fresh evidence bound to the new input: a reopened task must not consume the - // check result that justified the artifact just invalidated. - const fresh = gate.issueCheck("A", "protocol-check-host"); - log("check-issued", fresh.checkId); - trace({ kind: "check-issued", taskId: "A", ticket: fresh }); - const rerun = await check(options.baseline); - gate.submitCheck({ - ticket: fresh, - outcome: checkOutcome(rerun.verdict), - log: describeOutcomes(rerun).slice(0, 4_000), - }); - log("check-terminal", rerun.verdict); - trace({ - kind: "check-result", - taskId: "A", - verdict: rerun.verdict, - outcomes: rerun.outcomes.map((item) => ({ label: item.label, status: item.status })), - }); - installA( - tail + - ` - -Check ${fresh.checkId} finished with ${rerun.verdict} ` + - `(${describeOutcomes(rerun)}).`, - ); - await runTask("A"); - } else { - installB(tail); - await runTask("B"); - } - ({ files: abFiles, result: composed } = await compose()); - return true; - }; - /** Host-provable preconditions first: a requirement the host can evaluate is rejected - * before the dependent spends a worker call on an input it cannot use. True means the - * dependency was reopened and the loop must run again. */ - const handleUnmetPrecondition = async (): Promise => { - const unmet = firstUnmet("C"); - if (!unmet) return false; - const note = `C requires ${describe(unmet)}`; - log("dependency-rejected", note); - return await reopenDependency( - unmet.task as "A" | "B", - note, - ` - -A downstream task rejected your accepted artifact: it requires ${describe(unmet)}. ` + - "Produce a version whose evidence satisfies that requirement, or state why it cannot.", - ); - }; - /** The dependent discovered mid-attempt that its input is unusable. That report is - * evidence about its own execution, not a judgement about the dependency: the host - * still decides, and reopening is what makes the dependency run again. */ - const handlePushback = async (): Promise => { - const pushed = lastPushback!; - return await reopenDependency( - pushed.dependency as "A" | "B", - `C cannot satisfy ${pushed.requirement} from ${pushed.dependency}: ${pushed.evidence.slice(0, 300)}`, - ` - -The downstream task reported that your artifact cannot satisfy ${pushed.requirement}. ` + - `Its evidence: ${pushed.evidence.slice(0, 500)}`, - ); - }; - for (;;) { - if (await handleUnmetPrecondition()) continue; - installC(); - const c = await runTask("C"); - if (c !== "blocked-by-dependency") break; - if (await handlePushback()) continue; - break; - } - - if (blocked) verdicts.C = "blocked"; - const result = buildResult(); - roundLog.append({ - kind: "terminal", - at: new Date().toISOString(), - accepted: result.accepted, - verdicts: result.verdicts, - composed: { verdict: result.composed.verdict, files: [...result.composed.files] }, - ...(result.cancelled ? { cancelled: result.cancelled } : {}), - }); - return result; - } finally { - if (watcher) clearInterval(watcher); - gate.close(); - if (!options.databaseDir) rmSync(directory, { recursive: true, force: true }); - } -} diff --git a/src/integration/ooo-execution.ts b/src/integration/ooo-execution.ts index 5f496ea9..ffbb573e 100644 --- a/src/integration/ooo-execution.ts +++ b/src/integration/ooo-execution.ts @@ -71,34 +71,343 @@ export interface DispatchTask { claimed: boolean; externalEvent?: string; externalReady: boolean; + /** Bytes exist for this task. Delivery and acceptance are different facts: a delivered artifact + * whose verdict is not accepted (pending, rejected, or about another digest) leaves a task that + * cannot be claimed again and must not be selected - the coordinator recovers it with reopen(). */ + delivered?: boolean; + /** The run recorded a cancellation for this task. A cancellation is a fact about the task, so it + * gates dispatch the way a rejection gates a dependent: a cancelled task is not handed out, and + * nothing may read one as a closed input. Acceptance already refuses a cancelled task; the gap + * this closes is that the *eligibility* rule could not see it at all, so a cancelled task stayed + * selectable as the next dispatch. */ + cancelled?: boolean; } /** Input order and declarations belong to the coordinator, never the worker. * This checks eligibility, not whether arbitrary worker code is actually safe. */ -export function nextTask(plan: readonly DispatchTask[]): string | null { +/** + * The in-flight claim budget `slots` is declared by the run, and a plan may not be read with half a + * slot or none: a zero or fractional count is not a smaller budget, it is an unusable one, and + * rounding it silently would hide the caller's mistake. + */ +export function checkedSlots(slots: number): number { + if (!Number.isSafeInteger(slots) || slots < 1) + throw new Error("slots must be a positive integer"); + return slots; +} + +/** + * The candidates the shared rules make selectable, in the rule policy's order; `nextTask` returns its + * head and `startableTasks` is the part of it this run may start right now. + * + * Legality lives in the rules below and nowhere else: an ordering step may rank this set, and nothing + * may widen it. Two of the rules are legality conditions rather than preferences, and an ordering must + * not skip them: a task blocked by a stale input or an undeclared dependency is not selectable, and + * that block is not an external wait license. An earlier neighbour that *declares* an external wait is + * different: the wait is what the plan licenses, so the tasks after it stay selectable. + * + * `slots` bounds how many claims one run may hold at once, so a run that has spent its budget gets an + * empty set. A claimed task is out of the set either way. What keeps a dependent from starting early is + * its own dependency, which is still unaccepted while the claim is in flight - not the claim count. + * The count is over tasks that are still pending (see `pending` below): a delivered task nothing may + * claim again is not in the plan to be counted, and `openRound`/`reopen` owns recovering it. + */ +export function selectableTasks(plan: readonly DispatchTask[], slots = 1): readonly string[] { + return selection(plan, slots).legal; +} + +/** Claims one run already holds, over the same pending set the budget is spent on. A claim is in + * flight until its task is accepted, which is what takes the task out of `pending`. */ +function claimedInFlight(pending: readonly DispatchTask[]): number { + return pending.filter((task) => task.claimed).length; +} + +/** + * The legal set cut to the run's remaining claim budget: this is the part a caller may actually start, + * so it is the only part a claim licence may name. It is narrower than the legal set while earlier + * claims are in flight, and equal to it when the set is small enough to fit. + * + * The cut is applied after ordering (the caller orders `selectableTasks`), not to it: a budget that cut + * the candidate set first would let the rule's own order choose the pool a source is allowed to rank. + */ +export function startableTasks(plan: readonly DispatchTask[], slots = 1): readonly string[] { + const { legal, room } = selection(plan, slots); + return legal.slice(0, room); +} + +/** + * How many claims a run with this budget may still start, over the same pending set the rule counts. + * + * A caller that has its own order - an ordering step's answer, for instance - cuts that order with this + * number instead of re-deriving which tasks are pending, so the budget cannot come to mean two things. + */ +export function remainingSlots(plan: readonly DispatchTask[], slots = 1): number { + return selection(plan, slots).room; +} + +export function nextTask(plan: readonly DispatchTask[], slots = 1): string | null { + return selectableTasks(plan, slots)[0] ?? null; +} + +/** The plan indexed by id, refusing a duplicate: the one reading every rule in this module shares, + * so the legality of a candidate cannot come to mean two things. */ +export function taskIndex(plan: readonly DispatchTask[]): Map { const byId = new Map(plan.map((task) => [task.id, task])); if (byId.size !== plan.length) throw new Error("duplicate task"); - const current = (task: DispatchTask) => - !!task.sourceVersion && task.sourceVersion === task.observedVersion; - const valid = (id: string, visiting = new Set()): boolean => { - const task = byId.get(id); - if (!task || !task.accepted || !current(task) || visiting.has(id)) return false; - const path = new Set(visiting).add(id); - return task.dependencies.every((dependency) => valid(dependency, path)); - }; + return byId; +} + +/** Whether a task's own inputs are the ones the plan declared: a task read from a stale input is not a + * candidate, and nothing may read it as one. */ +function current(task: DispatchTask): boolean { + return !!task.sourceVersion && task.sourceVersion === task.observedVersion; +} + +/** Whether a task counts as an *accepted* dependency: accepted, not cancelled, its own inputs + * current, and the same for what it depends on. Acceptance here is the whole fact - a delivered + * artifact whose verdict is not in is not a satisfied dependency, which is what keeps an unverified + * answer from scheduling anything (see `sharedSessionLegal`). */ +function acceptedDependency( + byId: ReadonlyMap, + id: string, + visiting = new Set(), +): boolean { + const task = byId.get(id); + if (!task || !task.accepted || task.cancelled || !current(task) || visiting.has(id)) return false; + const path = new Set(visiting).add(id); + return task.dependencies.every((dependency) => acceptedDependency(byId, dependency, path)); +} + +/** The one implementation both readings share, so the budget cannot come to mean two things: `legal` is + * the ordered candidate set, and `room` is how much of it the run's remaining budget pays for. */ +function selection( + plan: readonly DispatchTask[], + slots: number, +): { legal: readonly string[]; room: number } { + checkedSlots(slots); + const byId = taskIndex(plan); + const valid = (id: string, visiting = new Set()) => + acceptedDependency(byId, id, visiting); const waiting = (task: DispatchTask) => !!task.externalEvent && !task.externalReady; const ready = (task: DispatchTask) => current(task) && + !task.cancelled && !waiting(task) && ["read-only", "isolated-artifact"].includes(task.effect) && task.dependencies.every((id) => valid(id)); - const pending = plan.filter((task) => !valid(task.id)); + // A task holding bytes nobody can claim is not a task to select, and it is not a reason to + // select nothing either: it is dropped from the plan, which is what the coordinator's reopen() + // is for. Selecting it would publish a handoff no reader could take. + const selectable = plan.filter((task) => task.accepted || !task.delivered); + const pending = selectable.filter((task) => !valid(task.id)); + const room = slots - claimedInFlight(pending); + const none = { legal: [], room: 0 } as const; // ponytail: scan the bounded experiment plan; no learned priorities or preemption. - if (pending.some((task) => task.claimed) || pending.filter(waiting).length > 1) return null; + if (room < 1 || pending.filter(waiting).length > 1) return none; const first = pending[0]; - if (!first) return null; - if (ready(first)) return first.id; + if (!first) return none; + const ids = (tasks: readonly DispatchTask[]) => + tasks.filter((task) => !task.claimed && ready(task)).map((task) => task.id); + if (ready(first)) return { legal: ids(pending), room }; // A stale/missing input or undeclared dependency is not an external wait license. - if (!current(first) || !waiting(first)) return null; - return pending.slice(1).find(ready)?.id ?? null; + if (!current(first) || !waiting(first)) return none; + return { legal: ids(pending.slice(1)), room }; +} + +/** What fusion legality needs and a `DispatchTask` does not carry: which executor may run a unit and + * which authority it acts under. `visible` is the unit's own declaration, never a pair's union - + * sharing a session may not widen what a unit can read. */ +export interface SessionDeclaration { + capability: string; + authority: string; + visible: readonly string[]; +} + +/** The runtime facts fusion reads: the same task view the selection rules use, each unit's session + * declaration, and the facts a speculation branch is still pending on. */ +export interface SessionPlan { + tasks: readonly DispatchTask[]; + declarations: Readonly>; + pendingBranches?: readonly string[]; +} + +function subset(inner: readonly string[], outer: readonly string[]): boolean { + return inner.every((name) => outer.includes(name)); +} + +/** Condition 1: capability, authority and data visibility are compatible, and reuse never widens what a + * unit may read. */ +function compatibleDeclarations(first: SessionDeclaration, next: SessionDeclaration): boolean { + return ( + first.capability === next.capability && + first.authority === next.authority && + subset(next.visible, first.visible) + ); +} + +/** Condition 3: a cancelled unit is neither executed nor carried as a session's next unit. */ +function neitherCancelled(first: DispatchTask, next: DispatchTask): boolean { + return !first.cancelled && !next.cancelled; +} + +/** Condition 5: the history is never reused across a fact whose branch is still pending - a rejected + * proposal does not make the model forget it, so the answer there is a new session. */ +function acrossAPendingBranch(before: string, after: string, pending: readonly string[]): boolean { + return pending.includes(before) || pending.includes(after); +} + +/** + * Whether `after` may continue `before`'s session - execution fusion, which reuses the execution + * resource and keeps every logical task: each unit still takes its own ticket, delivers its own + * artifact and crosses the host boundary on its own. + * + * The five conditions are the design's, and each is one line below so a violation has one name: + * + * 1. capability, authority and data visibility are compatible, and reuse never widens a read scope. + * 2. the successor's dependencies are accepted, and so is the unit that just ran: an unverified answer + * from the same Agent is not a satisfied dependency, it is the reason the session must end here. + * 3. a cancelled unit is neither executed nor carried as a session's next unit - identity, deadline, + * cancellation, verdict and cost stay per-unit facts, and this predicate only refuses to move. + * 4. a declared external wait that is not ready is the host yield boundary: the session ends there so + * the accepted prefix survives and the rest is rescheduled. + * 5. the history is never reused across a fact whose branch is still pending: a rejected proposal does + * not make the model forget it, so condition 5's answer is a new session, not a cleared one. + * + * Legality lives here and nowhere else; a ranking step orders the legal pairs and nothing widens them. + */ +export function sharedSessionLegal(before: string, after: string, plan: SessionPlan): boolean { + const byId = taskIndex(plan.tasks); + const first = byId.get(before); + const next = byId.get(after); + const firstDeclaration = plan.declarations[before]; + const nextDeclaration = plan.declarations[after]; + if (!first || !next || !firstDeclaration || !nextDeclaration) return false; + if (before === after) return false; + if (!compatibleDeclarations(firstDeclaration, nextDeclaration)) return false; + if (!first.accepted) return false; + if (next.dependencies.some((id) => !acceptedDependency(byId, id))) return false; + if (!neitherCancelled(first, next)) return false; + if (!!first.externalEvent && !first.externalReady) return false; + if (acrossAPendingBranch(before, after, plan.pendingBranches ?? [])) return false; + return true; +} + +/** The units that may continue one unit's session, in plan order. A chain is built one legal pair at a + * time, so "short ready chains, not a greedy swallow of the DAG" is a bound the caller sets - it is a + * policy, not a rule that would belong here. */ +export function fusionSuccessors(before: string, plan: SessionPlan): readonly string[] { + return plan.tasks + .map((task) => task.id) + .filter((after) => sharedSessionLegal(before, after, plan)); +} + +/** Every legal pair, in plan order: the candidate set a ranking policy chooses from. */ +export function fusionCandidates(plan: SessionPlan): readonly (readonly [string, string])[] { + return plan.tasks.flatMap((before) => + fusionSuccessors(before.id, plan).map((after) => [before.id, after] as const), + ); +} + +/** A guess about one declared, finite-valued fact: which predicate is being guessed, which version of it, + * and the value the candidate was prepared for. This is the design's + * `assumptions=[{predicateId, version, expected}]`, and it is a *declaration*: the summary binds to the + * assumption and never discovers for itself that the assumption was false. */ +export interface SpeculationAssumption { + predicateId: string; + version: string; + expected: string; +} + +/** What the authoritative evidence says about one predicate. `authoritative` is not a courtesy: an + * unattested reading is not evidence, and the design refuses to let one stand in for the fact. */ +export interface ResolvedPredicate { + predicateId: string; + version: string; + value: string; + authoritative: boolean; +} + +/** One candidate prepared ahead of the fact it guesses, plus what was prepared *from* the guess. */ +export interface SpeculationCandidate { + taskId: string; + assumptions: readonly SpeculationAssumption[]; + /** Units prepared to continue the branch. The first experiment allows none. */ + speculativeSuccessors: readonly string[]; + /** Irreversible external operations taken on the strength of the guess. The first experiment allows + * none: a wrong guess may cost tokens, never a write nobody can take back. */ + irreversibleOperations: readonly string[]; +} + +/** Whether this is the bounded speculation the design's first experiment permits: exactly one pending + * fact, and nothing prepared from it beyond the one candidate. Returns false rather than throwing so a + * caller can tell "not this shape" from the three outcomes. */ +export function isBoundedSpeculation(candidate: SpeculationCandidate): boolean { + return ( + candidate.assumptions.length === 1 && + candidate.speculativeSuccessors.length === 0 && + candidate.irreversibleOperations.length === 0 + ); +} + +export type SpeculationOutcome = "publish" | "wait" | "discard"; + +/** What the host must do about one bounded speculation candidate, and whether the session that prepared + * it may be reused for the real path. */ +export interface SpeculationDecision { + outcome: SpeculationOutcome; + /** A discarded branch's session may not be reused: the model has already seen the guess, and an answer + * taken from there is not an answer to the real question (design: "失效会话不能复用到真实路径"). */ + sessionReusable: boolean; + reason: string; +} + +/** + * The design's three outcomes, read from authoritative evidence at the publish boundary: + * + * - **true**: the evidence is the guessed value at the guessed version - the candidate may be published. + * - **false**: the evidence contradicts the guess - the candidate is discarded and its branch session is + * closed; the real path runs again under a new ticket, never from this session. + * - **unknown**: no authoritative evidence, or evidence about another version - the candidate stays + * unaccepted and the host waits. Waiting is not a failure, and it is not permission to publish: the + * design's "不确定就等待或 undecidable" is why a missing reading never becomes a silent true. + * + * Asking about a candidate that is not the bounded shape is a caller error, not an outcome, so it is + * refused by name instead of being folded into one of the three. */ +export function speculationOutcome( + candidate: SpeculationCandidate, + resolved: readonly ResolvedPredicate[], +): SpeculationDecision { + if (!isBoundedSpeculation(candidate)) + throw new Error(`${candidate.taskId}: not a bounded speculation candidate`); + const assumption = candidate.assumptions[0]!; + const fact = resolved.find((item) => item.predicateId === assumption.predicateId); + if (!fact) + return { + outcome: "wait", + sessionReusable: true, + reason: `no evidence for ${assumption.predicateId}`, + }; + if (!fact.authoritative) + return { + outcome: "wait", + sessionReusable: true, + reason: `${assumption.predicateId}: not authoritative`, + }; + if (fact.version !== assumption.version) + return { + outcome: "wait", + sessionReusable: true, + reason: `${assumption.predicateId}: evidence is about ${fact.version}, not ${assumption.version}`, + }; + if (fact.value === assumption.expected) + return { + outcome: "publish", + sessionReusable: true, + reason: `${assumption.predicateId}: holds`, + }; + return { + outcome: "discard", + sessionReusable: false, + reason: `${assumption.predicateId}: ${fact.value}, not ${assumption.expected}`, + }; } diff --git a/src/integration/ooo-fusion-plan.ts b/src/integration/ooo-fusion-plan.ts new file mode 100644 index 00000000..c81b8995 --- /dev/null +++ b/src/integration/ooo-fusion-plan.ts @@ -0,0 +1,219 @@ +/** + * Fusion planning: the one move a session may make at a run boundary, and the ceiling an offline + * measurement puts on how few sessions a plan can need. + * + * Legality is not decided here. `sharedSessionLegal` in `src/integration/ooo-execution.ts` owns the + * five conditions, and both halves below call it: the online move asks it about the facts the run + * holds, and the offline graph asks it about an optimistic projection of the same plan. Nothing in + * this file widens a legal pair; it orders legal pairs and prices them. + * + * The split is deliberate. A fused session is irreversible - two units that ran in one session cannot + * be un-fused - so the online half makes one move about the current session and never rewrites a + * committed one, while the offline half is free to compute a best case that no run may read as a + * decision, because it is computed from facts the run does not have yet. + */ +import { + fusionSuccessors, + sharedSessionLegal, + type DispatchTask, + type SessionPlan, +} from "./ooo-execution.ts"; + +/** + * The measured cost of starting one Agent session, from the D arm (2 units, `--slots 1`, 3 reps per + * bound, deepseek-v4-flash): the fused bound was consistently about 1 900 ms faster with tokens flat, + * which is what one avoided session startup is worth. A later measurement that prices startup + * differently changes this constant, not the method. + */ +export const MEASURED_SESSION_STARTUP_MS = 1_900; + +/** + * The plan as fusion would see it if everything went well. The offline half prices the best case, + * because a run's verdicts, cancellations and pending branches are facts it does not have yet; the + * projection leaves only condition 1 (capability, authority, visibility) and the pair identity to + * `sharedSessionLegal`. + */ +export function optimisticPlan(plan: SessionPlan): SessionPlan { + return { + tasks: plan.tasks.map((task) => ({ + ...task, + accepted: true, + cancelled: false, + externalReady: true, + })), + declarations: plan.declarations, + }; +} + +/** Every unit `id` transitively depends on, so a chain never runs a unit before what it needs. */ +function dependenciesOf(byId: ReadonlyMap, id: string): Set { + const found = new Set(); + const queue = [id]; + while (queue.length > 0) { + const current = queue.pop()!; + for (const dependency of byId.get(current)?.dependencies ?? []) { + if (found.has(dependency)) continue; + found.add(dependency); + queue.push(dependency); + } + } + return found; +} + +export interface FusionGraph { + /** The plan's units, in plan order - the order the driver walks and the tie-break it uses. */ + readonly units: readonly string[]; + /** `edges.get(a)` = the units that may continue `a`'s session, in plan order. */ + readonly edges: ReadonlyMap; + /** Whether the successor relation is transitive, which is what makes the chain-cover floor apply. */ + readonly transitive: boolean; +} + +/** + * The successor relation with the run's own facts projected away: the graph the offline half prices. + * + * Two restrictions make it a graph a chain cover can be computed on, and both are properties of the + * problem rather than conveniences: + * + * - a chain is a linear extension, so a unit may not be followed by one it transitively depends on; + * - a chain follows plan order, which is the order the driver walks its candidates in. The relation on + * its own is not a partial order: two independent units with compatible declarations may each follow + * the other, so it has two-cycles and Dilworth needs an acyclic restriction. Plan order is that + * restriction, and it is why the floor bounds order-respecting schedules. + */ +export function fusionGraph(plan: SessionPlan): FusionGraph { + const view = optimisticPlan(plan); + const byId = new Map(plan.tasks.map((task) => [task.id, task])); + const units = view.tasks.map((task) => task.id); + const index = new Map(units.map((id, position) => [id, position])); + const edges = new Map(); + for (const before of units) { + const needed = dependenciesOf(byId, before); + edges.set( + before, + units.filter( + (after) => + after !== before && + index.get(after)! > index.get(before)! && + !needed.has(after) && + sharedSessionLegal(before, after, view), + ), + ); + } + let transitive = true; + for (const [before, successors] of edges) { + for (const middle of successors) { + for (const after of edges.get(middle) ?? []) { + if (after !== before && !successors.includes(after)) transitive = false; + } + } + } + return { units, edges, transitive }; +} + +/** The units reachable from each unit, itself included: the relation's transitive closure. */ +function transitiveClosure(graph: FusionGraph): ReadonlyMap> { + const closure = new Map>(); + for (const unit of graph.units) { + const reached = new Set([unit]); + const queue = [unit]; + while (queue.length > 0) { + const current = queue.pop()!; + for (const next of graph.edges.get(current) ?? []) { + if (reached.has(next)) continue; + reached.add(next); + queue.push(next); + } + } + closure.set(unit, reached); + } + return closure; +} + +/** + * The least number of sessions the plan could need: the minimum chain cover of the relation, which by + * Dilworth equals its maximum antichain, computed as `units - maximumMatching` over the bipartite + * graph of the transitive closure (Konig). A floor, not a schedule - it is reached only when the + * chains happen to be feasible for the cap. + */ +export function chainCoverFloor(graph: FusionGraph): number { + const closure = transitiveClosure(graph); + /** right -> left, the matching found so far. */ + const paired = new Map(); + const augment = (left: string, visited: Set): boolean => { + for (const right of graph.units) { + if (right === left || !closure.get(left)!.has(right) || visited.has(right)) continue; + visited.add(right); + const holder = paired.get(right); + if (holder === undefined || augment(holder, visited)) { + paired.set(right, left); + return true; + } + } + return false; + }; + let matched = 0; + for (const left of graph.units) if (augment(left, new Set())) matched += 1; + return graph.units.length - matched; +} + +/** + * A feasible bound from greedy list scheduling at a declared per-session cap: walk the plan in order + * and append each unit to the open session that may legally take it, preferring the fullest so few + * sessions are opened. Feasible, not optimal - it is what this rule gets, and the gap to + * `chainCoverFloor` is the honest measure of what the rule leaves on the table. + */ +export function listScheduleSessions( + graph: FusionGraph, + cap: number, +): readonly (readonly string[])[] { + if (!Number.isInteger(cap) || cap < 1) + throw new Error("a session cap must be a positive integer"); + const sessions: string[][] = []; + for (const unit of graph.units) { + let best: string[] | undefined; + for (const session of sessions) { + if (session.length >= cap) continue; + if (!(graph.edges.get(session[session.length - 1]!) ?? []).includes(unit)) continue; + if (!best || session.length > best.length) best = session; + } + if (best) best.push(unit); + else sessions.push([unit]); + } + return sessions; +} + +export interface SessionMoveInput { + readonly plan: SessionPlan; + /** The unit that just ran in this session. */ + readonly current: string; + /** How many units this session has already carried. */ + readonly size: number; + /** The declared bound on units per session. */ + readonly bound: number; + /** The units the board still has on offer, in plan order. */ + readonly onOffer: readonly string[]; +} + +/** One move about the current session: admit the next legal successor, or close it by name. */ +export type SessionMove = + | { readonly kind: "admit"; readonly unit: string } + | { readonly kind: "close"; readonly reason: string }; + +/** + * Repair-first: the default is to continue the session that is already running, and a close carries + * the condition that closed it. Only facts may end a session early - a rejected verdict shows up as + * `current` not being accepted, a cancellation and an unmet dependency as `sharedSessionLegal`, and a + * declared external wait as condition 4 - so this is a pure function of the plan and its facts, with + * plan order breaking ties. Numeric optimisation belongs to the offline half and is not read here. + */ +export function nextSessionMove(input: SessionMoveInput): SessionMove { + if (input.size >= input.bound) return { kind: "close", reason: "the declared bound is reached" }; + const successor = fusionSuccessors(input.current, input.plan).find((id) => + input.onOffer.includes(id), + ); + if (successor === undefined) { + return { kind: "close", reason: "no legal successor is on offer" }; + } + return { kind: "admit", unit: successor }; +} diff --git a/src/integration/ooo-round-log.ts b/src/integration/ooo-round-log.ts deleted file mode 100644 index 3bd0f18b..00000000 --- a/src/integration/ooo-round-log.ts +++ /dev/null @@ -1,304 +0,0 @@ -// S2: a round can be replayed from its log instead of re-run. -// -// The durable-execution split this repository already relies on elsewhere is that -// orchestration must be deterministic and every uncertain side effect is an activity -// whose *result* is recorded. A model call is exactly such an activity: replaying a -// round must not call a model again, and the acceptance decisions must follow from the -// frozen inputs plus the recorded artifacts. -// -// Two properties are deliberately not claimed: -// - the log is not trusted. A replayed artifact goes back through the same host -// contract and the same host checks, so an edited or truncated log produces a -// rejection or an explicit failure, never a reproduced verdict. -// - model calls are not re-executed. A different worker answer is a different round, -// which is why the log records the answer rather than the prompt that produced it. -import { createHash } from "node:crypto"; -import { appendFileSync, readFileSync, writeFileSync } from "node:fs"; -import type { CheckTicket } from "./ooo-check.ts"; -import type { CycleWorker, WorkerMetrics } from "./ooo-cycle.ts"; - -/** One recorded round event, in the order the round produced it. */ -export type RoundEvent = - | { - kind: "plan"; - at: string; - tasks: string[]; - checks: string[]; - /** The revision the round's candidate worktrees are checked out from, so a replay - * verifies what the round verified instead of whatever `HEAD` happens to be now. */ - revision?: string; - /** The frozen verification rules. They are not part of any task's frozen work, so - * without this a replay could verify a differently-defined check and still report a - * reproduction: the verifier has to be frozen by the round, not by the process. */ - checkDigest?: string; - /** Dispatch order the round ran under. A replay of a sequential round as an out-of-order - * one is not the same round, so the mode is part of the identity too. */ - mode?: string; - } - | { kind: "check-issued"; at: string; taskId: string; ticket: CheckTicket } - | { - kind: "check-result"; - at: string; - taskId: string; - verdict: string; - outcomes: { label: string; status: string }[]; - } - | { kind: "claim"; at: string; taskId: string; attempt: number; digest: string; owner: string } - | { - kind: "artifact"; - at: string; - taskId: string; - attempt: number; - artifact: string; - metrics?: WorkerMetrics; - } - | { kind: "worker-failed"; at: string; taskId: string; attempt: number; reason: string } - | { - kind: "pushback"; - at: string; - taskId: string; - dependency: string; - requirement: string; - evidence: string; - } - | { kind: "verdict"; at: string; taskId: string; attempt: number; verdict: string } - | { - kind: "mutant"; - at: string; - taskId: string; - id: string; - /** `unmeasured` is not a verdict about the fault: the check could not run, so the - * premise is unproven rather than false. */ - outcome: "killed" | "survived" | "unmeasured"; - } - | { kind: "reopen"; at: string; taskId: string; requirement: string; invalidated: string[] } - | { - kind: "terminal"; - at: string; - accepted: Record; - verdicts: Record; - composed: { verdict: string; files: string[] }; - /** The round's explicit terminal decision, when it was not a normal completion. */ - cancelled?: string; - }; - -/** `Omit` over a discriminated union collapses to the properties every member shares, so - * an event without its timestamp has to be distributed over the members first. Without - * this, every `{ kind: "claim", taskId }` literal is rejected for an unknown property. */ -type DistributiveOmit = T extends unknown ? Omit : never; -export type RoundEventInput = DistributiveOmit; - -const KINDS = new Set([ - "plan", - "check-issued", - "check-result", - "claim", - "artifact", - "worker-failed", - "pushback", - "verdict", - "mutant", - "reopen", - "terminal", -]); - -/** Appends round events as JSON lines. Append-only within one round: a truncated or edited - * line is visible when the log is read back, and replay does not repair it. A new round - * starts a fresh file unless the caller asks to continue one (`append: true`), because a - * file that accumulates several rounds replays as a mixture of all of them. */ -export class RoundLog { - private readonly events: RoundEvent[] = []; - /** Written as a field, not a constructor parameter property: this repository's scripts - * run under Node's strip-only TypeScript mode, which does not support them. */ - private readonly path?: string; - - constructor(path?: string, options: { append?: boolean } = {}) { - this.path = path; - if (path && !options.append) writeFileSync(path, "", "utf8"); - } - - append(event: RoundEvent): void { - this.events.push(event); - if (this.path) appendFileSync(this.path, JSON.stringify(event) + "\n", "utf8"); - } - - recorded(): readonly RoundEvent[] { - return this.path ? readRoundLog(readFileSync(this.path, "utf8")) : this.events; - } - - /** The artifact a task produced for a given attempt, or null when the log does not - * contain one. Absence is never treated as "no artifact needed". */ - artifactFor(taskId: string, attempt: number): string | null { - const event = this.recorded().find( - (item) => item.kind === "artifact" && item.taskId === taskId && item.attempt === attempt, - ); - return event?.kind === "artifact" ? event.artifact : null; - } -} - -/** Reads a log back, refusing anything that is not a well-formed round event. A log the - * reader only half understands must fail loudly: a silently dropped event would make a - * replay look like a round that simply happened to differ. */ -export function readRoundLog(text: string): RoundEvent[] { - const events: RoundEvent[] = []; - text.split(/\r?\n/).forEach((line, index) => { - if (!line.trim()) return; - let parsed: unknown; - try { - parsed = JSON.parse(line); - } catch { - throw new Error(`round log line ${index + 1} is not JSON`); - } - if (!parsed || typeof parsed !== "object") - throw new Error(`round log line ${index + 1} is not an event`); - const event = parsed as { kind?: unknown; at?: unknown }; - if (typeof event.kind !== "string" || !KINDS.has(event.kind)) - throw new Error(`round log line ${index + 1} has unknown kind ${String(event.kind)}`); - if (typeof event.at !== "string" || !event.at) - throw new Error(`round log line ${index + 1} has no timestamp`); - events.push(parsed as RoundEvent); - }); - return events; -} - -/** The terminal event, or null while the round has no recorded end. */ -export function terminalEvent( - events: readonly RoundEvent[], -): Extract | null { - for (let index = events.length - 1; index >= 0; index -= 1) { - const event = events[index]!; - if (event.kind === "terminal") return event; - } - return null; -} - -/** A worker that answers from the log instead of from a model. Anything the log does not - * contain fails the replay rather than being invented: a replay that guesses is not - * evidence about the round it claims to reproduce. */ -export function recordedWorker(events: readonly RoundEvent[]): CycleWorker { - const artifacts = new Map(); - const failures = new Map(); - for (const event of events) { - if (event.kind === "artifact") - artifacts.set(`${event.taskId}#${event.attempt}`, event.artifact); - if (event.kind === "worker-failed") - failures.set(`${event.taskId}#${event.attempt}`, event.reason); - } - return async (taskId, frozen) => { - // The attempt comes from the frozen envelope the worker was handed, never from a call - // counter: reopened tasks interleave, and counting would silently pair the wrong - // attempt with the wrong artifact. - const attempt = frozen.work.attempt; - const failure = failures.get(`${taskId}#${attempt}`); - if (failure) throw new Error(failure); - const artifact = artifacts.get(`${taskId}#${attempt}`); - if (artifact === undefined) - throw new Error(`round log has no artifact for ${taskId} attempt ${attempt}`); - return artifact; - }; -} - -/** A digest of the verification rules a round runs under: the commands, their arguments and - * their labels. Computed the same way at record and replay time, from the same inputs. */ -export function checksDigest( - checks: readonly { label: string; command: string; args: readonly string[] }[], -): string { - const canonical = checks - .map((check) => ({ label: check.label, command: check.command, args: [...check.args] })) - .sort((left, right) => left.label.localeCompare(right.label)); - return createHash("sha256").update(JSON.stringify(canonical)).digest("hex"); -} - -/** A round's frozen work per task and attempt: the identity of everything the round was - * handed (baseline files, instruction, declared faults, budgets, limits). Every input - * change moves it, which is what makes it usable as the round's identity. */ -export function frozenDigests(log: readonly RoundEvent[]): Map { - const digests = new Map(); - for (const event of log) - if (event.kind === "claim") digests.set(`${event.taskId}#${event.attempt}`, event.digest); - return digests; -} - -/** The recorded plan, or null when the log has none. */ -export function recordedPlan( - events: readonly RoundEvent[], -): Extract | null { - return events.find((event) => event.kind === "plan") ?? null; -} - -/** Whether a replay was handed the same frozen work the log recorded. - * - * A replay given different inputs is a *different round*: it can still finish with the same - * verdicts by coincidence, and then the log looks reproduced when it was not. Comparing the - * frozen digests is the cheap check that names the difference instead: it covers the baseline - * files and every other input at once, because all of them feed the digest. */ -export function compareFrozen( - recorded: readonly RoundEvent[], - replayed: readonly RoundEvent[], -): string[] { - const before = frozenDigests(recorded); - const after = frozenDigests(replayed); - const differences: string[] = []; - const checksBefore = recordedPlan(recorded)?.checkDigest; - const checksAfter = recordedPlan(replayed)?.checkDigest; - if (checksBefore !== checksAfter) - differences.push( - `verification rules: ${checksBefore?.slice(0, 12) ?? "not recorded"} -> ` + - `${checksAfter?.slice(0, 12) ?? "not recorded"}`, - ); - const modeBefore = recordedPlan(recorded)?.mode; - const modeAfter = recordedPlan(replayed)?.mode; - if (modeBefore !== modeAfter) - differences.push( - `dispatch order: ${modeBefore ?? "not recorded"} -> ${modeAfter ?? "not recorded"}`, - ); - differences.push(...digestDifferences(before, after)); - return differences; -} - -/** One line per task whose frozen work differs, naming both sides. */ -function digestDifferences(before: Map, after: Map): string[] { - const differences: string[] = []; - for (const key of [...new Set([...before.keys(), ...after.keys()])].sort()) { - const left = before.get(key); - const right = after.get(key); - if (left === right) continue; - if (left === undefined) - differences.push(`${key}: not claimed in the log -> ${right!.slice(0, 12)}`); - else if (right === undefined) - differences.push(`${key}: ${left.slice(0, 12)} -> not claimed in the replay`); - else differences.push(`${key}: ${left.slice(0, 12)} -> ${right.slice(0, 12)}`); - } - return differences; -} - -/** Compares a replayed round with the terminal state its log recorded. Returns the - * differences in words, so a caller can print them instead of a bare false. */ -export function compareTerminal( - recorded: Extract, - replayed: { - accepted: Record; - verdicts: Record; - composed: { verdict: string; files: string[] }; - }, -): string[] { - const differences: string[] = []; - const keys = new Set([ - ...Object.keys(recorded.accepted), - ...Object.keys(replayed.accepted), - ...Object.keys(recorded.verdicts), - ...Object.keys(replayed.verdicts), - ]); - for (const key of [...keys].sort()) { - const before = `${recorded.verdicts[key] ?? "none"}/${recorded.accepted[key] !== undefined ? "accepted" : "not-accepted"}`; - const after = `${replayed.verdicts[key] ?? "none"}/${replayed.accepted[key] !== undefined ? "accepted" : "not-accepted"}`; - if (before !== after) differences.push(`${key}: ${before} -> ${after}`); - } - if (recorded.composed.verdict !== replayed.composed.verdict) - differences.push(`composed: ${recorded.composed.verdict} -> ${replayed.composed.verdict}`); - if (recorded.composed.files.join(",") !== replayed.composed.files.join(",")) - differences.push( - `composed files: ${recorded.composed.files.join(",")} -> ${replayed.composed.files.join(",")}`, - ); - return differences; -} diff --git a/src/integration/task-advisers.ts b/src/integration/task-advisers.ts new file mode 100644 index 00000000..1078f1af --- /dev/null +++ b/src/integration/task-advisers.ts @@ -0,0 +1,299 @@ +/** + * The seam between "what may be started" and "what is started first". + * + * The shared rules own legality (`selectableTasks`), and this module owns nothing else: an adviser + * is handed a bounded projection and may only *rank* the legal set it is given. The four steps the + * design fixes are visible in the shape of the types rather than in a convention: + * + * 1. `selectableTasks` computes the legal candidates. An adviser never computes or widens them. + * 2. A source sees `AdviceProjection` - the admitted, bounded view - and answers with suggestions. + * 3. `orderCandidates` orders *inside* that set: the result is always a permutation of its input. + * 4. `revalidateSuggestion` re-checks an adopted ranking at the claim or commit point, because a + * score computed earlier is not a write licence. + * + * Nothing here enables a capability. The module holds no state, opens no store and reads no graph, + * so it cannot leak a suggestion from one session or branch into another: the projection carries + * both identities and a suggestion whose provenance disagrees with them is refused by name. A run + * with no source, or with a source that is disabled or failing, orders the legal set exactly as the + * rule policy does. + * + * Two design sentences are load-bearing and are enforced by what the code *cannot* express: + * + * - A new task or a new dependency is not a suggestion. There is no field for one: the only action + * a suggestion may name is `next`, and `next` must land on a task that is already in the legal + * set. Changing the plan is an explicit plan revision, which is a different operation. + * - A soft premise is not a dependency. `assumptions` are carried as provenance and are never + * consulted for legality - a source that asserts "the dependency is satisfied" changes nothing, + * because membership of the legal set was decided before it was asked. + * + * `fuse` and `prepare` are named in the shared vocabulary and are not modelled (`UNMODELLED_ACTIONS`); + * a source that proposes either is refused as unsupported rather than scored. + */ +import type { Refusal } from "./task-semantics.ts"; + +/** The two owners the design names as optional advisers. Not a gate: a source is passed in. */ +export const ADVISER_KINDS = ["ha", "mgr"] as const; +export type AdviserKind = (typeof ADVISER_KINDS)[number]; + +/** + * What an adviser may see. + * + * Everything here is either an identity the run already has or a fact the shared layer already + * derived. The projection is built per decision and passed by value, which is what keeps one + * session's or branch's suggestion out of another's: there is no place to remember it. + */ +export interface AdviceProjection { + sessionId: string; + branchId: string; + /** The scoring parameters' version. A different version makes an old score a new score. */ + parametersVersion: string; + /** The projection's version; a changed projection is not the same input. */ + projectionVersion: string; + /** + * The observation order the scores were taken in, and whatever initial time state they started + * from. A source that cannot name them is re-scored rather than reported as a reproduction. + */ + observationOrder?: readonly string[]; + initialState?: string; + /** The legal candidates, in the rule policy's order. */ + legal: readonly string[]; + ready: readonly string[]; + accepted: readonly string[]; + blocked: Readonly>; +} + +/** Where a score came from, and what it was computed against. */ +export interface SuggestionProvenance { + sourceId: string; + kind: AdviserKind; + sessionId: string; + branchId: string; + parametersVersion: string; + projectionVersion: string; + observationOrder?: readonly string[]; + initialState?: string; +} + +export interface Suggestion { + /** + * The only action a suggestion may name. `fuse` and `prepare` are unmodelled shared actions: a + * suggestion cannot start speculative execution or merge units into one acceptance. + */ + action: string; + /** Must be a member of the legal set it was asked about. */ + taskId: string; + score: number; + provenance: SuggestionProvenance; + /** + * Soft premises and hypothesis markers. They explain a score; they never make a task legal, and + * they are not read for legality here. + */ + assumptions?: readonly string[]; +} + +export interface SuggestionSource { + readonly id: string; + readonly kind: AdviserKind; + /** + * A disabled source is not called at all. Its absence must not be silent, so the outcome records + * it as a fallback and the rule policy still answers. + */ + enabled?: boolean; + suggest(projection: AdviceProjection): readonly Suggestion[]; +} + +export interface AdviceFallback { + sourceId: string; + reason: string; +} + +/** A score that could not be attributed to the projection it claimed: it is new, not a replay. */ +export interface RescoredSuggestion { + sourceId: string; + taskId: string; + /** The inputs that were missing or different, so the caller can say what changed. */ + missing: readonly string[]; +} + +export interface AdviceOutcome { + /** A permutation of the input set: an ordering, never a different set. */ + order: readonly string[]; + adopted: readonly { sourceId: string; taskId: string; score: number }[]; + refusals: readonly Refusal[]; + fallbacks: readonly AdviceFallback[]; + rescored: readonly RescoredSuggestion[]; +} + +const RULE_POLICY_FIELD = "policy"; +/** How a refusal says "this proposal is not a suggestion the shared layer can adopt". */ +const SUGGESTION_FIELD = "suggestion"; + +/** + * Order the legal set with whatever optional sources are given. + * + * The policy is stated once and is deliberately dull: tasks a source corroborated come first, in + * score order (ties keep the rule order), and everything else keeps the rule order. With no + * adopted suggestion the result equals the input, so a run without a source is byte-identical to + * the rule policy - which is what the design's "no stable gain, keep the rule policy" needs to be + * measurable rather than asserted. + */ +export function orderCandidates( + legal: readonly string[], + projection: Omit, + sources: readonly SuggestionSource[] = [], +): AdviceOutcome { + const refusals: Refusal[] = []; + const fallbacks: AdviceFallback[] = []; + const rescored: RescoredSuggestion[] = []; + const adopted: { sourceId: string; taskId: string; score: number }[] = []; + const legalSet = new Set(legal); + const full: AdviceProjection = { ...projection, legal }; + + for (const source of sources) { + if (source.enabled === false) { + fallbacks.push({ sourceId: source.id, reason: "source is disabled" }); + continue; + } + let suggestions: readonly Suggestion[]; + try { + suggestions = source.suggest(full); + } catch (error) { + // A failing adviser is an absent adviser: the rule policy answers, and the run records why. + fallbacks.push({ + sourceId: source.id, + reason: `source failed: ${error instanceof Error ? error.message : String(error)}`, + }); + continue; + } + for (const suggestion of suggestions) { + const refusal = refuseSuggestion(suggestion, source, projection, legalSet); + if (refusal) { + refusals.push(refusal); + continue; + } + const missing = missingValidityInputs(suggestion.provenance, projection); + if (missing.length > 0) { + // The score exists but its inputs do not: this is a new score about the current state, and + // reporting it as a reproduction would claim a history nobody recorded. + rescored.push({ sourceId: source.id, taskId: suggestion.taskId, missing }); + continue; + } + adopted.push({ sourceId: source.id, taskId: suggestion.taskId, score: suggestion.score }); + } + } + + return { order: orderByAdopted(legal, adopted), adopted, refusals, fallbacks, rescored }; +} + +/** + * Re-check an adopted ranking where the write happens. + * + * A score is a statement about the state it was computed against. By the time a claim is written, + * the set may have moved - the task may be accepted, cancelled, or claimed by someone else - so the + * ranking is checked against the set as it is now, not as it was. + */ +export function revalidateSuggestion( + adopted: { sourceId: string; taskId: string }, + legalNow: readonly string[], +): Refusal | null { + return legalNow.includes(adopted.taskId) + ? null + : { + task: adopted.taskId, + field: RULE_POLICY_FIELD, + reason: + `the suggestion from ${adopted.sourceId} ranks a task that is no longer a legal ` + + "candidate: a score computed earlier is not a write licence", + }; +} + +function refuseSuggestion( + suggestion: Suggestion, + source: SuggestionSource, + projection: Omit, + legalSet: ReadonlySet, +): Refusal | null { + if (suggestion.action !== "next") { + return { + task: suggestion.taskId, + field: "action", + reason: + `a suggestion may only rank inside the legal set; ${suggestion.action} is not a modelled ` + + "action, and starting speculative execution or merging units is not a ranking", + }; + } + if (!legalSet.has(suggestion.taskId)) { + return { + task: suggestion.taskId, + field: SUGGESTION_FIELD, + reason: + `${source.id} suggested a task outside the legal candidate set; a score orders the set it ` + + "was given and cannot add to it, and a dependency is satisfied by an accepted artifact " + + "rather than by a premise", + }; + } + const provenance = suggestion.provenance; + if ( + provenance.sessionId !== projection.sessionId || + provenance.branchId !== projection.branchId + ) { + return { + task: suggestion.taskId, + field: "provenance", + reason: + `a suggestion scored in session/branch ${provenance.sessionId}/${provenance.branchId} is ` + + `not reused in ${projection.sessionId}/${projection.branchId}`, + }; + } + return null; +} + +/** + * Which inputs the projection would have to carry for this score to be a reproduction of it. + * + * A version that differs is reported the same way a missing input is: both mean the old score was + * taken from a different state, and neither may be presented as the same reading. + */ +function missingValidityInputs( + provenance: SuggestionProvenance, + projection: Omit, +): readonly string[] { + const missing: string[] = []; + if (provenance.parametersVersion !== projection.parametersVersion) + missing.push(`parametersVersion=${provenance.parametersVersion}`); + if (provenance.projectionVersion !== projection.projectionVersion) + missing.push(`projectionVersion=${provenance.projectionVersion}`); + if (!provenance.observationOrder?.length) missing.push("observationOrder"); + else if (!sameOrder(provenance.observationOrder, projection.observationOrder)) + missing.push(`observationOrder=${provenance.observationOrder.join(",")}`); + if (!provenance.initialState) missing.push("initialState"); + else if (provenance.initialState !== projection.initialState) + missing.push(`initialState=${provenance.initialState}`); + return missing; +} + +function sameOrder(left: readonly string[], right: readonly string[] | undefined): boolean { + return !!right && left.length === right.length && left.every((id, index) => id === right[index]); +} + +function orderByAdopted( + legal: readonly string[], + adopted: readonly { sourceId: string; taskId: string; score: number }[], +): readonly string[] { + if (adopted.length === 0) return legal; + const best = new Map(); + for (const entry of adopted) { + const current = best.get(entry.taskId); + if (current === undefined || entry.score > current) best.set(entry.taskId, entry.score); + } + const ruleOrder = new Map(legal.map((id, index) => [id, index])); + return [...legal].sort((left, right) => { + const leftScore = best.get(left); + const rightScore = best.get(right); + if (leftScore === undefined && rightScore === undefined) + return (ruleOrder.get(left) ?? 0) - (ruleOrder.get(right) ?? 0); + if (leftScore === undefined) return 1; + if (rightScore === undefined) return -1; + return rightScore - leftScore || (ruleOrder.get(left) ?? 0) - (ruleOrder.get(right) ?? 0); + }); +} diff --git a/src/integration/task-coordinator.ts b/src/integration/task-coordinator.ts new file mode 100644 index 00000000..7fd60ce2 --- /dev/null +++ b/src/integration/task-coordinator.ts @@ -0,0 +1,464 @@ +/** + * The shared coordinator's write path for a managed entry. + * + * A board entry that a run has adopted is not the board's alone to move. The design puts the + * lifecycle operations (`claim`, `deliver`, `judge`, `resolve`, retention) in the board and the + * coordination in the run: a generic write request on a managed entry is applied *inside the same + * transition* that appends the run's own fact, so the board transition and the run's record of it + * stand or fall together, and a direct board verb cannot move a managed entry beside its run. The + * store refuses that direct write (`requireManagedWriteScope`); this module is what the caller is + * supposed to reach instead. + * + * What is here is deliberately only the write path. Deriving ready/blocked/accepted stays in + * `task-semantics.ts` as pure functions over the same facts, and the board keeps the authoritative + * current state: this records the *transitions* the board's current state cannot keep. + */ +import type { NmgStore } from "../core/store.ts"; +import type { TransactionPort } from "../core/store/base.ts"; +import type { TaskBoardEntry } from "../core/types.ts"; + +/** The fact kind that says a run was cancelled. One home for the vocabulary: the write that appends + * it and the fence that reads it must agree, and neither may guess at the string. */ +export const RUN_CANCELLED_FACT = "run-cancelled"; + +/** The fact kind that binds a logical task to the board entry that carries it. The binding is a run + * fact and not a board column because the board cannot keep it: an entry that a later attempt + * replaces must not lose the record of what it used to carry. */ +export const ENTRY_BOUND_FACT = "entry-bound"; + +/** A managed transition records itself under its own kind, so the run's log keeps claim, delivery, + * judgement and resolution apart instead of collapsing them into one per-(task, attempt) row. */ +export function managedTransitionKind(verb: string): string { + return `board-${verb}`; +} + +/** Why this run refuses a managed write at this moment, or null when it accepts one. The two + * reasons are facts about the run's own log, so both are re-read inside the transition rather + * than remembered from when the caller decided to write. */ +export function managedWriteRefusal(store: NmgStore, runId: string): string | null { + if (!store.taskRunManifest(runId)) + return `run ${runId} is not registered; a managed write needs the run it belongs to`; + const cancelled = store.taskRunFacts(runId).find((fact) => fact.kind === RUN_CANCELLED_FACT); + if (cancelled) + return `run ${runId} was cancelled at sequence ${cancelled.sequence}; its managed entries take no further lifecycle writes`; + return null; +} + +export interface ManagedWriteRequest { + runId: string; + entryId: string; + /** The board verb being applied. It keys the run's fact, so retrying the same verb on the same + * attempt appends once instead of twice. */ + verb: string; + actorId: string; + /** The board transition itself. It runs inside the coordinated transition, on the same + * connection, so a verb that owns a boundary of its own must join with the port it is given. */ + apply: (port: TransactionPort) => T; +} + +export interface ManagedWriteOutcome { + entry: T; + fact: { sequence: number; recorded: boolean }; +} + +/** + * Route one board write on an entry through its run when a run manages it, and leave an unmanaged + * entry on exactly the path it always took. + * + * This is the single routing rule: the daemon's board verbs and every in-process writer use it, so + * "managed" is decided in one place instead of by each caller's belief about the entry. An entry + * that no run has bound costs one indexed lookup and no transaction of its own. + */ +export function coordinatedEntryWrite( + store: NmgStore, + request: { + verb: string; + entryId: string; + actorId: string; + apply: () => T; + }, +): T { + const binding = store.taskRunForEntry(request.entryId); + if (!binding) return request.apply(); + return coordinatedBoardWrite(store, { + runId: binding.runId, + entryId: request.entryId, + verb: request.verb, + actorId: request.actorId, + apply: () => request.apply(), + }).entry; +} + +/** + * Apply one board transition to a managed entry, with the run's fact in the same transaction. + * + * The order is the design's: check the run's own state, re-read the binding at the moment of the + * write (not when the caller decided to make it), apply the board verb, record the transition. Any + * failure - including one the caller catches - is the transition's failure, and the store's + * boundary rolls the whole thing back rather than leaving a verdict without its binding. + */ +export function coordinatedBoardWrite( + store: NmgStore, + request: ManagedWriteRequest, +): ManagedWriteOutcome { + return store.coordinateRunWrite(request.runId, (port) => { + const refusal = managedWriteRefusal(store, request.runId); + if (refusal) throw new Error(refusal); + const binding = store.taskRunForEntry(request.entryId); + if (!binding) + throw new Error( + `entry ${request.entryId} is not adopted by a run, so there is nothing to coordinate it with`, + ); + if (binding.runId !== request.runId) + throw new Error( + `entry ${request.entryId} belongs to run ${binding.runId}, not ${request.runId}`, + ); + const entry = request.apply(port); + // The board is authoritative for current state, so the payload only has to make the transition + // readable later: which agent moved it, and to what state the board moved. + const status = + typeof entry === "object" && entry !== null && "status" in entry + ? String((entry as { status: unknown }).status) + : null; + const fact = store.appendTaskRunFact( + { + runId: request.runId, + kind: managedTransitionKind(request.verb), + taskId: binding.taskId, + attempt: binding.attempt, + entryId: request.entryId, + payload: JSON.stringify({ actorId: request.actorId, status }), + }, + port, + ); + return { entry, fact }; + }); +} + +export interface EntryBindingRequest { + runId: string; + /** The logical task, as the run froze it. */ + taskId: string; + /** The board channel the entry lives on. */ + boardTaskId: string; + entryId: string; + /** Which attempt of that task the entry carries. A retry is a new attempt with its own entry. */ + attempt?: number; +} + +/** + * Bind one board entry to one of the run's frozen tasks, so that the entry's later lifecycle writes + * go through the run's coordinated transition instead of beside it. + * + * Every condition here is a fact this store already holds rather than a caller's claim: the run is + * registered and live, the task was frozen (a run cannot adopt an entry for work it never froze), + * the entry really is on the channel the caller names, and one entry carries one task. Re-binding + * the same task and attempt to the same entry is a retry and is not recorded twice; binding it to a + * different entry is refused rather than silently dropped, because the stored fact is keyed by task + * and attempt and would otherwise hide the disagreement. + */ +export function bindRunEntry( + store: NmgStore, + request: EntryBindingRequest, + port?: TransactionPort, +): { sequence: number; recorded: boolean } { + const attempt = request.attempt ?? 1; + const work = (inner: TransactionPort): { sequence: number; recorded: boolean } => { + const refusal = managedWriteRefusal(store, request.runId); + if (refusal) throw new Error(refusal); + if (!isFrozen(store, request.runId, request.taskId)) + throw new Error( + `run ${request.runId} never froze task ${request.taskId}; there is no task to bind an entry to`, + ); + if (!store.getTaskBoardEntryById(request.boardTaskId, request.entryId)) + throw new Error( + `no board entry ${request.entryId} on ${request.boardTaskId}; the binding names the entry it carries`, + ); + const bound = store.taskRunForEntry(request.entryId); + if (bound && (bound.runId !== request.runId || bound.taskId !== request.taskId)) + throw new Error( + `entry ${request.entryId} already carries task ${bound.taskId} of run ${bound.runId}; one entry carries one task`, + ); + const existing = boundFact(store, request.runId, request.taskId, attempt); + if (existing && existing.entryId !== request.entryId) + throw new Error( + `task ${request.taskId} of run ${request.runId} already carries entry ${existing.entryId}; another entry is another attempt, not a rebinding`, + ); + if (existing) return { sequence: existing.sequence, recorded: false }; + return store.appendTaskRunFact( + { + runId: request.runId, + kind: ENTRY_BOUND_FACT, + taskId: request.taskId, + attempt, + entryId: request.entryId, + // The channel is part of naming the entry (the board's own read by id needs it), so the + // fact records it and a status reader does not search every channel to resolve a binding. + payload: JSON.stringify({ boardTaskId: request.boardTaskId }), + }, + inner, + ); + }; + return port ? work(port) : store.coordinateRunWrite(request.runId, work); +} + +/** Whether this run froze that task. Frozen is what a binding is a binding *to*. */ +function isFrozen(store: NmgStore, runId: string, taskId: string): boolean { + return store.taskRunTasks(runId).some((task) => task.taskId === taskId); +} + +/** The board entry a caller wants created, in the board's own creation shape. */ +export type NewBoardEntry = Parameters[0]; + +/** + * Create a board entry and adopt it into a run's frozen task in one transition. + * + * Adopting is part of creating the entry rather than a second call after it: an entry that exists + * without its binding is an unmanaged hole, and a caller that crashed between two calls would leave + * one - the board would keep the entry and no run would manage it, so a direct board write could + * move it beside the run that believes it owns the transition. + */ +export function createBoundEntry( + store: NmgStore, + request: { + entry: NewBoardEntry; + runId: string; + taskId: string; + attempt?: number; + }, +): { entry: TaskBoardEntry; bound: { sequence: number; recorded: boolean } } { + return store.writeTransaction((port) => { + const entry = store.putTaskBoardEntry(request.entry, port); + const bound = bindRunEntry( + store, + { + runId: request.runId, + taskId: request.taskId, + attempt: request.attempt, + boardTaskId: entry.taskId, + entryId: entry.id, + }, + port, + ); + return { entry, bound }; + }); +} + +/** The binding already recorded for this task and attempt, if any. */ +function boundFact( + store: NmgStore, + runId: string, + taskId: string, + attempt: number, +): { sequence: number; entryId: string | null } | undefined { + const fact = store + .taskRunFacts(runId) + .find( + (candidate) => + candidate.kind === ENTRY_BOUND_FACT && + candidate.taskId === taskId && + candidate.attempt === attempt, + ); + return fact && { sequence: fact.sequence, entryId: fact.entryId }; +} + +// ---- The run surface: the transitions a run goes through, in the order it goes through them. +// This is the one place a caller outside this process can reach them from - the daemon's `taskRun` +// method and its board `put` adoption both call these - so the order and the checks live here +// rather than in the transport, which only validates shapes. + +export interface RunRegistrationRequest { + runId: string; + /** A digest of the frozen plan, so a later plan cannot be presented as this run's. */ + planDigest: string; + policy: string; + revision: string; + /** The retention relation: entries this run references are not ordinary TTL candidates. */ + retention: string; +} + +/** + * Register a run's frozen identity. This is the first transition of a run and the only one that does + * not need a run to exist already: a plan freeze, a binding and every lifecycle write on a managed + * entry are refused for a run this store does not know. + * + * The store owns the rule that re-registering a different plan is refused (a run cannot be re-opened + * onto a different plan without becoming a different run); this wrapper exists so the surface's + * entry point is named in the coordinator rather than in the transport. + */ +export function registerRun(store: NmgStore, request: RunRegistrationRequest): { runId: string } { + store.registerTaskRun(request); + return { runId: request.runId }; +} + +/** One task of a run's plan, as it is frozen. */ +export interface RunPlanTask { + taskId: string; + position: number; + revision: string; + input: string; + dependencies: readonly string[]; + effect: string; + waitEvent?: string | null; + operation?: string; + kind?: string; + patchFiles?: readonly string[] | null; + patchEditable?: readonly string[] | null; +} + +export interface RunPlanFreezeRequest { + runId: string; + /** The array order is the plan order: a position is not a second field the caller can contradict. */ + tasks: readonly RunPlanTaskInput[]; +} + +/** + * Freeze a plan in one transition: the plan is what every later decision is read against, so a + * half-frozen plan is not a state a caller should be able to observe. + * + * Every dependency must name a task of this run's plan - one frozen earlier in this run, or one in + * this request. A dangling dependency is refused by name here instead of making the task + * permanently unready: this freeze is the last moment at which the plan is still only a proposal. + */ +export function freezeRunPlan( + store: NmgStore, + request: RunPlanFreezeRequest, +): { runId: string; frozen: number } { + const refusal = managedWriteRefusal(store, request.runId); + if (refusal) throw new Error(refusal); + if (new Set(request.tasks.map((task) => task.taskId)).size !== request.tasks.length) + throw new Error(`run ${request.runId}: one freeze cannot name the same task twice`); + const known = new Set([ + ...store.taskRunTasks(request.runId).map((task) => task.taskId), + ...request.tasks.map((task) => task.taskId), + ]); + for (const task of request.tasks) { + for (const dependency of task.dependencies) { + if (dependency === task.taskId) + throw new Error(`task ${task.taskId} depends on itself, which can never be satisfied`); + if (!known.has(dependency)) + throw new Error( + `task ${task.taskId} depends on ${dependency}, which this run's plan does not freeze`, + ); + } + } + return store.coordinateRunWrite(request.runId, (port) => { + // The array order is the plan order: the position comes from here, not from the request. + request.tasks.forEach((task, position) => + store.freezeTaskRunTask({ ...task, runId: request.runId, position }, port), + ); + return { runId: request.runId, frozen: request.tasks.length }; + }); +} + +export interface RunCancellationRequest { + runId: string; + /** Omit to cancel the run; name a task to cancel that task of its plan. */ + taskId?: string; + reason?: string; +} + +/** + * Cancel a run, or one task of its plan. This is the only writer of the cancellation fact the + * managed-write fence and the dispatch derivation both read: a cancellation that only a test could + * append would leave the state those rules enforce unreachable from any real client. + * + * A run-level cancellation carries an empty task id, which is the schema's convention for a fact + * that belongs to the run rather than to a task. Cancelling twice is a retry and is recorded once, + * so a client that lost the response does not have to guess whether its cancel took. + */ +export function cancelRun( + store: NmgStore, + request: RunCancellationRequest, +): { sequence: number; recorded: boolean } { + if (!store.taskRunManifest(request.runId)) + throw new Error(`run ${request.runId} is not registered; there is nothing to cancel`); + if (request.taskId !== undefined && !isFrozen(store, request.runId, request.taskId)) + throw new Error( + `run ${request.runId} never froze task ${request.taskId}; there is nothing to cancel`, + ); + return store.coordinateRunWrite(request.runId, (port) => + store.appendTaskRunFact( + { + runId: request.runId, + kind: RUN_CANCELLED_FACT, + taskId: request.taskId ?? "", + payload: request.reason ? JSON.stringify({ reason: request.reason }) : null, + }, + port, + ), + ); +} + +/** One task of a run's plan, before its position is filled in from the request order. */ +export type RunPlanTaskInput = Omit; + +/** One binding of the run's plan to the board entry that carries it. */ +export interface RunBinding { + taskId: string; + attempt: number; + entryId: string | null; + sequence: number; + /** The entry as the board has it now, or null when the fact predates the channel payload or the + * entry is gone. Absence is reported and never remedied: this view writes nothing. */ + entry: { taskId: string; status: string; claimedBy: string | null; ackedBy: string[] } | null; +} + +/** + * The run's record as stored: its frozen identity, its plan, its appended facts, and each binding + * resolved to the entry it carries. + * + * Deliberately not derived here: ready/blocked/accepted. That is the shared pure function over these + * facts, and a second answer computed on this side would be the competing truth the design forbids. + */ +export interface RunStatus { + runId: string; + manifest: ReturnType; + tasks: ReturnType; + facts: ReturnType; + bindings: RunBinding[]; +} + +/** A read: it registers nothing, appends nothing and migrates nothing. */ +export function taskRunStatus(store: NmgStore, runId: string): RunStatus { + const facts = store.taskRunFacts(runId); + const bindings: RunBinding[] = []; + for (const fact of facts) { + if (fact.kind !== ENTRY_BOUND_FACT) continue; + const channel = boundChannel(fact.payload); + const entry = + channel && fact.entryId ? store.getTaskBoardEntryById(channel, fact.entryId) : null; + bindings.push({ + taskId: fact.taskId, + attempt: fact.attempt, + entryId: fact.entryId, + sequence: fact.sequence, + entry: entry + ? { + taskId: entry.taskId, + status: entry.status, + claimedBy: entry.claimedBy ?? null, + ackedBy: entry.ackedBy ?? [], + } + : null, + }); + } + return { + runId, + manifest: store.taskRunManifest(runId), + tasks: store.taskRunTasks(runId), + facts, + bindings, + }; +} + +/** The channel a binding fact recorded, or null for a fact written before it carried one. */ +function boundChannel(payload: string | null): string | null { + if (!payload) return null; + try { + const parsed = JSON.parse(payload) as { boardTaskId?: unknown }; + return typeof parsed.boardTaskId === "string" ? parsed.boardTaskId : null; + } catch { + return null; + } +} diff --git a/src/integration/task-semantics-interleavings.ts b/src/integration/task-semantics-interleavings.ts new file mode 100644 index 00000000..1a3530b1 --- /dev/null +++ b/src/integration/task-semantics-interleavings.ts @@ -0,0 +1,654 @@ +/** + * The design's last offline acceptance: enumerate the legal event interleavings of at most four + * units and check, for every publication they allow, the publication's obligations, its inputs and + * its source. + * + * This is a checker, not a second model. It owns no notion of acceptance: it reads the derived view + * (`deriveStatus`, which itself calls the one acceptance predicate) and asks whether what that view + * publishes is supported by the facts the interleaving recorded. + * + * The two publications are checked for what each one actually claims: + * + * - a **dispatch** (`ready`) claims the run may hand this task out, so every declared input must be + * closed: present, current, un-cancelled and itself accepted by the same predicate. That is the + * design's "后继未过验收时不能偷跑"; + * - a **completion** (the accepted set closed over the same predicate) claims this unit's bytes are + * accepted and that what it read from is closed: the verdict must judge those bytes, the bytes + * must exist, the unit must not be cancelled, and every input must be present, current, + * un-cancelled, accepted and attributable - an input whose verdict judged other bytes is not + * evidence about the input now. That is "历史 event 中曾 accepted 不表示当前仍 accepted" and + * "候选存在不等于接受". + * + * A completion is the dependency-closed set rather than the raw per-unit answer, because a completion + * is the run saying "this unit is done", which is the same closure `nextTask`'s eligibility computes. + * The per-unit acceptance query stays per-unit and is what the closure is built from: one predicate, + * asked twice, which is what the design requires instead of two notions of "accepted". + * + * Passing says this finite model satisfies the listed properties. It does not say any Agent program + * is correct - the design's own caveat. + */ +import { + compileTaskUnits, + deriveStatus, + isAccepted, + type CompileInput, + type Obligation, + type RecordedFacts, + type TaskUnit, +} from "./task-semantics.ts"; +import { checkedSlots } from "./ooo-execution.ts"; +import { acceptedClosure } from "./task-semantics-model.ts"; + +/** The design's cap for the enumeration. Above it this refuses rather than grows. */ +export const MAX_INTERLEAVING_UNITS = 4; + +/** The budgets every enumeration checks by default: one slot, the rule the repository had, and two, + * the smallest declared budget that lets two claims coexist - which is what the C arm varies. A + * caller may declare others; each must be a positive integer. */ +export const DEFAULT_BUDGETS: readonly number[] = [1, 2]; +/** Events per enumerated interleaving. This cap is what keeps the enumeration finite: four units of + * three events each already merge 369,600 ways, so six is where "枚举合法交错" stays a check a test + * runs rather than a tool nobody runs. */ +export const MAX_INTERLEAVING_EVENTS = 6; + +export type InterleavingEventKind = + | "deliver" + | "redeliver" + | "judge-accept" + | "judge-reject" + | "judge-undecidable" + /** An accepted verdict bound to bytes other than the ones the unit delivered. */ + | "judge-stale-digest" + | "cancel" + | "claim" + /** The input the unit read is no longer the current one. */ + | "revision-drift" + | "external-ready"; + +export interface InterleavingEvent { + unit: string; + kind: InterleavingEventKind; +} + +/** One unit's script: the events about that unit, in the order they can happen. An interleaving is + * an order-preserving merge of the scripts, and a script set is one script per unit. */ +export interface UnitScript { + unit: string; + events: readonly InterleavingEventKind[]; +} + +export interface PublicationViolation { + obligation: Obligation; + /** Which publication this is about: a dispatch or a completion. */ + publication: "dispatch" | "completion"; + unit: string; + /** Index of the event whose prefix this was checked at. */ + atStep: number; + reason: string; +} + +export interface InterleavingFinding extends PublicationViolation { + interleaving: readonly InterleavingEvent[]; +} + +/** A property a declared claim budget must have, checked on a view the budget produced. These are + * not the unit obligations: a budget is a property of the run, not of a task. */ +export interface BudgetViolation { + property: BudgetProperty; + /** The budget whose view broke it. */ + budget: number; + unit: string; + atStep: number; + reason: string; +} + +export type BudgetProperty = + /** A task someone is working is not offered to a second worker, whatever the budget. */ + | "claimed-task-is-not-startable" + /** A smaller budget's candidates stay candidates: a bigger budget adds, it does not replace. */ + | "a-bigger-budget-keeps-the-smaller-candidates"; + +export interface BudgetFinding extends BudgetViolation { + interleaving: readonly InterleavingEvent[]; +} + +export interface InterleavingReport { + units: readonly string[]; + interleavings: number; + steps: number; + /** Publications checked, summed over the declared budgets: each budget publishes its own set. */ + dispatches: number; + completions: number; + /** The declared budgets this report walked. */ + budgets: readonly number[]; + /** How often a bigger budget published a task a smaller one did not: the count that makes the + * budget's own effect on the model visible rather than assumed. */ + widened: number; + /** Refusal, when the input cannot be enumerated at all. */ + refused?: string; + findings: readonly InterleavingFinding[]; + /** What a declared budget did to the view that the declared budgets may not do. */ + budgetFindings: readonly BudgetFinding[]; +} + +const bytes = (unit: string, generation = 0): string => `${unit}-artifact-v${generation}`; +const JUDGE_VERDICTS: Readonly> = { + "judge-accept": "accepted", + "judge-reject": "rejected", + "judge-undecidable": "undecidable", +}; + +interface MutableFacts { + artifacts: Record; + verdicts: Record; + revisions: Record; + sourceRevisions: Record; + cancellations: string[]; + claimed: string[]; + externalReady: string[]; +} + +/** What one event records. Delivery records the bytes and the revision they were built from, so a + * later drift event is a fact about the run rather than a rewritten artifact. */ +export function recordEvent(facts: MutableFacts, event: InterleavingEvent): void { + const { unit, kind } = event; + if (kind === "deliver") { + facts.artifacts[unit] = bytes(unit); + facts.revisions[unit] = "rev-1"; + return; + } + if (kind === "redeliver") { + // New bytes for the same input revision: the verdict that judged the old bytes no longer + // describes these, which is how acceptance withdraws without anyone cancelling anything. + facts.artifacts[unit] = bytes(unit, 1); + facts.revisions[unit] ??= "rev-1"; + return; + } + if (kind === "judge-stale-digest") { + facts.verdicts[unit] = { digest: bytes(unit, 9), verdict: "accepted" }; + return; + } + const verdict = JUDGE_VERDICTS[kind]; + if (verdict) { + // A verdict before any delivery is recorded with a digest for bytes that do not exist: the + // interleaving may do that, and the publication check is what has to notice. + facts.verdicts[unit] = { digest: facts.artifacts[unit] ?? bytes(unit), verdict }; + return; + } + if (kind === "cancel") facts.cancellations.push(unit); + else if (kind === "claim") facts.claimed.push(unit); + else if (kind === "external-ready") facts.externalReady.push(unit); + else if (kind === "revision-drift") facts.sourceRevisions[unit] = "rev-2"; +} + +function emptyFacts(): MutableFacts { + return { + artifacts: {}, + verdicts: {}, + revisions: {}, + sourceRevisions: {}, + cancellations: [], + claimed: [], + externalReady: [], + }; +} + +/** The facts one publication check reads, and where its findings go. */ +interface CheckContext { + byId: ReadonlyMap; + facts: RecordedFacts; + atStep: number; + violations: PublicationViolation[]; +} + +function fail( + context: CheckContext, + publication: PublicationViolation["publication"], + unit: string, + obligation: Obligation, + reason: string, +): void { + context.violations.push({ obligation, publication, unit, atStep: context.atStep, reason }); +} + +const artifactOf = (facts: RecordedFacts, id: string): string | undefined => facts.artifacts?.[id]; +const verdictOf = ( + facts: RecordedFacts, + id: string, +): { digest: string; verdict: string } | undefined => facts.verdicts?.[id]; +const cancelled = (facts: RecordedFacts, id: string): boolean => + facts.cancellations?.includes(id) ?? false; + +// What a published input has to satisfy, whichever publication names it. Whether an input is +// current, and whether its bytes are the bytes its verdict judged, is the one acceptance predicate's +// answer rather than a second comparison here: a drifted input is an unaccepted input. +function checkInputs( + context: CheckContext, + unit: TaskUnit, + publication: PublicationViolation["publication"], +): void { + for (const dependency of unit.inputs.dependencies) { + if (!artifactOf(context.facts, dependency)) { + fail( + context, + publication, + unit.id, + "input-closure", + `published while its declared input ${dependency} has no artifact`, + ); + continue; + } + if (cancelled(context.facts, dependency)) + fail( + context, + publication, + unit.id, + "stoppable-voidable", + `published although its input ${dependency} was cancelled`, + ); + const dependencyUnit = context.byId.get(dependency); + if (dependencyUnit && !isAccepted(dependencyUnit, context.facts)) + fail( + context, + publication, + unit.id, + "input-closure", + `published although its input ${dependency} is not accepted`, + ); + } +} + +// A dispatch hands the unit out, so everything it will read has to be closed first, and a cancelled +// task is not handed out at all. +function checkDispatch(context: CheckContext, unit: TaskUnit): void { + checkInputs(context, unit, "dispatch"); + if (cancelled(context.facts, unit.id)) + fail( + context, + "dispatch", + unit.id, + "stoppable-voidable", + "dispatched although it was cancelled", + ); +} + +// A completion claims this unit's bytes are accepted: the verdict must judge those bytes, the bytes +// must exist, the unit must not be cancelled, and its inputs must be closed. +function checkCompletion(context: CheckContext, unit: TaskUnit): void { + const artifact = artifactOf(context.facts, unit.id); + const verdict = verdictOf(context.facts, unit.id); + // explicit-acceptance: a verdict is acceptance of the bytes it judged, not of the task. + if (!verdict || verdict.verdict !== "accepted") + fail( + context, + "completion", + unit.id, + "explicit-acceptance", + `published as complete without an accepted verdict (${verdict?.verdict ?? "none"})`, + ); + else if (verdict.digest !== artifact) + fail( + context, + "completion", + unit.id, + "explicit-acceptance", + `the accepted verdict judged ${verdict.digest} while the published bytes are ${artifact}`, + ); + // artifact-handoff: what is published is an artifact, and there is no artifact without bytes. + if (!artifact) + fail( + context, + "completion", + unit.id, + "artifact-handoff", + "published as complete with no delivered artifact", + ); + if (cancelled(context.facts, unit.id)) + fail( + context, + "completion", + unit.id, + "stoppable-voidable", + "published as complete although it was cancelled", + ); + checkInputs(context, unit, "completion"); +} + +/** + * Whether what the view publishes is supported by the recorded facts. The publications are given + * rather than derived, so a hand-built violation can be handed in: that is what lets a deleted + * condition here be caught by its own case instead of only by luck. + */ +export function checkPublications( + units: readonly TaskUnit[], + facts: RecordedFacts, + publications: { dispatches?: readonly string[]; completions?: readonly string[] }, + atStep = 0, +): PublicationViolation[] { + const context: CheckContext = { + byId: new Map(units.map((unit) => [unit.id, unit])), + facts, + atStep, + violations: [], + }; + for (const id of publications.dispatches ?? []) { + const unit = context.byId.get(id); + if (unit) checkDispatch(context, unit); + } + for (const id of publications.completions ?? []) { + const unit = context.byId.get(id); + if (unit) checkCompletion(context, unit); + } + return context.violations; +} + +/** Every interleaving of the scripts: an order-preserving merge, one event per step. */ +function interleavings(scripts: readonly UnitScript[]): InterleavingEvent[][] { + const out: InterleavingEvent[][] = []; + const walk = (positions: number[], prefix: InterleavingEvent[]): void => { + if (positions.every((position, index) => position === scripts[index]!.events.length)) { + out.push([...prefix]); + return; + } + for (const [index, script] of scripts.entries()) { + const position = positions[index]!; + const kind = script.events[position]; + if (kind === undefined) continue; + const next = [...positions]; + next[index] = position + 1; + walk(next, [...prefix, { unit: script.unit, kind }]); + } + }; + walk( + scripts.map(() => 0), + [], + ); + return out; +} + +function refusal(units: readonly string[], refused: string): InterleavingReport { + return { + units: [...units], + interleavings: 0, + steps: 0, + dispatches: 0, + completions: 0, + budgets: [], + widened: 0, + refused, + findings: [], + budgetFindings: [], + }; +} + +/** + * What a declared budget must not do to the view it produced, given the publications themselves. + * + * The publications are given rather than derived, so a hand-built violation can be handed in: that is + * what lets a deleted condition here be caught by its own case instead of only by luck. + */ +export function checkBudget(input: { + budget: number; + ready: readonly string[]; + claimed: readonly string[]; + /** What the next smaller declared budget published, when there is one. */ + smallerReady?: readonly string[]; + atStep?: number; +}): BudgetViolation[] { + const atStep = input.atStep ?? 0; + const out: BudgetViolation[] = []; + for (const unit of input.ready) + if (input.claimed.includes(unit)) + out.push({ + property: "claimed-task-is-not-startable", + budget: input.budget, + unit, + atStep, + reason: `the ${input.budget}-slot view offers ${unit} although it is already claimed`, + }); + for (const unit of input.smallerReady ?? []) + if (!input.ready.includes(unit)) + out.push({ + property: "a-bigger-budget-keeps-the-smaller-candidates", + budget: input.budget, + unit, + atStep, + reason: `the ${input.budget}-slot view dropped ${unit}, which a smaller budget published`, + }); + return out; +} + +/** + * One prefix as every declared budget sees it: what each budget publishes, and the violations of the + * budget's own obligations. Returned rather than pushed, so the walk over budgets and the walk over + * interleavings each stay one loop deep - and the dependency-closed accepted set, which no budget + * changes, is computed once per prefix rather than once per budget. + */ +function budgetViews( + compiled: ReturnType, + facts: RecordedFacts, + budgets: readonly number[], + step: number, +): { + published: string[][]; + accepted: readonly string[]; + findings: PublicationViolation[]; + budgetFindings: BudgetViolation[]; + widened: number; +} { + const published: string[][] = []; + const findings: PublicationViolation[] = []; + const budgetFindings: BudgetViolation[] = []; + // A run's completions are the dependency-closed accepted set: the same closure the dispatch rule + // computes, over the one acceptance predicate. + const accepted = [...acceptedClosure(compiled.units, (unit) => isAccepted(unit, facts))]; + let widened = 0; + let smaller: readonly string[] | undefined; + for (const budget of budgets) { + const ready = [...deriveStatus(compiled.units, facts, budget).ready]; + if (smaller) widened += ready.filter((id) => !smaller!.includes(id)).length; + for (const violation of checkPublications( + compiled.units, + facts, + { dispatches: ready, completions: accepted }, + step, + )) + findings.push(violation); + for (const violation of checkBudget({ + budget, + ready, + claimed: facts.claimed ?? [], + ...(smaller ? { smallerReady: smaller } : {}), + atStep: step, + })) + budgetFindings.push(violation); + published.push(ready); + smaller = ready; + } + return { published, accepted, findings, budgetFindings, widened }; +} + +/** + * Enumerate every legal interleaving of one script set and check every publication at every prefix. + * A prefix is checked rather than only the final state, because "published earlier and voided later" + * is exactly the interleaving this exists for. + * + * Every prefix is walked once per declared budget: a run that admits N claims publishes a set the + * one-slot run does not, and that set has to satisfy the same obligations plus the budget's own. + */ +export function enumerateInterleavings(input: { + plan: CompileInput["plan"]; + specs?: CompileInput["specs"]; + requires?: CompileInput["requires"]; + scripts: readonly UnitScript[]; + budgets?: readonly number[]; +}): InterleavingReport { + const compiled = compileTaskUnits({ + plan: input.plan, + specs: input.specs, + requires: input.requires, + }); + const ids = compiled.units.map((unit) => unit.id); + if (!compiled.legal) + return refusal( + ids, + `refused: the plan has ${compiled.refusals.length} refusal(s), so no interleaving is legal`, + ); + if (compiled.units.length > MAX_INTERLEAVING_UNITS) + return refusal( + ids, + `refused: ${compiled.units.length} units exceeds the enumeration's cap of ${MAX_INTERLEAVING_UNITS}`, + ); + const known = new Set(ids); + const scripted = new Set(); + for (const script of input.scripts) { + if (!known.has(script.unit)) + return refusal( + ids, + `refused: the script for ${script.unit} names a unit the plan does not have`, + ); + if (scripted.has(script.unit)) + return refusal( + ids, + `refused: ${script.unit} has two scripts, so the merge would not be a legal order of that unit's events`, + ); + scripted.add(script.unit); + } + const total = input.scripts.reduce((sum, script) => sum + script.events.length, 0); + if (total > MAX_INTERLEAVING_EVENTS) + return refusal( + ids, + `refused: ${total} events exceeds the enumeration's cap of ${MAX_INTERLEAVING_EVENTS}`, + ); + + const findings: InterleavingFinding[] = []; + const budgetFindings: BudgetFinding[] = []; + const budgets = [...(input.budgets ?? DEFAULT_BUDGETS)]; + for (const budget of budgets) { + try { + checkedSlots(budget); + } catch { + return refusal(ids, `refused: budget ${String(budget)} is not a positive integer`); + } + } + let steps = 0; + let dispatches = 0; + let completions = 0; + let widened = 0; + const enumerated = interleavings(input.scripts); + for (const interleaving of enumerated) { + const facts = emptyFacts(); + for (const [step, event] of interleaving.entries()) { + recordEvent(facts, event); + steps += 1; + const views = budgetViews(compiled, facts, budgets, step); + for (const ready of views.published) dispatches += ready.length; + completions += views.accepted.length; + widened += views.widened; + for (const violation of views.findings) findings.push({ ...violation, interleaving }); + for (const violation of views.budgetFindings) + budgetFindings.push({ ...violation, interleaving }); + } + } + return { + units: ids, + interleavings: enumerated.length, + steps, + dispatches, + completions, + budgets, + widened, + findings, + budgetFindings, + }; +} + +export interface TableReport extends InterleavingReport { + sets: number; + perSet: readonly InterleavingReport[]; +} + +/** Run a table of script sets: the design's shapes are several orders of the same small plan, and a + * script set is one per-unit order, so the table is what covers them. */ +export function enumerateTable(input: { + plan: CompileInput["plan"]; + specs?: CompileInput["specs"]; + requires?: CompileInput["requires"]; + sets: readonly (readonly UnitScript[])[]; + budgets?: readonly number[]; +}): TableReport { + const perSet = input.sets.map((scripts) => + enumerateInterleavings({ + plan: input.plan, + specs: input.specs, + requires: input.requires, + scripts, + ...(input.budgets ? { budgets: input.budgets } : {}), + }), + ); + const refused = perSet.find((report) => report.refused); + return { + units: perSet[0]?.units ?? [], + sets: perSet.length, + perSet, + interleavings: perSet.reduce((sum, report) => sum + report.interleavings, 0), + steps: perSet.reduce((sum, report) => sum + report.steps, 0), + dispatches: perSet.reduce((sum, report) => sum + report.dispatches, 0), + completions: perSet.reduce((sum, report) => sum + report.completions, 0), + budgets: perSet[0]?.budgets ?? [], + widened: perSet.reduce((sum, report) => sum + report.widened, 0), + findings: perSet.flatMap((report) => report.findings), + budgetFindings: perSet.flatMap((report) => report.budgetFindings), + ...(refused?.refused ? { refused: refused.refused } : {}), + }; +} + +/** The plan the scripts are written against: one unit and one that depends on it. */ +export const DESIGN_PLAN: CompileInput["plan"] = [ + ["P", "rev-1", [], "isolated-artifact", null, null], + ["D", "rev-1", ["P"], "read-only", null, null], +]; + +/** + * The script sets the design's traps are made of: bytes before and after the verdict, a verdict + * bound to other bytes, a cancellation around a verdict, a redelivery after acceptance, an input + * that drifted after the artifact was built, and the dependent whose input was rejected. Each set is + * a legal order of one unit's events, so an interleaving of it is a legal order of the run's. + */ +export const DESIGN_SCRIPT_SETS: readonly (readonly UnitScript[])[] = [ + [{ unit: "P", events: ["cancel"] }], + [{ unit: "P", events: ["deliver", "judge-accept"] }], + [{ unit: "P", events: ["judge-accept", "deliver"] }], + [{ unit: "P", events: ["deliver", "judge-stale-digest"] }], + [{ unit: "P", events: ["deliver", "judge-accept", "redeliver"] }], + [{ unit: "P", events: ["deliver", "revision-drift", "judge-accept"] }], + [{ unit: "P", events: ["deliver", "cancel", "judge-accept"] }], + [{ unit: "P", events: ["deliver", "judge-reject"] }], + [{ unit: "P", events: ["deliver", "judge-undecidable"] }], + [{ unit: "P", events: ["claim", "deliver", "judge-accept"] }], + [{ unit: "P", events: ["deliver", "external-ready"] }], + [ + { unit: "P", events: ["deliver", "judge-reject"] }, + { unit: "D", events: ["deliver", "judge-accept"] }, + ], + [ + { unit: "P", events: ["deliver", "judge-accept"] }, + { unit: "D", events: ["deliver", "judge-accept"] }, + ], + [ + { unit: "P", events: ["deliver", "judge-accept", "redeliver"] }, + { unit: "D", events: ["deliver", "judge-accept"] }, + ], + [ + { unit: "P", events: ["deliver", "judge-accept"] }, + { unit: "D", events: ["deliver", "judge-stale-digest"] }, + ], + [ + { unit: "P", events: ["deliver", "cancel"] }, + { unit: "D", events: ["deliver", "judge-accept"] }, + ], + [ + { unit: "P", events: ["deliver"] }, + { unit: "D", events: ["deliver", "judge-accept"] }, + ], +]; diff --git a/src/integration/task-semantics-model.ts b/src/integration/task-semantics-model.ts index 474cbfd8..9550372e 100644 --- a/src/integration/task-semantics-model.ts +++ b/src/integration/task-semantics-model.ts @@ -60,19 +60,21 @@ function permutations(ids: readonly string[]): string[][] { } /** Dependency closure: a unit is accepted only if it was accepted itself and none of - * the units it read from was rejected or undecidable. */ -function acceptedClosure( + * the units it read from was rejected or undecidable. It takes the unit-level predicate + * rather than a table of declared outcomes, because the closure is asked about two + * different authorities - the model's declared outcomes and a run's recorded facts - + * and both must resolve "accepted" through the one acceptance predicate. */ +export function acceptedClosure( units: readonly TaskUnit[], - outcomes: ModelInput["outcomes"], + own: (unit: TaskUnit) => boolean, ): Set { const accepted = new Set(); - const own = (id: string) => (outcomes[id] ?? "undecidable") === "accepted"; let changed = true; while (changed) { changed = false; for (const unit of units) { const ok = - own(unit.id) && unit.inputs.dependencies.every((dependency) => accepted.has(dependency)); + own(unit) && unit.inputs.dependencies.every((dependency) => accepted.has(dependency)); if (ok && !accepted.has(unit.id)) { accepted.add(unit.id); changed = true; @@ -116,7 +118,12 @@ function run( waits, rollbacks, retries, - accepted: [...acceptedClosure(units, input.outcomes)].sort(), + accepted: [ + ...acceptedClosure( + units, + (unit) => (input.outcomes[unit.id] ?? "undecidable") === "accepted", + ), + ].sort(), }; } diff --git a/src/integration/task-semantics.ts b/src/integration/task-semantics.ts index 801b2c39..c28c2943 100644 --- a/src/integration/task-semantics.ts +++ b/src/integration/task-semantics.ts @@ -3,7 +3,7 @@ * exist (PatchTaskSpec, FrozenPatchTask, BoardTicket, cycle requirements, board * verdicts). It adds no schema, opens no database, calls no model, and publishes * nothing — see docs/design/task-unit-semantics.md and the proposed decision - * record docs/decisions/proposed/2026-09-13-task-unit-semantics.md. + * record docs/decisions/implemented/2026-09-13-task-unit-semantics.md. * * Two rules from that design are enforced here rather than left to convention: * @@ -17,7 +17,7 @@ * kind gets a refusal naming the task and the field. */ import { createHash } from "node:crypto"; -import { nextTask, type DispatchTask } from "./ooo-execution.ts"; +import { startableTasks, type DispatchTask } from "./ooo-execution.ts"; import { CONCLUSION_KINDS, MAX_PATCH_BUDGET, @@ -28,7 +28,13 @@ import { type PatchLimits, } from "./ooo-patch.ts"; import type { PatchTaskSpec, ProbePlan, ProbeOperation } from "./ooo-board.ts"; -import type { Requirement } from "./ooo-cycle.ts"; + +/** A declared precondition on a dependency's accepted artifact. The host evaluates + * these mechanically, so a downstream pushback is a checkable fact and not a mood. */ +export type Requirement = + | { kind: "verified"; task: string } + | { kind: "mutant-killed"; task: string; id: string } + | { kind: "test-title"; task: string; token: string }; /** The obligations a legal unit must satisfy (design §最小合法任务单元). */ export const OBLIGATIONS = [ @@ -444,6 +450,15 @@ export interface RecordedFacts { cancellations?: readonly string[]; /** External waits that have actually become ready. */ externalReady?: readonly string[]; + /** Tasks with a live lease. A claim is a recorded fact about the run, not a property of the + * artifact: the store owns it (claim owner, lease, attempt) and eligibility has to see it, or a + * task already being worked on becomes selectable again. */ + claimed?: readonly string[]; + /** The revision each unit's input was declared at, as the store recorded it. A unit carries its + * own declared revision, but "is this input still current" is a fact about the run: the store's + * source revision is the identity of the plan the work was admitted under, and it is not the + * same string as a task's declared revision. */ + sourceRevisions?: Readonly>; } /** The facts a run records about one artifact. Delivery and acceptance are different @@ -484,7 +499,13 @@ export function isAccepted(unit: Pick, facts: Recor digest: artifact, verdict: recorded?.verdict ?? null, judgedDigest: recorded?.digest ?? null, - currentRevision: recordedRevision === undefined || recordedRevision === unit.revision, + // The revision comparison is between two facts the store recorded: the revision the input was + // admitted under and the one observed since. A unit's own declared revision is the compiler's + // view of the plan and stands in only when no source revision was recorded - comparing a plan's + // declared revision against an observed one asks two different questions. + currentRevision: + recordedRevision === undefined || + recordedRevision === (facts.sourceRevisions?.[unit.id] ?? unit.revision), cancelled: facts.cancellations?.includes(unit.id) ?? false, }); } @@ -493,21 +514,27 @@ export function dispatchTasks(units: readonly TaskUnit[], facts: RecordedFacts): return units.map((unit) => ({ id: unit.id, effect: unit.effects.effect, - sourceVersion: unit.revision, + sourceVersion: facts.sourceRevisions?.[unit.id] ?? unit.revision, observedVersion: facts.revisions?.[unit.id] ?? unit.revision, dependencies: [...unit.inputs.dependencies], accepted: isAccepted(unit, facts), - claimed: false, + claimed: facts.claimed?.includes(unit.id) ?? false, externalEvent: unit.waitEvent ?? undefined, externalReady: facts.externalReady?.includes(unit.id) ?? false, + delivered: facts.artifacts?.[unit.id] !== undefined, + // The same recorded fact the acceptance predicate reads, carried to the eligibility rule: a + // cancellation is not a property of the artifact, and both rules have to see it. + cancelled: facts.cancellations?.includes(unit.id) ?? false, })); } -/** Status and dependency release share the derived dispatch state and the existing - * `nextTask` eligibility rule; neither gets its own notion of "accepted". */ +/** Status and dependency release share the derived dispatch state and the existing eligibility rule; + * neither gets its own notion of "accepted". `ready` is what a run may start now: one task at the + * default budget, and the run's remaining claim budget's worth when it declared more. */ export function deriveStatus( units: readonly TaskUnit[], facts: RecordedFacts, + slots = 1, ): { ready: readonly string[]; blocked: readonly { id: string; waitingFor: readonly string[] }[]; @@ -515,15 +542,15 @@ export function deriveStatus( } { const accepted = units.filter((unit) => isAccepted(unit, facts)).map((unit) => unit.id); const acceptedIds = new Set(accepted); - const candidate = nextTask(dispatchTasks(units, facts)); + const ready = startableTasks(dispatchTasks(units, facts), slots); const blocked = units .filter((unit) => !isAccepted(unit, facts)) .map((unit) => ({ id: unit.id, waitingFor: unit.inputs.dependencies.filter((dependency) => !acceptedIds.has(dependency)), })) - .filter((entry) => entry.waitingFor.length > 0 || entry.id !== candidate); - return { ready: candidate ? [candidate] : [], blocked, accepted }; + .filter((entry) => entry.waitingFor.length > 0 || !ready.includes(entry.id)); + return { ready, blocked, accepted }; } /** Actions this slice can derive. Fusion and speculative preparation are named but diff --git a/src/rcp/providers.ts b/src/rcp/providers.ts index bfa30338..333b0237 100644 --- a/src/rcp/providers.ts +++ b/src/rcp/providers.ts @@ -397,6 +397,32 @@ function resolveRouteTestInputs( return { testFiles }; } +/** The TAP summary the rule reads, for the reason only. The verdict stays in `testOutputPassed`, + * which the trusted verifier owns; this exists so a rejection names the count that rejected it. + * "route tests did not pass the TAP acceptance rule" alone sent readers to the wrong file: a run + * whose only defect was one declared skip reads like a broken test. */ +function tapCounts(output: string): string { + const count = (name: string): number => { + const matches = [...output.matchAll(new RegExp(`^# ${name} (\\d+)\\r?$`, "gm"))]; + return matches.length === 1 ? Number(matches[0]![1]) : Number.NaN; + }; + if (Number.isNaN(count("tests"))) return "no TAP summary in the output"; + return (["tests", "pass", "fail", "cancelled", "skipped", "todo"] as const) + .map((name) => `${name} ${count(name)}`) + .join(", "); +} + +/** The skipped and todo cases the rule rejects. A green summary hides them, and they are usually + * not the change's fault - an optional dependency that is not installed skips by design - so the + * reason names them instead of leaving the reader to rerun the group and search. */ +function tapSkippedCases(output: string): string[] { + return output + .split("\n") + .filter((line) => /^ok \d+ - .* # (SKIP|TODO)( |$)/u.test(line)) + .map((line) => line.replace(/^ok \d+ - /u, "")) + .slice(0, 2); +} + function routeTestCheckResult( name: string, routeId: string, @@ -408,11 +434,15 @@ function routeTestCheckResult( const exitFailed = Boolean(result.error || result.signal || result.status !== 0); // Same acceptance rule as the trusted baseline: TAP must report tests > 0, // pass == tests, and no fail/cancelled/skipped/todo. A run that executed - // nothing, or only skipped tests, is not a pass. + // nothing, or only skipped tests, is not a pass. The rule is deliberate and + // stays as it is; only the reason it reports is made readable here. const tapFailed = !exitFailed && !testOutputPassed(stdout); const failed = exitFailed || tapFailed; + const skipped = tapSkippedCases(stdout); const reason = tapFailed - ? `route tests did not pass the TAP acceptance rule: ${routeId}` + ? `route tests did not pass the TAP acceptance rule: ${routeId} (${tapCounts(stdout)}${ + skipped.length > 0 ? `; rejected on: ${skipped.join(" | ")}` : "" + })` : testFailureReason(result, exitFailed); return { name, diff --git a/tests/cli/commands.test.ts b/tests/cli/commands.test.ts index add1f286..c0d387bd 100644 --- a/tests/cli/commands.test.ts +++ b/tests/cli/commands.test.ts @@ -367,6 +367,33 @@ test("CLI board put exposes directed delivery", () => { ); }); +test("CLI exposes the run surface's operator half", () => { + const status = NMG_CLI_COMMANDS.find((spec) => spec.words.join(" ") === "run status")!; + assert.equal(status.method, "taskRun"); + assert.deepEqual( + status.buildParams({ flags: new Set(), options: new Map(), positionals: ["run-1"] }), + { action: "status", runId: "run-1" }, + ); + const cancel = NMG_CLI_COMMANDS.find((spec) => spec.words.join(" ") === "run cancel")!; + assert.equal(cancel.method, "taskRun"); + assert.deepEqual( + cancel.buildParams({ + flags: new Set(), + options: new Map([ + ["task", ["P"]], + ["reason", ["budget"]], + ]), + positionals: ["run-1"], + }), + { action: "cancel", runId: "run-1", taskId: "P", reason: "budget" }, + ); + // Cancelling the whole run omits the task, which is what makes the fact a run-level one. + assert.deepEqual( + cancel.buildParams({ flags: new Set(), options: new Map(), positionals: ["run-1"] }), + { action: "cancel", runId: "run-1" }, + ); +}); + test("CLI board discover exposes the system-layer agent roster", () => { const command = NMG_CLI_COMMANDS.find((spec) => spec.words.join(" ") === "board discover")!; assert.deepEqual( diff --git a/tests/cli/service-drain.test.ts b/tests/cli/service-drain.test.ts new file mode 100644 index 00000000..899983e0 --- /dev/null +++ b/tests/cli/service-drain.test.ts @@ -0,0 +1,55 @@ +/** + * The design's D8: shutdown is a sequence, and the middle step needs an await. + * + * Stop new work is a synchronous decision and close() already made it. Letting the work in flight + * finish is not, which is why it is a separate call: a close() that returned while calls it had + * already accepted were still running would report a clean shutdown over writes that were answered + * as accepted. So this asserts both halves - the drain waits, and the close refuses to skip it. + */ +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { NmgService } from "../../src/cli/service.ts"; +import { removeTempDirectory } from "../helpers/temp-directory.ts"; +import { stripProviderEnv } from "../helpers/test-env.ts"; + +stripProviderEnv(); + +test("drain lets the calls already accepted finish, and close refuses to skip that", async () => { + const directory = mkdtempSync(join(tmpdir(), "nmg-cli-drain-")); + const service = new NmgService({ databasePath: join(directory, "nmg.sqlite"), environment: {} }); + try { + // Idle: there is nothing to wait for, and that is not the same as timing out. + await service.drain(); + assert.equal(service.inFlight, 0); + + // A call that has been accepted and not yet answered. `search` opens the store on first use, so + // at this line it is running rather than already finished - which is the whole point: the + // counter has to see work that a synchronous close() could not. + const call = service.invoke("search", { query: "anything", limit: 1 }); + assert.equal(service.inFlight, 1, "an accepted call is counted until it is answered"); + assert.throws( + () => service.close(), + /drain before closing/u, + "closing over an accepted call is refused rather than reported as a clean shutdown", + ); + await assert.rejects( + () => service.drain(0), + /still in flight/u, + "a drain that cannot finish says so instead of returning quietly", + ); + await call.catch(() => undefined); + assert.equal(service.inFlight, 0, "and it stops being counted once it is answered"); + + // Close stays one-way, and draining an already-closed service is not an error. + service.close(); + await service.drain(); + await assert.rejects(() => service.invoke("hello"), /takes no new work/u); + } finally { + service.close(); + removeTempDirectory(directory); + } +}); diff --git a/tests/cli/task-run-surface.test.ts b/tests/cli/task-run-surface.test.ts new file mode 100644 index 00000000..4c525782 --- /dev/null +++ b/tests/cli/task-run-surface.test.ts @@ -0,0 +1,423 @@ +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { NmgService } from "../../src/cli/service.ts"; +import { removeTempDirectory } from "../helpers/temp-directory.ts"; +import { stripProviderEnv } from "../helpers/test-env.ts"; + +// In-process NmgService inherits process.env; keep recall lexical (test-env.ts). +stripProviderEnv(); + +/** One daemon and one temp database per case. The run surface is what another process reaches, so + * every case here goes through `service.invoke`, never through the coordinator directly. */ +function withService( + body: (service: NmgService, databasePath: string) => Promise, +): () => Promise { + return async () => { + const directory = mkdtempSync(join(tmpdir(), "nmg-task-run-")); + const databasePath = join(directory, "nmg.sqlite"); + const service = new NmgService({ databasePath, environment: {} }); + try { + await body(service, databasePath); + } finally { + service.close(); + removeTempDirectory(directory); + } + }; +} + +const CHANNEL = "round-1"; +const RUN = "run-1"; + +async function registerAndFreeze(service: NmgService): Promise { + await service.invoke("taskRun", { + action: "register", + runId: RUN, + planDigest: "sha256:plan", + policy: "ordered", + revision: "r1", + retention: "evidence", + }); + await service.invoke("taskRun", { + action: "freeze", + runId: RUN, + tasks: [ + { + taskId: "P", + revision: "r1", + input: "prepare the artifact", + dependencies: [], + effect: "artifact", + operation: "prepare", + }, + { + taskId: "J", + revision: "r1", + input: "judge the artifact", + dependencies: ["P"], + effect: "verdict", + operation: "judge", + }, + ], + }); +} + +/** Put one entry on the round's channel, adopting it into `taskId` in the same transition. */ +async function adopt( + service: NmgService, + taskId: string, + attempt?: number, +): Promise<{ entryId: string; taskId: string }> { + const written = await service.invoke("taskBoard", { + action: "put", + taskId: CHANNEL, + agentId: "agent-a", + sourceSessionId: "session-a", + kind: "handoff", + content: `carry ${taskId}`, + adopt: { runId: RUN, taskId, attempt }, + }); + if (written.action !== "put") throw new Error("expected a put result"); + return { entryId: written.entry.id, taskId: written.entry.taskId }; +} + +async function status(service: NmgService) { + const result = await service.invoke("taskRun", { action: "status", runId: RUN }); + if (result.action !== "status") throw new Error("expected a status result"); + return result.status; +} + +test( + "a run registers, freezes a plan, adopts entries, and reads it all back", + withService(async (service) => { + // The run surface is what another process reaches, so the daemon has to advertise it: a client + // that wants to adopt an entry gates that field on the method being there (an earlier daemon in + // the same epoch would ignore `adopt` and create an unmanaged entry). + const hello = await service.invoke("hello"); + assert.ok(hello.methods.includes("taskRun")); + await registerAndFreeze(service); + const written = await service.invoke("taskBoard", { + action: "put", + taskId: CHANNEL, + agentId: "agent-a", + sourceSessionId: "session-a", + kind: "handoff", + content: "carry J", + adopt: { runId: RUN, taskId: "J" }, + }); + if (written.action !== "put") throw new Error("expected a put result"); + assert.deepEqual(written.bound && { ...written.bound, sequence: undefined }, { + sequence: undefined, + recorded: true, + }); + + const view = await status(service); + assert.equal(view.manifest?.planDigest, "sha256:plan"); + assert.deepEqual( + view.tasks.map((task) => [task.taskId, task.position, task.dependencies]), + [ + ["P", 0, []], + ["J", 1, ["P"]], + ], + ); + assert.deepEqual(view.bindings, [ + { + taskId: "J", + attempt: 1, + entryId: written.entry.id, + sequence: view.bindings[0]!.sequence, + entry: { taskId: CHANNEL, status: "open", claimedBy: null, ackedBy: [] }, + }, + ]); + assert.deepEqual( + view.facts.map((fact) => fact.kind), + ["entry-bound"], + ); + }), +); + +test( + "adoption is part of the transition that creates the entry, so a refusal leaves no entry", + withService(async (service) => { + await registerAndFreeze(service); + await assert.rejects( + service.invoke("taskBoard", { + action: "put", + taskId: CHANNEL, + agentId: "agent-a", + kind: "handoff", + content: "carry a task the run never froze", + adopt: { runId: RUN, taskId: "T9" }, + }), + /never froze task T9/, + ); + const read = await service.invoke("taskBoard", { + action: "read", + taskId: CHANNEL, + agentId: "agent-b", + }); + if (read.action !== "read") throw new Error("expected a read result"); + assert.deepEqual(read.entries, []); + }), +); + +test( + "a managed entry's lifecycle write goes through the run, and the run records it", + withService(async (service) => { + await registerAndFreeze(service); + const { entryId } = await adopt(service, "J"); + const claimed = await service.invoke("taskBoard", { + action: "claim", + taskId: CHANNEL, + agentId: "agent-b", + entryId, + leaseSeconds: 600, + }); + if (claimed.action !== "claim") throw new Error("expected a claim result"); + const view = await status(service); + assert.deepEqual( + view.facts.map((fact) => fact.kind), + ["entry-bound", "board-claim"], + ); + assert.equal(view.bindings[0]!.entry?.status, "open"); + assert.equal(view.bindings[0]!.entry?.claimedBy, "agent-b"); + }), +); + +test( + "cancelling a run is recorded once, stops its managed writes, and is readable", + withService(async (service) => { + await registerAndFreeze(service); + const { entryId } = await adopt(service, "J"); + const cancelled = await service.invoke("taskRun", { + action: "cancel", + runId: RUN, + reason: "budget", + }); + if (cancelled.action !== "cancel") throw new Error("expected a cancel result"); + assert.equal(cancelled.recorded, true); + const again = await service.invoke("taskRun", { action: "cancel", runId: RUN }); + if (again.action !== "cancel") throw new Error("expected a cancel result"); + assert.equal(again.recorded, false); + + const view = await status(service); + const cancellation = view.facts.find((fact) => fact.kind === "run-cancelled"); + assert.equal(cancellation?.taskId, ""); + assert.equal(cancellation?.payload, JSON.stringify({ reason: "budget" })); + + await assert.rejects( + service.invoke("taskBoard", { + action: "claim", + taskId: CHANNEL, + agentId: "agent-b", + entryId, + leaseSeconds: 600, + }), + /was cancelled at sequence \d+; its managed entries take no further lifecycle writes/, + ); + }), +); + +test( + "cancelling one task names it, and a task the plan never froze cannot be cancelled", + withService(async (service) => { + await registerAndFreeze(service); + const one = await service.invoke("taskRun", { action: "cancel", runId: RUN, taskId: "P" }); + if (one.action !== "cancel") throw new Error("expected a cancel result"); + assert.equal(one.recorded, true); + const view = await status(service); + assert.equal(view.facts.find((fact) => fact.kind === "run-cancelled")?.taskId, "P"); + await assert.rejects( + service.invoke("taskRun", { action: "cancel", runId: RUN, taskId: "T9" }), + /never froze task T9; there is nothing to cancel/, + ); + }), +); + +test( + "a freeze cannot dangle, repeat a task, or lean on itself", + withService(async (service) => { + await service.invoke("taskRun", { + action: "register", + runId: RUN, + planDigest: "sha256:plan", + policy: "ordered", + revision: "r1", + retention: "evidence", + }); + const task = (taskId: string, dependencies: string[]) => ({ + taskId, + revision: "r1", + input: taskId, + dependencies, + effect: "artifact", + }); + await assert.rejects( + service.invoke("taskRun", { action: "freeze", runId: RUN, tasks: [task("P", ["T9"])] }), + /task P depends on T9, which this run's plan does not freeze/, + ); + await assert.rejects( + service.invoke("taskRun", { action: "freeze", runId: RUN, tasks: [task("P", ["P"])] }), + /task P depends on itself/, + ); + await assert.rejects( + service.invoke("taskRun", { + action: "freeze", + runId: RUN, + tasks: [task("P", []), task("P", [])], + }), + /one freeze cannot name the same task twice/, + ); + // A plan that froze nothing is the state the refusals left behind, so the run still has no plan. + const view = await status(service); + assert.deepEqual(view.tasks, []); + }), +); + +test( + "a refused freeze leaves the plan exactly as it was", + withService(async (service) => { + await registerAndFreeze(service); + // The store refuses to replace a frozen task with different content, and that refusal happens + // while the batch is being applied: Q must not survive as a half-frozen plan. + await assert.rejects( + service.invoke("taskRun", { + action: "freeze", + runId: RUN, + tasks: [ + { + taskId: "Q", + revision: "r1", + input: "later work", + dependencies: ["P"], + effect: "artifact", + }, + { + taskId: "P", + revision: "r1", + input: "a different input", + dependencies: [], + effect: "artifact", + }, + ], + }), + /already froze task P with a different definition/, + ); + const view = await status(service); + assert.deepEqual( + view.tasks.map((task) => task.taskId), + ["P", "J"], + ); + }), +); + +test( + "a cancelled run takes no further plan", + withService(async (service) => { + await registerAndFreeze(service); + await service.invoke("taskRun", { action: "cancel", runId: RUN }); + await assert.rejects( + service.invoke("taskRun", { + action: "freeze", + runId: RUN, + tasks: [ + { + taskId: "Q", + revision: "r1", + input: "later work", + dependencies: ["P"], + effect: "artifact", + }, + ], + }), + /was cancelled at sequence \d+; its managed entries take no further lifecycle writes/, + ); + const view = await status(service); + assert.deepEqual( + view.tasks.map((task) => task.taskId), + ["P", "J"], + ); + }), +); + +test( + "status is a read: an unknown run has no manifest and is not registered by being asked", + withService(async (service) => { + const view = await status(service); + assert.equal(view.manifest, null); + assert.deepEqual(view.tasks, []); + assert.deepEqual(view.facts, []); + // Still unregistered afterwards: a plan freeze and a cancellation both refuse by name. + await assert.rejects( + service.invoke("taskRun", { + action: "freeze", + runId: RUN, + tasks: [{ taskId: "P", revision: "r1", input: "i", dependencies: [], effect: "e" }], + }), + /run run-1 is not registered/, + ); + await assert.rejects( + service.invoke("taskRun", { action: "cancel", runId: RUN }), + /run run-1 is not registered; there is nothing to cancel/, + ); + }), +); + +test( + "a retry is a new attempt, not a rebinding of the one that came before", + withService(async (service) => { + await registerAndFreeze(service); + const first = await adopt(service, "J", 1); + const second = await adopt(service, "J", 2); + const view = await status(service); + assert.deepEqual( + view.bindings.map((binding) => [binding.attempt, binding.entryId]), + [ + [1, first.entryId], + [2, second.entryId], + ], + ); + // Re-pointing attempt 1 at the newer entry is refused rather than silently recorded. + await assert.rejects( + service.invoke("taskRun", { + action: "bind", + runId: RUN, + taskId: "J", + boardTaskId: CHANNEL, + entryId: second.entryId, + attempt: 1, + }), + /already carries entry .*; another entry is another attempt, not a rebinding/, + ); + }), +); + +test( + "the surface refuses a run it cannot name and a task the plan does not hold", + withService(async (service) => { + await assert.rejects( + service.invoke("taskRun", { + action: "bind", + runId: RUN, + taskId: "P", + boardTaskId: CHANNEL, + entryId: "1_1", + }), + /run run-1 is not registered/, + ); + await registerAndFreeze(service); + await assert.rejects( + service.invoke("taskRun", { + action: "bind", + runId: RUN, + taskId: "T9", + boardTaskId: CHANNEL, + entryId: "1_1", + }), + /never froze task T9; there is no task to bind an entry to/, + ); + }), +); diff --git a/tests/core/store-readonly-open.test.ts b/tests/core/store-readonly-open.test.ts new file mode 100644 index 00000000..3a7fd8df --- /dev/null +++ b/tests/core/store-readonly-open.test.ts @@ -0,0 +1,72 @@ +// The read-only factory is a way to look at a store that already exists: it does not create the file, +// does not migrate it, and cannot write to it. The offline host that views a finished round's private +// database is the caller this exists for. +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import test from "node:test"; +import { NmgStoreBase } from "../../src/core/store/base.ts"; + +const hash = (path: string) => createHash("sha256").update(readFileSync(path)).digest("hex"); +const noop = { chainId: "no-such-chain", memoryId: "no-such-memory" }; + +test("a read-only open neither creates, migrates nor writes", () => { + const dir = mkdtempSync(join(tmpdir(), "nmg-readonly-")); + test.after(() => rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 })); + + // A missing store is refused by name, and the refusal does not create it. + const missing = join(dir, "missing.sqlite"); + assert.throws( + () => new NmgStoreBase(missing, undefined, { readOnly: true }), + /does not exist/u, + "a read-only open of a missing store is refused, not created", + ); + assert.equal(existsSync(missing), false, "the refusal created the file it was asked to read"); + + // A real store, written and closed by its owner. + const database = join(dir, "store.sqlite"); + const owner = new NmgStoreBase(database); + assert.equal(owner.removeMemoryFromChain(noop), false, "the owner can write"); + owner.close(); + const before = hash(database); + + // The view reads the same file and cannot write it: the refusal comes from the handle itself, so + // setting query_only is not what makes this safe. + const view = new NmgStoreBase(database, undefined, { readOnly: true }); + assert.throws( + () => view.removeMemoryFromChain(noop), + /readonly|read-only|attempt to write|not authorized/iu, + "a write through the read-only handle is refused", + ); + view.close(); + assert.equal( + hash(database), + before, + "the read-only view changed the database, or checkpointed it", + ); + + // Releasing the view leaves the owner able to write: read-only is a capability of one connection. + const again = new NmgStoreBase(database); + assert.equal(again.removeMemoryFromChain(noop), false, "the owner can still write"); + again.close(); + + // An existing file with no recognisable schema is not an empty store: it is refused, and migrating + // it is exactly what the refusal must not do. + const empty = join(dir, "empty.sqlite"); + new DatabaseSync(empty).close(); + assert.throws( + () => new NmgStoreBase(empty, undefined, { readOnly: true }), + /no recognisable schema/u, + "a schemaless file is refused rather than read as an empty run", + ); + const probe = new DatabaseSync(empty, { readOnly: true }); + assert.equal( + (probe.prepare("SELECT COUNT(*) AS c FROM sqlite_master").get() as { c: number }).c, + 0, + "the refusal migrated the file", + ); + probe.close(); +}); diff --git a/tests/core/store-transaction-port.test.ts b/tests/core/store-transaction-port.test.ts new file mode 100644 index 00000000..c5ff6a03 --- /dev/null +++ b/tests/core/store-transaction-port.test.ts @@ -0,0 +1,193 @@ +/** + * The store owns the transaction boundary (docs/design/task-unit-semantics.md, "事务参与与连接生命周期"). + * A round does not open its own transaction: it joins one synchronous state transition through the + * port the store issued for it. These tests are about the refusals, because that is where the + * contract can be broken silently: a second BEGIN, a port from somewhere else, a port used after + * its callback returned, a callback that keeps the transaction open across an await, and a failure + * the caller swallows while the transaction is already half written. + */ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import test, { type TestContext } from "node:test"; + +import { NmgStore } from "../../src/core/store.ts"; +import type { TransactionPort } from "../../src/core/store/base.ts"; + +function withStore(t: TestContext, run: (store: NmgStore) => void): void { + const dir = mkdtempSync(join(tmpdir(), "nmg-port-")); + const store = new NmgStore(join(dir, "nmg.sqlite")); + t.after(() => { + try { + store.close(); + } catch { + // One test closes the connection itself, to make ROLLBACK fail. That is its subject. + } + rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + }); + run(store); +} + +const entry = (content: string) => ({ + taskId: "port-channel", + agentId: "worker-1", + kind: "note" as const, + content, + expiresAt: new Date(Date.now() + 86_400_000).toISOString(), +}); + +const contents = (store: NmgStore): string[] => + store + .readTaskBoard({ taskId: "port-channel", includeResolved: true }) + .entries.map((row) => row.content) + .sort(); + +test("the store commits what a transition wrote and returns its value", (t) => { + withStore(t, (store) => { + const returned = store.writeTransaction((port) => { + const row = store.putTaskBoardEntry(entry("written through the port"), port); + return row.id; + }); + assert.equal(typeof returned, "string"); + assert.deepEqual(contents(store), ["written through the port"]); + }); +}); + +test("a write entry reached inside a transition without a port is refused, not nested", (t) => { + withStore(t, (store) => { + assert.throws( + () => + store.writeTransaction(() => { + // This is the mistake the contract exists for: the same call that works on its own + // would otherwise run a second BEGIN inside the first. + store.putTaskBoardEntry(entry("nested")); + }), + /already open: join it with the port it issued/u, + ); + assert.deepEqual(contents(store), [], "the refused write left nothing behind"); + }); +}); + +test("a transition may not open another transition of its own", (t) => { + withStore(t, (store) => { + assert.throws( + () => store.writeTransaction(() => store.writeTransaction(() => 1)), + /already open/u, + ); + }); +}); + +test("a port from another store is refused", (t) => { + withStore(t, (first) => { + const dir = mkdtempSync(join(tmpdir(), "nmg-port-other-")); + const second = new NmgStore(join(dir, "nmg.sqlite")); + t.after(() => { + second.close(); + rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + }); + first.writeTransaction((port: TransactionPort) => { + assert.throws( + () => second.withPort(port, () => 1), + /not the store's live transaction scope/u, + ); + }); + }); +}); + +test("a port used after its callback returned is refused", (t) => { + withStore(t, (store) => { + let escaped: TransactionPort | null = null; + store.writeTransaction((port) => { + escaped = port; + return 1; + }); + assert.notEqual(escaped, null); + assert.throws( + () => store.withPort(escaped!, () => 1), + /not the store's live transaction scope/u, + ); + assert.throws( + () => store.putTaskBoardEntry(entry("late"), escaped!), + /not the store's live transaction scope/u, + ); + assert.deepEqual(contents(store), []); + }); +}); + +test("a callback that would hold the transaction open across an await is refused", (t) => { + withStore(t, (store) => { + assert.throws( + () => + store.writeTransaction(() => { + store.putTaskBoardEntry(entry("async"), undefined); + return Promise.resolve(1); + }), + /already open/u, + ); + assert.throws(() => store.writeTransaction(() => Promise.resolve(1)), /must be synchronous/u); + assert.deepEqual(contents(store), [], "nothing half written survives the refusal"); + }); +}); + +test("a failure the caller swallows still forbids the commit", (t) => { + withStore(t, (store) => { + assert.throws( + () => + store.writeTransaction((port) => { + store.putTaskBoardEntry(entry("first"), port); + try { + store.withPort(port, () => { + throw new Error("the second write failed"); + }); + } catch { + // The caller decides to carry on. The transaction does not: the first write already + // happened and the outermost is the only one that may decide to keep it. + } + return "carried on"; + }), + /rollback-only/u, + ); + assert.deepEqual(contents(store), [], "a caught failure does not become a commit"); + }); +}); + +test("a failed rollback quarantines the connection instead of pretending it is usable", (t) => { + withStore(t, (store) => { + const raw = store as unknown as { db: DatabaseSync }; + assert.throws( + () => + store.writeTransaction(() => { + raw.db.close(); + throw new Error("the connection is gone"); + }), + /the connection is gone/u, + ); + assert.throws( + () => store.writeTransaction(() => 1), + /quarantined/u, + "the store must not accept work on a connection whose state is unknown", + ); + }); +}); + +test("the store runs its transaction boundary in exactly one place", () => { + // A mechanical check rather than a reminder: a hand-rolled BEGIN reappearing anywhere in the + // store is what makes a second boundary possible, and no behavioural test notices a path that + // still works by accident. + const source = readFileSync(new URL("../../src/core/store/base.ts", import.meta.url), "utf8"); + const count = (needle: string) => source.split(needle).length - 1; + assert.equal(count("BEGIN IMMEDIATE"), 1, "one BEGIN, in writeTransaction"); + assert.equal(count('"COMMIT"'), 1, "one COMMIT, in writeTransaction"); + assert.equal(count('"ROLLBACK"'), 1, "one ROLLBACK, in writeTransaction"); + const owned = source.slice(source.indexOf("writeTransaction"), source.indexOf("withPort")); + assert.equal(owned.includes("BEGIN IMMEDIATE"), true, "the BEGIN belongs to writeTransaction"); + assert.equal(owned.includes('"COMMIT"'), true, "and so does the COMMIT"); + // The one ROLLBACK lives in the store's own helper, which only the boundary calls. + assert.equal( + owned.split("this.rollback()").length - 1, + count("this.rollback()"), + "every rollback call is one the boundary made", + ); +}); diff --git a/tests/core/store/current-value-window.test.ts b/tests/core/store/current-value-window.test.ts new file mode 100644 index 00000000..d2cf3bde --- /dev/null +++ b/tests/core/store/current-value-window.test.ts @@ -0,0 +1,130 @@ +/** + * The current-value window and its clock grace. + * + * A write stamps validity and expiry from JavaScript, a read compares against SQLite's `now`, and the + * two clock readers disagree at millisecond granularity. Measured on this machine, a row stamped + * `...T02:58:38.468Z` was read back while SQLite's `now` said `...38.467Z`, so `valid_from <= now` was + * false and a memory written a moment earlier read as "not active" - about one run in 1500, which is + * how a product suite failed intermittently under load. These cases pin the window: it is widened by a + * named grace on both boundaries and never narrowed, and a value dated well into the future is still + * excluded, because that is what the comparison is for. + * + * Every boundary below is stamped **in SQL**, so the stamp and the read share one clock source and the + * expectation does not depend on how long the test takes. + */ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import test from "node:test"; +import { + CLOCK_GRACE_MS, + clockNow, + currentlyValid, + notExpired, +} from "../../../src/core/store/clock.ts"; +import { NmgStore } from "../../../src/core/store.ts"; + +const half = (CLOCK_GRACE_MS / 2000).toFixed(3); + +/** A memory whose validity boundaries are stamped in SQL, then reopened through the store: the read + * then compares SQLite's clock against SQLite's clock. */ +function withStamped(set: string, run: (store: NmgStore, id: string) => void): void { + const directory = mkdtempSync(join(tmpdir(), "nmg-window-")); + const path = join(directory, "test.sqlite"); + const opened = new NmgStore(path); + const saved = opened.remember({ + statement: "the project name is Atlas", + nodeName: "project name", + memoryType: "fact", + sourceActor: "user", + }); + opened.close(); + const raw = new DatabaseSync(path); + raw.prepare(`UPDATE memory_records SET ${set} WHERE id = ?`).run(saved.memory.id); + raw.close(); + const store = new NmgStore(path); + try { + run(store, saved.memory.id); + } finally { + store.close(); + rmSync(directory, { force: true, recursive: true }); + } +} + +test("a value stamped a moment in the future is current, not missing", () => { + // The flake itself, made deterministic: the stamp is half a grace ahead of SQLite's own `now`, which + // is what a write's JavaScript timestamp looks like to a read that follows it by microseconds. + withStamped( + `valid_from = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '+${half} seconds')`, + (store, id) => { + assert.equal(store.demoteMemory(id, "no longer relevant").residence, "stg"); + }, + ); +}); + +test("a value dated well into the future is still not current", () => { + withStamped(`valid_from = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '+60 seconds')`, (store, id) => { + assert.throws(() => store.demoteMemory(id, "no longer relevant"), /is not active/); + }); +}); + +test("a value that expired a moment ago is still current", () => { + withStamped( + `expires_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-${half} seconds')`, + (store, id) => { + assert.equal(store.demoteMemory(id, "expired a clock tick ago").residence, "stg"); + }, + ); +}); + +test("a value that expired a minute ago is not current", () => { + withStamped(`expires_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-60 seconds')`, (store, id) => { + assert.throws(() => store.demoteMemory(id, "long expired"), /is not active/); + }); +}); + +test("a just-written memory is never read as not active", () => { + // The regression, at the rate the flake actually occurred: writes and reads of the same memory, in + // one store, with nothing between them. + const directory = mkdtempSync(join(tmpdir(), "nmg-window-loop-")); + const store = new NmgStore(join(directory, "test.sqlite")); + try { + for (let i = 0; i < 400; i += 1) { + const saved = store.remember({ + statement: `the project name is Atlas ${i}`, + nodeName: "project name", + memoryType: "fact", + sourceActor: "user", + }); + assert.equal( + store.demoteMemory(saved.memory.id, "no longer relevant").residence, + "stg", + `write ${i} could not be read back`, + ); + } + } finally { + store.close(); + rmSync(directory, { force: true, recursive: true }); + } +}); + +test("the window widens by the grace on both boundaries and never narrows", () => { + assert.ok(CLOCK_GRACE_MS > 0, "a grace of zero is the bug this window exists for"); + const later = clockNow("later"); + const earlier = clockNow("earlier"); + assert.match(currentlyValid("m"), /m\.valid_from <= strftime\(.*'\+/); + assert.ok( + currentlyValid("m").includes(later), + "valid_from is compared against the later instant", + ); + assert.ok( + currentlyValid("m").includes(earlier), + "valid_until is compared against the earlier one", + ); + assert.ok(notExpired("m").includes(earlier), "expiry is compared against the earlier one"); + // SQLite has no `milliseconds` modifier, and an unknown one makes the whole expression NULL, which + // silently excludes every row instead of failing loudly. + assert.doesNotMatch(later, /milliseconds/); +}); diff --git a/tests/core/store/schema.test.ts b/tests/core/store/schema.test.ts index 3296f291..d47ad443 100644 --- a/tests/core/store/schema.test.ts +++ b/tests/core/store/schema.test.ts @@ -42,6 +42,9 @@ test("migrate creates the core graph tables", () => { "embedding_index_state", "retrieval_traces", "task_board_entries", + "task_run_manifest", + "task_run_tasks", + "task_run_facts", ]) { assert.ok(tables.has(expected), `expected table ${expected}`); } diff --git a/tests/core/store/task-runs.test.ts b/tests/core/store/task-runs.test.ts new file mode 100644 index 00000000..dace6f35 --- /dev/null +++ b/tests/core/store/task-runs.test.ts @@ -0,0 +1,254 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { NmgStore } from "../../../src/core/store.ts"; + +/** Windows can still hold a handle to a just-closed store for a few milliseconds, so a plain + * recursive remove intermittently fails with EPERM on an otherwise green run. */ +const REMOVE_TEMP_TREE = { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }; +/** Far enough out that nothing here is ever pruned as expired. */ +const NEVER = new Date(Date.now() + 86_400_000).toISOString(); + +function withStore(run: (store: NmgStore) => void): void { + const directory = mkdtempSync(join(tmpdir(), "nmg-task-runs-")); + const store = new NmgStore(join(directory, "test.sqlite")); + try { + run(store); + } finally { + store.close(); + rmSync(directory, REMOVE_TEMP_TREE); + } +} + +function register( + store: NmgStore, + runId: string, + planDigest = "plan-a", + policy = "checks-a", +): void { + store.registerTaskRun({ runId, planDigest, policy, revision: "v1", retention: "keep:evidence" }); +} + +function freeze( + store: NmgStore, + runId: string, + taskId: string, + input = `${taskId} input`, + position = 0, +): void { + store.freezeTaskRunTask({ + runId, + taskId, + position, + revision: "v1", + input, + dependencies: [], + effect: "isolated-artifact", + }); +} + +function put(store: NmgStore, taskId: string, content: string): string { + return store.putTaskBoardEntry({ + taskId, + agentId: "host", + kind: "handoff", + content, + expiresAt: NEVER, + }).id; +} + +test("a run registers once, and a second plan for the same run is refused", () => { + withStore((store) => { + register(store, "run-1"); + // A retry after a lost response re-registers the same identity: a no-op, not a conflict. + register(store, "run-1"); + assert.equal(store.taskRunManifest("run-1")?.planDigest, "plan-a"); + + assert.throws( + () => register(store, "run-1", "plan-b"), + /already froze a different plan/, + "a run cannot be re-opened onto a different plan", + ); + assert.equal( + store.taskRunManifest("run-1")?.planDigest, + "plan-a", + "the refusal changed nothing", + ); + assert.equal(store.taskRunManifest("run-2"), null); + }); +}); + +test("freezing a task twice is a no-op, and a different definition for it is refused", () => { + withStore((store) => { + register(store, "run-1"); + freeze(store, "run-1", "T1"); + freeze(store, "run-1", "T1"); + assert.equal(store.taskRunTasks("run-1").length, 1); + + assert.throws( + () => freeze(store, "run-1", "T1", "a different input"), + /already froze task T1 with a different definition/, + ); + assert.throws( + () => freeze(store, "run-absent", "T1"), + /is not registered/, + "a task cannot be frozen into a run that was never registered", + ); + }); +}); + +test("appending the same fact twice records it once and keeps the first sequence", () => { + withStore((store) => { + register(store, "run-1"); + const first = store.appendTaskRunFact({ runId: "run-1", kind: "entry-bound", taskId: "T1" }); + assert.deepEqual(first, { sequence: 1, recorded: true }); + + // The retry carries the same identity (run, kind, task, attempt), so it is the same fact. + const retry = store.appendTaskRunFact({ runId: "run-1", kind: "entry-bound", taskId: "T1" }); + assert.deepEqual(retry, { sequence: 1, recorded: false }); + assert.equal(store.taskRunFacts("run-1").length, 1); + + // The next attempt of the same kind is a different fact, and gets the next sequence. + const second = store.appendTaskRunFact({ + runId: "run-1", + kind: "entry-bound", + taskId: "T1", + attempt: 1, + }); + assert.deepEqual(second, { sequence: 2, recorded: true }); + + // A run-level fact carries no task id, and does not collide with a task's fact of the same kind. + const runLevel = store.appendTaskRunFact({ runId: "run-1", kind: "entry-bound" }); + assert.deepEqual(runLevel, { sequence: 3, recorded: true }); + }); +}); + +test("the facts as of one sequence are a prefix of the log, not a filter over it", () => { + withStore((store) => { + register(store, "run-1"); + for (const kind of ["a", "b", "c"]) + store.appendTaskRunFact({ runId: "run-1", kind, taskId: "T1" }); + assert.deepEqual( + store.taskRunFacts("run-1", 2).map((fact) => fact.kind), + ["a", "b"], + ); + assert.deepEqual( + store.taskRunFacts("run-1").map((fact) => fact.sequence), + [1, 2, 3], + ); + }); +}); + +test("two runs in one store do not see each other's tasks or facts", () => { + withStore((store) => { + register(store, "run-1", "plan-a", "checks-a"); + register(store, "run-2", "plan-a", "checks-a"); + // The same task id in two runs is two frozen tasks, not one collision. + freeze(store, "run-1", "T1", "input of run 1"); + freeze(store, "run-2", "T1", "input of run 2"); + assert.equal(store.taskRunTasks("run-1")[0]?.input, "input of run 1"); + assert.equal(store.taskRunTasks("run-2")[0]?.input, "input of run 2"); + + const entryOne = store.appendTaskRunFact({ + runId: "run-1", + kind: "entry-bound", + taskId: "T1", + entryId: put(store, "board", "for run 1"), + }); + const entryTwo = store.appendTaskRunFact({ + runId: "run-2", + kind: "entry-bound", + taskId: "T1", + entryId: put(store, "board", "for run 2"), + }); + assert.deepEqual([entryOne.sequence, entryTwo.sequence], [1, 1], "each run counts its own"); + assert.equal(store.taskRunFacts("run-1").length, 1); + assert.equal(store.taskRunFacts("run-2").length, 1); + }); +}); + +test("a board write and a run fact land together, and neither lands alone", () => { + withStore((store) => { + register(store, "run-1"); + // One transition: the entry the board records and the binding the run records are written + // through the same port, so the store's boundary decides for both. + const entryId = store.writeTransaction((port) => { + const entry = store.putTaskBoardEntry( + { taskId: "board", agentId: "host", kind: "handoff", content: "managed", expiresAt: NEVER }, + port, + ); + store.appendTaskRunFact( + { runId: "run-1", kind: "entry-bound", taskId: "T1", entryId: entry.id }, + port, + ); + return entry.id; + }); + assert.equal(store.taskRunForEntry(entryId)?.runId, "run-1"); + assert.equal(store.taskRunFacts("run-1").length, 1); + + // Now the same shape with the fact failing: the run was never registered. The board row was + // already written inside this transaction, and it must not survive the failure with it. + assert.throws( + () => + store.writeTransaction((port) => { + const entry = store.putTaskBoardEntry( + { + taskId: "board", + agentId: "host", + kind: "handoff", + content: "orphan", + expiresAt: NEVER, + }, + port, + ); + store.appendTaskRunFact( + { runId: "run-absent", kind: "entry-bound", entryId: entry.id }, + port, + ); + }), + /is not registered/, + ); + assert.deepEqual( + store.readTaskBoard({ taskId: "board" }).entries.map((entry) => entry.content), + ["managed"], + "the verdict-side row of a failed transition is not left behind", + ); + assert.equal(store.taskRunFacts("run-absent").length, 0); + + // The store is still usable: one failed transition does not quarantine a connection that + // rolled back cleanly. + assert.deepEqual(store.appendTaskRunFact({ runId: "run-1", kind: "entry-bound", attempt: 1 }), { + sequence: 2, + recorded: true, + }); + }); +}); + +test("an entry bound by two runs is refused rather than answered with one of them", () => { + withStore((store) => { + register(store, "run-1"); + register(store, "run-2"); + const entryId = put(store, "board", "shared by mistake"); + store.appendTaskRunFact({ runId: "run-1", kind: "entry-bound", taskId: "T1", entryId }); + assert.equal(store.taskRunForEntry(entryId)?.taskId, "T1"); + store.appendTaskRunFact({ runId: "run-2", kind: "entry-bound", taskId: "T1", entryId }); + assert.throws(() => store.taskRunForEntry(entryId), /is bound by 2 runs/); + }); +}); + +test("reads create nothing: an unknown run stays unknown", () => { + withStore((store) => { + assert.equal(store.taskRunManifest("ghost"), null); + assert.deepEqual(store.taskRunTasks("ghost"), []); + assert.deepEqual(store.taskRunFacts("ghost"), []); + assert.equal(store.taskRunForEntry("no-such-entry"), null); + // If any read had registered the run as a side effect, this would not refuse. + assert.throws( + () => store.appendTaskRunFact({ runId: "ghost", kind: "x" }), + /is not registered/, + ); + }); +}); diff --git a/tests/extensions/nmg/index.test.ts b/tests/extensions/nmg/index.test.ts index b5995d89..ef647a8f 100644 --- a/tests/extensions/nmg/index.test.ts +++ b/tests/extensions/nmg/index.test.ts @@ -86,12 +86,12 @@ test("Pi adapter exposes stable memory tools plus the unified Lab capability ent try { assert.deepEqual( [...extensionHarness().tools.keys()], - ["nmg_lab", "ooo_round", "nmg_remember", "nmg_get", "nmg_search", "nmg_board"], + ["nmg_lab", "nmg_remember", "nmg_get", "nmg_search", "nmg_board"], ); process.env.NMG_ENABLE_COORDINATION = "off"; assert.deepEqual( [...extensionHarness().tools.keys()], - ["nmg_lab", "ooo_round", "nmg_remember", "nmg_get", "nmg_search"], + ["nmg_lab", "nmg_remember", "nmg_get", "nmg_search"], ); } finally { if (previous === undefined) delete process.env.NMG_ENABLE_LAB_TOOLS; @@ -113,7 +113,7 @@ test("Lab reasoning tool persists scratch state and injects it only after compac const { handlers, tools } = extensionHarness(); assert.deepEqual( [...tools.keys()], - ["nmg_reason", "nmg_lab", "ooo_round", "nmg_remember", "nmg_get", "nmg_search", "nmg_board"], + ["nmg_reason", "nmg_lab", "nmg_remember", "nmg_get", "nmg_search", "nmg_board"], ); const sessionManager = { getSessionId: () => "reasoning-session", diff --git a/tests/extensions/nmg/ooo-round.test.ts b/tests/extensions/nmg/ooo-round.test.ts deleted file mode 100644 index 6aa90a49..00000000 --- a/tests/extensions/nmg/ooo-round.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -// The Pi adaptation is a thin surface over the round entry point: its job is to bind parameters -// correctly and refuse what it cannot answer, so the tests are about that, not about running rounds. -import assert from "node:assert/strict"; -import { test } from "node:test"; - -import { - describeStart, - roundArgv, - type OooRoundParams, -} from "../../../.pi/extensions/nmg/ooo-round.ts"; - -test("every action refuses what it cannot answer, by name", () => { - const missingRunDir = () => roundArgv({ action: "status" } satisfies OooRoundParams); - assert.throws(missingRunDir, /runDir is required/); - assert.throws( - () => roundArgv({ action: "submit", runDir: "r" }), - /submit requires specPath/, - "a round without a spec would have to be invented", - ); - assert.throws( - () => roundArgv({ action: "cancel", runDir: "r" }), - /cancel requires reason/, - "a cancellation without a reason is not a decision that survives a restart", - ); -}); - -test("the arguments are the reviewed entry point's, action by action", () => { - assert.deepEqual(roundArgv({ action: "status", runDir: "run1" }), [ - "--experimental-strip-types", - "evals/ooo-execution/round-cli.ts", - "status", - "--run-dir", - "run1", - ]); - assert.deepEqual(roundArgv({ action: "submit", specPath: "s.json", runDir: "run1" }), [ - "--experimental-strip-types", - "evals/ooo-execution/round-cli.ts", - "submit", - "s.json", - "--run-dir", - "run1", - ]); - // `--live` is the switch that spends tokens, so it must only ever appear when asked for. - assert.ok( - !roundArgv({ action: "submit", specPath: "s.json", runDir: "run1" }).includes("--live"), - "a round without an explicit live flag must not call a provider", - ); - assert.ok( - roundArgv({ action: "submit", specPath: "s.json", runDir: "run1", live: true }).includes( - "--live", - ), - ); - assert.deepEqual(roundArgv({ action: "cancel", runDir: "run1", reason: "operator stopped it" }), [ - "--experimental-strip-types", - "evals/ooo-execution/round-cli.ts", - "cancel", - "--run-dir", - "run1", - "--reason", - "operator stopped it", - ]); -}); - -test("a started round is described by where it runs and what it will cost", () => { - const text = describeStart( - { action: "submit", specPath: "s.json", runDir: "run1" }, - { pid: 42, logPath: "run1/cli.log" }, - ); - assert.match(text, /detached \(pid 42\)/); - assert.match(text, /recorded answers \(no model calls\)/, "the default must be visible as free"); - assert.match( - describeStart( - { action: "submit", specPath: "s.json", runDir: "run1", live: true }, - { pid: 1, logPath: "l" }, - ), - /this spends tokens/, - ); -}); diff --git a/tests/integration/agent-surface.test.ts b/tests/integration/agent-surface.test.ts index 730fc631..59fd786e 100644 --- a/tests/integration/agent-surface.test.ts +++ b/tests/integration/agent-surface.test.ts @@ -150,7 +150,7 @@ test("a delivered and judged entry renders its deliverable and verdict, not just }, ], }, - { taskId: "ooo-process-probe" }, + { taskId: "ooo-probe:agent-surface" }, ); assert.match(rendered, /\[claimed by worker-7\]/u); diff --git a/tests/integration/ooo-acceptance-one-predicate.test.ts b/tests/integration/ooo-acceptance-one-predicate.test.ts new file mode 100644 index 00000000..2d39d83a --- /dev/null +++ b/tests/integration/ooo-acceptance-one-predicate.test.ts @@ -0,0 +1,115 @@ +import assert from "node:assert/strict"; +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; +import { acceptedFact, isAccepted, type TaskUnit } from "../../src/integration/task-semantics.ts"; + +const root = new URL("../../", import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, "$1"); +const read = (relative: string) => readFileSync(join(root, relative), "utf8"); + +function sourceFiles(directory: string, found: string[] = []): string[] { + for (const name of readdirSync(join(root, directory))) { + const relative = `${directory}/${name}`; + if (statSync(join(root, relative)).isDirectory()) sourceFiles(relative, found); + else if (name.endsWith(".ts")) found.push(relative); + } + return found; +} + +const count = (source: string, needle: string) => source.split(needle).length - 1; + +/** The rule the design states as "接受查询与依赖解锁必须调用同一谓词". A document + * cannot fail a build; a second copy of the rule can drift, so the rule is pinned + * here mechanically and the two callers are shown to agree case by case. + * + * These checks count occurrences instead of slicing a function body. A slice whose + * end boundary is not found silently runs to the end of the file, which once made + * this very test pass while the call site underneath it had been renamed. */ +test("acceptance has one home, and both readers reach it", () => { + const semantics = read("src/integration/task-semantics.ts"); + const board = read("src/integration/ooo-board.ts"); + + // One home: the predicate is defined once in the whole integration layer. + const definitions = sourceFiles("src").filter((file) => + read(file).includes("export function acceptedFact("), + ); + assert.deepEqual(definitions, ["src/integration/task-semantics.ts"]); + + // Both readers call it rather than carrying their own copy of the decision. + assert.equal(count(board, "acceptedFact({"), 1, "the board's read path must call the predicate"); + assert.equal(count(semantics, "acceptedFact({"), 1, "the derived view must call the predicate"); + + // The comparison the predicate owns is written down once, inside the predicate. + assert.equal(count(semantics, "judgedDigest === fact.digest"), 1, "one place decides acceptance"); + assert.equal(count(board, "judgedDigest === fact.digest"), 0, "ooo-board.ts decides on its own"); + assert.equal(count(board, "verdict ==="), 0, "ooo-board.ts decides on its own"); +}); + +/** Agreement over one facts snapshot: the status query (the board's read) and + * dependency release (the derived view) must not disagree about the same row. */ +test("the two readers agree over the same recorded facts", () => { + const unit: TaskUnit = { id: "T", revision: "rev-1" } as TaskUnit; + const cases = [ + { + name: "delivered and judged", + verdict: { verdict: "accepted", digest: "rev-1" }, + expected: true, + }, + { + name: "verdict about another digest", + verdict: { verdict: "accepted", digest: "rev-1-old" }, + expected: false, + }, + { name: "rejected", verdict: { verdict: "rejected", digest: "rev-1" }, expected: false }, + { name: "no verdict at all", verdict: undefined, expected: false }, + { + name: "undecidable is not acceptance", + verdict: { verdict: "undecidable", digest: "rev-1" }, + expected: false, + }, + ] as const; + + for (const item of cases) { + const facts = { + artifacts: { T: "rev-1" }, + revisions: { T: "rev-1" }, + verdicts: item.verdict ? { T: item.verdict } : {}, + }; + const derived = isAccepted(unit, facts); + const asked = acceptedFact({ + artifact: "rev-1", + digest: "rev-1", + verdict: item.verdict?.verdict ?? null, + judgedDigest: item.verdict?.digest ?? null, + currentRevision: true, + cancelled: false, + }); + assert.equal(derived, item.expected, `derived view: ${item.name}`); + assert.equal(asked, item.expected, `predicate: ${item.name}`); + } + + // The case the design calls out by name: bytes on disk are not acceptance. + const bytes = { artifacts: { T: "rev-1" }, revisions: { T: "rev-1" } }; + assert.equal(isAccepted(unit, bytes), false, "an artifact with no verdict is not accepted"); + + // Revision drift, and cancellation, withdraw acceptance on both paths. + assert.equal( + isAccepted(unit, { + artifacts: { T: "rev-1" }, + revisions: { T: "rev-2" }, + verdicts: { T: { verdict: "accepted", digest: "rev-1" } }, + }), + false, + "an artifact built from a superseded revision is not accepted", + ); + assert.equal( + isAccepted(unit, { + artifacts: { T: "rev-1" }, + revisions: { T: "rev-1" }, + verdicts: { T: { verdict: "accepted", digest: "rev-1" } }, + cancellations: ["T"], + }), + false, + "a cancelled run accepts nothing", + ); +}); diff --git a/tests/integration/ooo-advisers.test.ts b/tests/integration/ooo-advisers.test.ts new file mode 100644 index 00000000..d1853ba6 --- /dev/null +++ b/tests/integration/ooo-advisers.test.ts @@ -0,0 +1,327 @@ +/** + * The advice seam: what a suggestion may change, and what it may not. + * + * An optional HA or MGR source may rank inside the legal candidate set. It may not widen that set, + * unlock a dependency, cross a session or branch, replay a score whose inputs are gone, or start a + * speculative execution. Each case below is one of those refusals, and each is offline: a fixture + * source plays the adviser, so this proves the seam's rules rather than a model's behaviour. + * + * The two cases at the end are the wiring rather than the rules: an admission with no source answers + * exactly as the rule does, and one with a source can only reorder what the shared rules already + * made legal. + */ +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { + BoardAdmission, + type PatchTaskSpec, + type ProbePlan, +} from "../../src/integration/ooo-board.ts"; +import { nextTask, selectableTasks } from "../../src/integration/ooo-execution.ts"; +import { + orderCandidates, + revalidateSuggestion, + type AdviceProjection, + type Suggestion, + type SuggestionProvenance, + type SuggestionSource, +} from "../../src/integration/task-advisers.ts"; +import { compileTaskUnits, dispatchTasks } from "../../src/integration/task-semantics.ts"; + +const SCOPE = { + sessionId: "session-1", + branchId: "branch-1", + parametersVersion: "params-1", + projectionVersion: "projection-1", + observationOrder: ["projection-1"], + initialState: "initial-1", +} as const; + +/** What the projection says when the set holds two selectable tasks. */ +const projection = (legal: readonly string[]): Omit => ({ + ...SCOPE, + legal: [], + ready: [...legal], + accepted: [], + blocked: {}, +}); + +const provenance = (overrides: Partial = {}): SuggestionProvenance => ({ + sourceId: "fixture-adviser", + kind: "ha", + ...SCOPE, + ...overrides, +}); + +const suggest = ( + taskId: string, + score: number, + overrides: Partial = {}, +): Suggestion => ({ + action: "next", + taskId, + score, + provenance: provenance(), + ...overrides, +}); + +/** A source that answers with whatever the test hands it. */ +const source = ( + id: string, + suggestions: readonly Suggestion[], + overrides: Partial = {}, +): SuggestionSource => ({ + id, + kind: "ha", + suggest: () => suggestions, + ...overrides, +}); + +test("a suggestion outside the legal set is refused, however high it scores", () => { + const outcome = orderCandidates(["A", "B"], projection(["A", "B"]), [ + source("fixture-ha", [suggest("C", 1e9), suggest("B", 0.2)]), + ]); + assert.equal(outcome.refusals.length, 1); + assert.equal(outcome.refusals[0]!.task, "C"); + assert.match(outcome.refusals[0]!.reason, /outside the legal candidate set/u); + // The set is the input set: a high score buys a different order inside it, never a new member. + assert.deepEqual([...outcome.order].sort(), ["A", "B"]); + assert.equal(outcome.order[0], "B"); + assert.deepEqual( + outcome.adopted.map((entry) => entry.taskId), + ["B"], + ); +}); + +test("a soft premise cannot unlock a dependency the shared rules refused", () => { + const REV = "input-v1"; + const plan: ProbePlan = [ + ["B", REV, [], "isolated-artifact", null, null], + ["A", REV, ["B"], "isolated-artifact", null, null], + ]; + const units = compileTaskUnits({ plan, specs: {} }); + assert.ok(units.legal, "the fixture plan is legal"); + const dispatch = dispatchTasks(units.units, {}); + // A depends on B and no artifact has been accepted, so A is not a candidate at all. + assert.deepEqual(selectableTasks(dispatch), ["B"]); + + const outcome = orderCandidates(["B"], projection(["B"]), [ + source("fixture-mgr", [ + // The premise is the MGR shape the design warns about: a soft gate or a hypothesis that reads + // as "the dependency is satisfied". Legality was decided before this source was asked. + suggest("A", 1e9, { assumptions: ["requires: B", "hypothesis: B is satisfied"] }), + ]), + ]); + assert.equal(outcome.refusals.length, 1); + assert.equal(outcome.refusals[0]!.task, "A"); + assert.match(outcome.refusals[0]!.reason, /outside the legal candidate set/u); + assert.deepEqual(outcome.order, ["B"]); + assert.deepEqual(outcome.adopted, []); + + // Control: the same suggestion is adopted once B really is accepted, so the refusal above is + // about legality and not about the fixture being unable to score anything. + const accepted = dispatchTasks(units.units, { + artifacts: { B: "artifact-b" }, + verdicts: { B: { digest: "artifact-b", verdict: "accepted" } }, + }); + assert.deepEqual(selectableTasks(accepted), ["A"]); +}); + +test("an unmodelled action is refused rather than scored", () => { + // `fuse` and `prepare` are named in the shared vocabulary and not modelled: a suggestion cannot + // start speculative execution or merge units into one acceptance by ranking them. + for (const action of ["fuse", "prepare"]) { + const outcome = orderCandidates(["A", "B"], projection(["A", "B"]), [ + source("fixture-mgr", [suggest("B", 9, { action })]), + ]); + assert.equal(outcome.refusals.length, 1, `${action} is refused`); + assert.equal(outcome.refusals[0]!.field, "action"); + assert.match(outcome.refusals[0]!.reason, /not a modelled action/u); + assert.deepEqual(outcome.order, ["A", "B"]); + } +}); + +test("a disabled or failing source falls back to the rule policy, and says why", () => { + const failing: SuggestionSource = { + id: "fixture-failing", + kind: "mgr", + suggest: () => { + throw new Error("no trained state is available"); + }, + }; + const outcome = orderCandidates(["A", "B"], projection(["A", "B"]), [ + source("fixture-disabled", [suggest("B", 5)], { enabled: false }), + failing, + source("fixture-silent", []), + ]); + assert.deepEqual(outcome.order, ["A", "B"]); // the rule order, unchanged + assert.deepEqual(outcome.adopted, []); + assert.deepEqual( + outcome.fallbacks.map((entry) => entry.sourceId), + ["fixture-disabled", "fixture-failing"], + ); + assert.match(outcome.fallbacks[0]!.reason, /disabled/u); + assert.match(outcome.fallbacks[1]!.reason, /failed: no trained state is available/u); +}); + +test("a score from another session or branch is not reused", () => { + const otherSession = orderCandidates(["A", "B"], projection(["A", "B"]), [ + source("fixture-ha", [suggest("B", 9, { provenance: provenance({ sessionId: "session-2" }) })]), + ]); + assert.equal(otherSession.refusals.length, 1); + assert.match(otherSession.refusals[0]!.reason, /not reused in session-1\/branch-1/u); + const otherBranch = orderCandidates(["A", "B"], projection(["A", "B"]), [ + source("fixture-ha", [suggest("B", 9, { provenance: provenance({ branchId: "branch-2" }) })]), + ]); + assert.match(otherBranch.refusals[0]!.reason, /session-1\/branch-1/u); +}); + +test("a changed parameter or projection version makes an old score a new one", () => { + const changedParameters = orderCandidates(["A", "B"], projection(["A", "B"]), [ + source("fixture-ha", [ + suggest("B", 9, { provenance: provenance({ parametersVersion: "params-2" }) }), + ]), + ]); + assert.deepEqual(changedParameters.rescored, [ + { sourceId: "fixture-ha", taskId: "B", missing: ["parametersVersion=params-2"] }, + ]); + // Not adopted: a score about different parameters cannot order the current set. + assert.deepEqual(changedParameters.order, ["A", "B"]); + const changedProjection = orderCandidates(["A", "B"], projection(["A", "B"]), [ + source("fixture-ha", [ + suggest("B", 9, { provenance: provenance({ projectionVersion: "projection-2" }) }), + ]), + ]); + assert.deepEqual(changedProjection.rescored[0]!.missing, ["projectionVersion=projection-2"]); +}); + +test("a score that cannot name its own history is re-scored, never reported as a reproduction", () => { + const outcome = orderCandidates(["A", "B"], projection(["A", "B"]), [ + source("fixture-ha", [ + suggest("B", 9, { + provenance: provenance({ observationOrder: undefined, initialState: undefined }), + }), + ]), + ]); + assert.deepEqual(outcome.rescored, [ + { sourceId: "fixture-ha", taskId: "B", missing: ["observationOrder", "initialState"] }, + ]); + assert.deepEqual(outcome.adopted, []); + assert.deepEqual(outcome.order, ["A", "B"]); + // A different observation order is the same kind of answer: the old reading was taken elsewhere. + const reordered = orderCandidates(["A", "B"], projection(["A", "B"]), [ + source("fixture-ha", [ + suggest("B", 9, { provenance: provenance({ observationOrder: ["projection-0"] }) }), + ]), + ]); + assert.deepEqual(reordered.rescored[0]!.missing, ["observationOrder=projection-0"]); +}); + +test("the ordering is a permutation of the legal set, and the rule order is the default", () => { + assert.deepEqual(orderCandidates(["A", "B", "C"], projection(["A", "B", "C"])).order, [ + "A", + "B", + "C", + ]); + const tied = orderCandidates(["A", "B", "C"], projection(["A", "B", "C"]), [ + source("fixture-ha", [suggest("C", 1), suggest("B", 1)]), + ]); + // Equal scores keep the rule order, so a source cannot shuffle by enumeration. + assert.deepEqual(tied.order, ["B", "C", "A"]); + assert.deepEqual([...tied.order].sort(), ["A", "B", "C"]); +}); + +test("an adopted ranking is re-checked where the write happens", () => { + assert.equal(revalidateSuggestion({ sourceId: "fixture-ha", taskId: "A" }, ["A", "B"]), null); + const refused = revalidateSuggestion({ sourceId: "fixture-ha", taskId: "A" }, ["B"]); + assert.equal(refused?.task, "A"); + assert.match(refused!.reason, /no longer a legal candidate/u); + assert.match(refused!.reason, /not a write licence/u); +}); + +const REV = "input-v1"; +/** Two independent tasks: both are legal candidates at once, and the rule policy picks the first. */ +const parallelPlan: ProbePlan = [ + ["A", REV, [], "isolated-artifact", null, null], + ["B", REV, [], "isolated-artifact", null, null], +]; + +const spec = (instruction: string, path: string): PatchTaskSpec => ({ + instruction, + files: { [path]: "export const a = 1;\n" }, + editable: [path], + verify: async () => "accept" as const, +}); + +const SPECS: Readonly> = { + A: spec("A works.", "src/a.ts"), + B: spec("B works.", "src/b.ts"), +}; + +const open = (advisers?: readonly SuggestionSource[]) => { + const directory = mkdtempSync(join(tmpdir(), "nmg-advisers-")); + return new BoardAdmission( + join(directory, "round.sqlite"), + parallelPlan, + SPECS, + advisers + ? { runId: "run-advised", advisers, adviceScope: { ...SCOPE } } + : { runId: "run-plain" }, + ); +}; + +/** Ranks `taskId` first when the shared rules made it legal, and says nothing otherwise. */ +const prefer = (taskId: string, score = 1): SuggestionSource => ({ + id: "fixture-adviser", + kind: "ha", + suggest: (view) => + view.legal.includes(taskId) + ? [{ action: "next", taskId, score, provenance: provenance() }] + : [], +}); + +test("the round's own answer is the shared rule's answer, not an ordering's", () => { + const units = compileTaskUnits({ plan: parallelPlan, specs: SPECS }); + assert.ok(units.legal, "the fixture plan is legal"); + const dispatch = dispatchTasks(units.units, {}); + assert.equal(nextTask(dispatch), "A", "the rule policy picks the head"); + assert.equal(selectableTasks(dispatch)[0], nextTask(dispatch)); + assert.equal(open().next(), nextTask(dispatch), "the round asks the shared rule"); + + // The head rule is a legality condition, and an ordering must not skip it: A's input has drifted, + // which is not an external wait, so nothing is selectable even though B alone would be. + const stale = dispatchTasks(units.units, { + revisions: { A: "other" }, + sourceRevisions: { A: REV }, + }); + assert.deepEqual(selectableTasks(stale), [], "a head blocked by a stale input is not skipped"); + assert.equal(nextTask(stale), null); +}); + +test("an admission with no source answers exactly as the rule does, and a source only reorders", () => { + const plain = open(); + assert.equal(plain.next(), "A", "the rule policy picks the first legal candidate"); + assert.equal(plain.lastAdviceOutcome(), null, "no source means no advice to record"); + + const advised = open([prefer("B")]); + assert.equal(advised.next(), "B", "a source reorders inside the legal set"); + assert.deepEqual(advised.lastAdviceOutcome()?.adopted, [ + { sourceId: "fixture-adviser", taskId: "B", score: 1 }, + ]); + assert.deepEqual(advised.lastAdviceOutcome()?.refusals, []); + + // The same admission with a source that ranks a task nobody made legal: the rule answer stands. + const refused = open([prefer("Z")]); + assert.equal(refused.next(), "A"); + assert.equal(refused.lastAdviceOutcome()?.adopted.length, 0); + assert.deepEqual(refused.lastAdviceOutcome()?.refusals, []); + assert.deepEqual(refused.lastAdviceOutcome()?.order, ["A", "B"]); + + // The claim-side check the design names: the ranking is not a write licence. + assert.equal(advised.refuseStaleRanking("B", ["A", "B"]), null); + assert.match(advised.refuseStaleRanking("B", ["A"])!.reason, /no longer a legal candidate/u); +}); diff --git a/tests/integration/ooo-evidence-drivers.test.ts b/tests/integration/ooo-evidence-drivers.test.ts index 5c89391d..e18cb8cb 100644 --- a/tests/integration/ooo-evidence-drivers.test.ts +++ b/tests/integration/ooo-evidence-drivers.test.ts @@ -2,22 +2,34 @@ * Smoke tests for the OoO evidence drivers in evals/ooo-execution/. * * They exist because those files are *not* tests: tsc and eslint skip evals/, and nothing in CI ran - * them, so one of them quietly rotted — probe-check-duration.ts still imported round-log.ts from the - * directory that file had left. Moving them into the repository made them reviewable but not safe; + * them, so one of them quietly rotted — `probe-check-duration.ts`, since retired with the round, still + * imported `round-log.ts` from the directory that file had left. Moving them into the repository made them reviewable but not safe; * this file is what makes an edit that breaks them fail a gate. * * Each driver is invoked the way a reviewer would invoke it, against a scratch store, and the * assertions are on observable outcomes (exit status, refused-by-name message, the JSON the driver * prints) rather than on the driver's internals. + * + * The board is served for the drivers rather than opened by them: a driver is a client of the daemon + * that owns the round's store, and the two cases below are what hold that boundary - one behavioural + * (no daemon, no run) and one structural (no driver imports the store). */ import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; +import { spawn, type ChildProcess } from "node:child_process"; import { createHash } from "node:crypto"; -import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; + +import { httpCall } from "../../src/cli/http-client.ts"; import test from "node:test"; +import { boardCall, roundDaemon, runCall } from "../../evals/ooo-execution/round-client.ts"; +import type { NmgMethodResult } from "../../src/cli/protocol.ts"; +import { readServerState, serverStatePath } from "../../src/cli/lifecycle.ts"; +import type { ServerState } from "../../src/cli/lifecycle.ts"; + const REPOSITORY = resolve(import.meta.dirname, "..", ".."); const DRIVER = (name: string) => join(REPOSITORY, "evals", "ooo-execution", name); @@ -26,18 +38,34 @@ interface Run { out: string; } -function runDriver(name: string, args: readonly string[], cwd = REPOSITORY): Run { +/** + * One driver run, spawned rather than run inline. + * + * `spawn`, not `spawnSync`: the test process *hosts* the daemon these drivers call, so a synchronous + * spawn would block the event loop that has to answer them - the driver would sit in a fetch until + * its headers timed out. The refusal cases do not need the host, but the same helper serves both. + */ +function runDriver(name: string, args: readonly string[], cwd = REPOSITORY): Promise { // A child that inherits NODE_TEST_CONTEXT believes it is inside this test run and refuses to // start its own: "node:test run() is being called recursively". The drivers are processes, not // suites, so the marker must not reach them. const env = { ...process.env }; delete env.NODE_TEST_CONTEXT; - const result = spawnSync( - process.execPath, - ["--experimental-strip-types", DRIVER(name), ...args], - { cwd, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, env }, - ); - return { status: result.status, out: `${result.stdout ?? ""}${result.stderr ?? ""}` }; + return new Promise((resolve) => { + const child = spawn(process.execPath, ["--experimental-strip-types", DRIVER(name), ...args], { + cwd, + env, + windowsHide: true, + }); + let out = ""; + child.stdout.on("data", (chunk: Buffer) => { + out += chunk.toString("utf8"); + }); + child.stderr.on("data", (chunk: Buffer) => { + out += chunk.toString("utf8"); + }); + child.on("close", (status) => resolve({ status, out })); + }); } const scratchDirectory = () => mkdtempSync(join(tmpdir(), "nmg-evidence-drivers-")); @@ -45,139 +73,463 @@ const scratchDirectory = () => mkdtempSync(join(tmpdir(), "nmg-evidence-drivers- const digestOf = (path: string): string => createHash("sha256").update(readFileSync(path)).digest("hex"); -test("every evidence driver starts and refuses a missing flag by name", () => { +/** + * A round host in its own process: the test is a client of it, exactly as a driver is. + * + * Hosting inside this process would make the test both the server and the caller, which the round + * client refuses (see the self-call case) - and the refusal is right: whether such a call can be + * answered depends on this process not blocking, which is the assumption that produced a 305-second + * failure the first time. + */ +async function startHost( + storePath: string, +): Promise<{ state: () => ServerState; stop: () => Promise }> { + const env = { ...process.env }; + delete env.NODE_TEST_CONTEXT; + const child: ChildProcess = spawn( + process.execPath, + [ + "--experimental-strip-types", + DRIVER("round-host.ts"), + "--store", + storePath, + "--idle-ms", + "60000", + ], + { cwd: REPOSITORY, env, stdio: "ignore", windowsHide: true }, + ); + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const state = readServerState(serverStatePath(storePath)); + if (state?.port && state.pid !== undefined) { + return { + state: () => { + const current = readServerState(serverStatePath(storePath)); + if (!current?.port) throw new Error(`the host stopped serving ${storePath}`); + return current; + }, + stop: async () => { + // Ask rather than kill, so the host's release path is what runs: a host that died holding + // its lease would leave the next host on this store with a lease it cannot take. + try { + await httpCall(readServerState(serverStatePath(storePath)) ?? { pid: 0 }, "shutdown"); + } catch { + // Already gone; the wait below still applies. + } + const released = Date.now() + 4_000; + while ( + Date.now() < released && + readServerState(serverStatePath(storePath)) !== undefined + ) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + child.kill(); + }, + }; + } + await new Promise((resolve) => setTimeout(resolve, 25)); + } + child.kill(); + throw new Error(`the round host never published an endpoint for ${storePath}`); +} + +/** One board operation as a client of the round's host: the test does not open the store either. */ +async function board( + host: { state: () => ServerState }, + params: Parameters[1], +): Promise { + const result = await boardCall(host.state(), params); + if (result.action !== params.action) { + throw new Error(`the host answered ${result.action} to a ${params.action}`); + } + return result; +} + +/** One run transition, likewise from outside: registering and freezing are the runner's acts. */ +async function run( + host: { state: () => ServerState }, + params: Parameters[1], +): Promise { + const result = await runCall(host.state(), params); + if (result.action !== params.action) { + throw new Error(`the host answered ${result.action} to a ${params.action}`); + } + return result; +} + +test("a client refuses to call the endpoint its own process serves", () => { + const directory = scratchDirectory(); + const storePath = join(directory, "self.sqlite"); + // This process writes the lease, so by the lease's own record it is the host. That is the shape a + // harness takes when it hosts in-process and then calls itself over HTTP - and the shape that cannot + // work, because whether the call can be answered depends on this process not blocking. + writeFileSync( + serverStatePath(storePath), + JSON.stringify({ + pid: process.pid, + startedAt: new Date().toISOString(), + transport: "http", + host: "127.0.0.1", + port: 1, + token: "t", + }), + "utf8", + ); + assert.throws(() => roundDaemon(storePath), /a host calls the round's entry in-process/u); +}); + +test("a host releases its lease when it stops, so the next host can take the store", async () => { + const directory = scratchDirectory(); + const storePath = join(directory, "handover.sqlite"); + const first = await startHost(storePath); + assert.ok(readServerState(serverStatePath(storePath)), "the first host publishes a lease"); + await first.stop(); + assert.equal( + readServerState(serverStatePath(storePath)), + undefined, + "a stopped host leaves no lease behind: a lease held by a dead process is a store nothing can serve", + ); + const second = await startHost(storePath); + try { + const served = await board(second, { action: "list", agentId: "probe" }); + assert.equal(served.action, "list", "the second host serves the store the first released"); + } finally { + await second.stop(); + } +}); + +test("a call to a host that never answers gives up in seconds and names the reason", async () => { + const directory = scratchDirectory(); + const storePath = join(directory, "blocked.sqlite"); + // A server that accepts the connection and never answers is what a blocked host looks like from a + // client. Its lease names another pid, because a client refuses the endpoint it serves itself. + const server = createServer(() => {}); + await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve())); + try { + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + writeFileSync( + serverStatePath(storePath), + JSON.stringify({ + pid: 999_999, + startedAt: new Date().toISOString(), + transport: "http", + host: "127.0.0.1", + port, + token: "t", + }), + "utf8", + ); + const state = roundDaemon(storePath); + const started = Date.now(); + // The race is what makes this a bounded check of a bound: without the client's own limit the + // transport would hold for minutes, and this case would report that instead of hanging. + const gaveUp = Promise.race([ + boardCall(state, { action: "read", taskId: "c", agentId: "a" }, { timeoutMs: 250 }), + new Promise((_, reject) => + setTimeout(() => reject(new Error("the client did not give up within 5s")), 5_000), + ), + ]); + await assert.rejects(gaveUp, /did not answer within 250ms: 127\.0\.0\.1:\d+ is served/u); + assert.ok( + Date.now() - started < 5_000, + "the bound is what ended the wait, not the transport's", + ); + } finally { + server.closeAllConnections?.(); + server.close(); + } +}); + +test("a managed round's lifecycle from a driver process lands in the run's own log", async () => { + const directory = scratchDirectory(); + const storePath = join(directory, "round.sqlite"); + const channel = "evidence-managed"; + const runId = "run-evidence"; + const suite = join(directory, "managed-suite.test.ts"); + writeFileSync( + suite, + 'import test from "node:test";\n\ntest("a managed round has work to deliver", () => {});\n', + "utf8", + ); + const out = join(directory, "managed-output.txt"); + const host = await startHost(storePath); + try { + // The round's record first: a run, its frozen plan, and the entry it carries adopted in the same + // transition that creates the entry - the runner's acts, all of them over the wire. + await run(host, { + action: "register", + runId, + planDigest: "sha256:evidence-plan", + policy: "ordered", + revision: "1", + retention: "evidence", + }); + await run(host, { + action: "freeze", + runId, + tasks: [ + { + taskId: "T1", + revision: "1", + input: "run the managed suite", + dependencies: [], + effect: "artifact", + }, + ], + }); + const put = await boardCall(host.state(), { + action: "put", + taskId: channel, + agentId: "round-host", + kind: "handoff", + content: JSON.stringify({ id: "T1", revision: "1", attempt: 1, input: "managed" }), + expiresAt: new Date(Date.now() + 3_600_000).toISOString(), + adopt: { runId, taskId: "T1" }, + }); + if (put.action !== "put") throw new Error("expected a put result"); + assert.equal(put.bound?.recorded, true, "the entry was adopted by the put that created it"); + const entryId = put.entry.id; + + // A separate process does the work and delivers; another separate process judges it. + const worked = await runDriver("board-worker.ts", [ + "--daemon", + storePath, + "--channel", + channel, + "--out", + out, + "--suites", + suite, + "--agent", + "worker-managed", + "--entry", + entryId, + ]); + assert.equal(worked.status, 0, worked.out); + const judged = await runDriver("board-judge.ts", [ + "--daemon", + storePath, + "--channel", + channel, + "--entry", + entryId, + "--agent", + "judge-managed", + "--verdict", + "accepted", + "--reason", + "digest re-verified from the bytes", + ]); + assert.equal(judged.status, 0, judged.out); + const managedVerdict = JSON.parse(judged.out.trim().split("\n").at(-1)!) as { + verdict: string; + deliveryVerified: boolean; + }; + assert.deepEqual( + { verdict: managedVerdict.verdict, verified: managedVerdict.deliveryVerified }, + { verdict: "accepted", verified: true }, + ); + + // Every one of those acts is in the run's log, written by the daemon in the transaction that + // moved the board - which is the whole point of adopting the entry rather than creating it. + const status = await run(host, { action: "status", runId }); + if (status.action !== "status") throw new Error("expected a status result"); + const kinds = status.status.facts.map((fact) => fact.kind); + assert.deepEqual(kinds, ["entry-bound", "board-claim", "board-deliver", "board-judge"]); + assert.deepEqual(status.status.bindings, [ + { + taskId: "T1", + attempt: 1, + entryId, + sequence: status.status.bindings[0]!.sequence, + // Delivery and judgement record the artifact and the verdict; only `resolve` moves the + // entry's status, so an accepted entry is still open - the state the run's log exists to keep. + entry: { taskId: channel, status: "open", claimedBy: "worker-managed", ackedBy: [] }, + }, + ]); + } finally { + await host.stop(); + } +}); + +test("a driver refuses without a daemon, and no driver opens a database of its own", async () => { + const directory = scratchDirectory(); + // Behavioural: the store exists, nothing serves it, and the driver says so by name instead of + // opening the file. This is the shape that would otherwise rot back in silently. + const store = join(directory, "scratch.sqlite"); + const refused = await runDriver("board-deliver.ts", [ + "--daemon", + store, + "--channel", + "c", + "--entry", + "e", + "--agent", + "a", + "--digest", + "0".repeat(64), + ]); + assert.notEqual(refused.status, 0); + assert.match(refused.out, /no daemon is serving/u); + + // Structural: the whole boundary is that these three files reach the board through the daemon, so + // a re-added store import has to fail here rather than only under a real round. + const drivers = readdirSync(join(REPOSITORY, "evals", "ooo-execution")).filter((name) => + /^board-(worker|deliver|judge)\.ts$/u.test(name), + ); + assert.equal(drivers.length, 3, "the three board drivers are expected to exist"); + for (const driver of drivers) { + const source = readFileSync(join(REPOSITORY, "evals", "ooo-execution", driver), "utf8"); + assert.doesNotMatch( + source, + /core\/store|NmgStoreBase/u, + `${driver} must reach the board through the daemon, not through the store`, + ); + assert.match(source, /round-client\.ts/u, `${driver} must use the round client`); + } +}); + +test("every evidence driver starts and refuses a missing flag by name", async () => { // A driver that cannot even load fails here too, which is the failure this file exists for: the // probe's stale import meant the script the record cited could not run at all. const cases: Array<{ driver: string; args: string[]; expected: RegExp }> = [ { driver: "board-worker.ts", args: [], expected: /--channel is required/u }, { driver: "board-worker.ts", args: ["--channel", "c"], expected: /--out is required/u }, - { driver: "board-judge.ts", args: ["--channel", "c"], expected: /--entry is required/u }, - { driver: "board-deliver.ts", args: ["--channel", "c"], expected: /--entry is required/u }, { - driver: "probe-check-duration.ts", - args: ["--check-ms", "0"], - expected: /--out is required/u, + driver: "board-worker.ts", + args: ["--channel", "c", "--out", "o", "--daemon", "s"], + expected: /no daemon is serving/u, }, + { driver: "board-judge.ts", args: ["--channel", "c"], expected: /--entry is required/u }, + { driver: "board-deliver.ts", args: ["--channel", "c"], expected: /--entry is required/u }, + { driver: "round-host.ts", args: [], expected: /--store is required/u }, ]; for (const { driver, args, expected } of cases) { - const result = runDriver(driver, args); + const result = await runDriver(driver, args); assert.notEqual(result.status, 0, `${driver} ${args.join(" ")} must refuse`); assert.match(result.out, expected, `${driver} ${args.join(" ")}`); } }); -test("the board drivers run the protocol end to end on a scratch store", async () => { +test("the board drivers run the protocol end to end through the daemon that serves the store", async () => { const directory = scratchDirectory(); const store = join(directory, "scratch.sqlite"); const channel = "evidence-driver-smoke"; - const { NmgStoreBase } = await import("../../src/core/store/base.ts"); - const board = new NmgStoreBase(store); - const expiresAt = new Date(Date.now() + 3_600_000).toISOString(); - // Directed, so the channel's single outstanding serial slot cannot block the second entry. - const entry = board.putTaskBoardEntry({ - taskId: channel, - agentId: "smoke-setup", - kind: "handoff", - to: "smoke-holder", - content: JSON.stringify({ id: "S", revision: "1", attempt: 1, input: "smoke" }), - expiresAt, - }); - const claimed = board.claimTaskBoardEntry({ - taskId: channel, - entryId: entry.id, - agentId: "smoke-holder", - leaseSeconds: 600, - }); - board.close(); - assert.equal(claimed.claimedBy, "smoke-holder"); + const host = await startHost(store); + try { + const expiresAt = new Date(Date.now() + 3_600_000).toISOString(); + // Directed, so the channel's single outstanding serial slot cannot block the second entry. + const put = await board(host, { + action: "put", + taskId: channel, + agentId: "smoke-setup", + kind: "handoff", + to: "smoke-holder", + content: JSON.stringify({ id: "S", revision: "1", attempt: 1, input: "smoke" }), + expiresAt, + }); + if (put.action !== "put") throw new Error("expected a put result"); + const entry = put.entry; + const claimed = await board(host, { + action: "claim", + taskId: channel, + entryId: entry.id, + agentId: "smoke-holder", + leaseSeconds: 600, + }); + assert.equal(claimed.action === "claim" && claimed.entry.claimedBy, "smoke-holder"); - const artifact = join(directory, "artifact.txt"); - writeFileSync(artifact, "the bytes the digest is computed from\n", "utf8"); - const delivered = runDriver("board-deliver.ts", [ - "--store", - store, - "--channel", - channel, - "--entry", - entry.id, - "--agent", - "smoke-holder", - "--ref", - artifact, - "--summary", - "smoke delivery", - ]); - assert.equal(delivered.status, 0, delivered.out); - const delivery = JSON.parse(delivered.out.trim().split("\n").at(-1)!) as { - deliverableDigest: string; - digestRecomputedFromRef: boolean; - }; - assert.equal( - delivery.digestRecomputedFromRef, - true, - "the digest comes from the bytes, not the caller", - ); - assert.match(delivery.deliverableDigest, /^[0-9a-f]{64}$/u); + const artifact = join(directory, "artifact.txt"); + writeFileSync(artifact, "the bytes the digest is computed from\n", "utf8"); + const delivered = await runDriver("board-deliver.ts", [ + "--daemon", + store, + "--channel", + channel, + "--entry", + entry.id, + "--agent", + "smoke-holder", + "--ref", + artifact, + "--summary", + "smoke delivery", + ]); + assert.equal(delivered.status, 0, delivered.out); + const delivery = JSON.parse(delivered.out.trim().split("\n").at(-1)!) as { + deliverableDigest: string; + digestRecomputedFromRef: boolean; + }; + assert.equal( + delivery.digestRecomputedFromRef, + true, + "the digest comes from the bytes, not the caller", + ); + assert.match(delivery.deliverableDigest, /^[0-9a-f]{64}$/u); - // A digest the caller claims but the bytes deny must be refused. - const lied = runDriver("board-deliver.ts", [ - "--store", - store, - "--channel", - channel, - "--entry", - entry.id, - "--agent", - "smoke-holder", - "--ref", - artifact, - "--digest", - "0".repeat(64), - ]); - assert.notEqual(lied.status, 0); - assert.match(lied.out, /does not match the bytes/u); + // A digest the caller claims but the bytes deny must be refused. + const lied = await runDriver("board-deliver.ts", [ + "--daemon", + store, + "--channel", + channel, + "--entry", + entry.id, + "--agent", + "smoke-holder", + "--ref", + artifact, + "--digest", + "0".repeat(64), + ]); + assert.notEqual(lied.status, 0); + assert.match(lied.out, /does not match the bytes/u); - const selfJudge = runDriver("board-judge.ts", [ - "--store", - store, - "--channel", - channel, - "--entry", - entry.id, - "--agent", - "smoke-holder", - "--verdict", - "accepted", - "--reason", - "self", - ]); - assert.notEqual(selfJudge.status, 0); - assert.match(selfJudge.out, /refusing to judge my own deliverable/u); + const selfJudge = await runDriver("board-judge.ts", [ + "--daemon", + store, + "--channel", + channel, + "--entry", + entry.id, + "--agent", + "smoke-holder", + "--verdict", + "accepted", + "--reason", + "self", + ]); + assert.notEqual(selfJudge.status, 0); + assert.match(selfJudge.out, /refusing to judge my own deliverable/u); - const judged = runDriver("board-judge.ts", [ - "--store", - store, - "--channel", - channel, - "--entry", - entry.id, - "--agent", - "smoke-reviewer", - "--verdict", - "accepted", - "--reason", - "digest re-verified from the bytes", - ]); - assert.equal(judged.status, 0, judged.out); - const verdict = JSON.parse(judged.out.trim().split("\n").at(-1)!) as { - verdict: string; - judgedBy: string; - deliveryVerified: boolean; - }; - assert.deepEqual( - { verdict: verdict.verdict, judgedBy: verdict.judgedBy, verified: verdict.deliveryVerified }, - { verdict: "accepted", judgedBy: "smoke-reviewer", verified: true }, - ); + const judged = await runDriver("board-judge.ts", [ + "--daemon", + store, + "--channel", + channel, + "--entry", + entry.id, + "--agent", + "smoke-reviewer", + "--verdict", + "accepted", + "--reason", + "digest re-verified from the bytes", + ]); + assert.equal(judged.status, 0, judged.out); + const verdict = JSON.parse(judged.out.trim().split("\n").at(-1)!) as { + verdict: string; + judgedBy: string; + deliveryVerified: boolean; + }; + assert.deepEqual( + { verdict: verdict.verdict, judgedBy: verdict.judgedBy, verified: verdict.deliveryVerified }, + { verdict: "accepted", judgedBy: "smoke-reviewer", verified: true }, + ); + } finally { + await host.stop(); + } }); test("the worker claims, runs the named suite and delivers its digest", async () => { @@ -192,75 +544,59 @@ test("the worker claims, runs the named suite and delivers its digest", async () "utf8", ); const out = join(directory, "worker-output.txt"); + const host = await startHost(store); + try { + const setup = await runDriver("board-worker.ts", [ + "--daemon", + store, + "--channel", + channel, + "--out", + out, + "--suites", + suite, + ]); + // No handoff is published yet, so this run proves the driver loaded and refused honestly. + assert.notEqual(setup.status, 0); + assert.match(setup.out, /no open handoff/u); + assert.equal(existsSync(out), false, "a refused run must not leave an artifact behind"); - const setup = runDriver("board-worker.ts", [ - "--store", - store, - "--channel", - channel, - "--out", - out, - "--suites", - suite, - ]); - // No handoff is published yet, so this run proves the driver loaded and refused honestly. - assert.notEqual(setup.status, 0); - assert.match(setup.out, /no open handoff/u); - assert.equal(existsSync(out), false, "a refused run must not leave an artifact behind"); - - const { NmgStoreBase } = await import("../../src/core/store/base.ts"); - const board = new NmgStoreBase(store); - board.putTaskBoardEntry({ - taskId: channel, - agentId: "smoke-setup", - kind: "handoff", - content: JSON.stringify({ id: "W", revision: "1", attempt: 1, input: "run the smoke suite" }), - expiresAt: new Date(Date.now() + 3_600_000).toISOString(), - }); - board.close(); + await board(host, { + action: "put", + taskId: channel, + agentId: "smoke-setup", + kind: "handoff", + content: JSON.stringify({ id: "W", revision: "1", attempt: 1, input: "run the smoke suite" }), + expiresAt: new Date(Date.now() + 3_600_000).toISOString(), + }); - const delivered = runDriver("board-worker.ts", [ - "--store", - store, - "--channel", - channel, - "--out", - out, - "--suites", - suite, - ]); - assert.equal(delivered.status, 0, delivered.out); - const report = JSON.parse(delivered.out.trim().split("\n").at(-1)!) as { - summary: string; - digest: string; - artifactBytes: number; - verdict: string | null; - }; - assert.match(report.summary, /tests=1 pass=1 fail=0/u); - assert.equal(report.verdict, null, "the worker must not judge its own delivery"); - assert.equal(report.artifactBytes, readFileSync(out).byteLength); - assert.equal(report.digest, digestOf(out)); + const delivered = await runDriver("board-worker.ts", [ + "--daemon", + store, + "--channel", + channel, + "--out", + out, + "--suites", + suite, + ]); + assert.equal(delivered.status, 0, delivered.out); + const report = JSON.parse(delivered.out.trim().split("\n").at(-1)!) as { + summary: string; + digest: string; + artifactBytes: number; + verdict: string | null; + }; + assert.match(report.summary, /tests=1 pass=1 fail=0/u); + assert.equal(report.verdict, null, "the worker must not judge its own delivery"); + assert.equal(report.artifactBytes, readFileSync(out).byteLength); + assert.equal(report.digest, digestOf(out)); + } finally { + await host.stop(); + } }); -test("the duration probe runs a grid and records what it measured", () => { - const directory = scratchDirectory(); - const result = runDriver("probe-check-duration.ts", [ - "--out", - directory, - "--check-ms", - "0", - "--worker-ms", - "0", - ]); - assert.equal(result.status, 0, result.out); - assert.match(result.out, /check ms \| worker ms \| ooo wall ms/u); - assert.match(result.out, /verdicts identical across the grid: true/u); - const recorded = JSON.parse(readFileSync(join(directory, "check-duration.json"), "utf8")) as { - points: Array<{ checkMs: number; hiddenShare: number }>; - }; - assert.ok(recorded.points.length > 0, "the probe records the points it measured"); - assert.ok( - recorded.points.every((point) => Number.isFinite(point.hiddenShare)), - "every recorded share is a number rather than a silent zero", - ); -}); +// The duration probe's grid case is gone with its subject: `probe-check-duration.ts` read the round's +// log, and both were retired in +// `docs/decisions/implemented/2026-09-18-retire-the-round-instrument.md`. The arms' own timing lives in +// `plan-driver.ts`'s `PlanRun` (wall, host and per-unit millis), which its suite pins. diff --git a/tests/integration/ooo-external-window.test.ts b/tests/integration/ooo-external-window.test.ts new file mode 100644 index 00000000..0e07e09f --- /dev/null +++ b/tests/integration/ooo-external-window.test.ts @@ -0,0 +1,121 @@ +/** + * The design's C4: an unknown external result in a crash window is not guessed - not as success, + * and not as "it never happened". + * + * The store enforces this more strongly than a comment could: an external event can only be marked + * ready when the check issued for the waiting task has reported a terminal, bound result. So the + * crash window - between the external operation being issued and its result being recorded - cannot + * be closed by anyone deciding the answer arrived, and the wait is read back as pending rather than + * promoted or discarded. + */ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import test from "node:test"; + +import { + BoardAdmission, + type PatchTaskSpec, + type ProbePlan, +} from "../../src/integration/ooo-board.ts"; + +const plan: ProbePlan = [ + ["A", "", [], "isolated-artifact", "protocol-regression", null], + ["B", "", [], "isolated-artifact", null, null], +]; +const specs: Record = { + A: { + instruction: "A.", + files: { "a.ts": "export const a = 1;\n" }, + editable: ["a.ts"], + verify: async () => "accept", + }, + B: { + instruction: "B.", + files: { "b.ts": "export const b = 1;\n" }, + editable: ["b.ts"], + verify: async () => "accept", + }, +}; + +function rows(path: string, sql: string): Record[] { + const reader = new DatabaseSync(path, { readOnly: true }); + try { + return reader.prepare(sql).all() as Record[]; + } finally { + reader.close(); + } +} + +function scratch(t: { after: (fn: () => void) => void }) { + const directory = mkdtempSync(join(tmpdir(), "nmg-external-window-")); + const opened: BoardAdmission[] = []; + t.after(() => { + // The crash simulation in the test closes a gate on purpose, so a second close is expected. + for (const gate of opened) { + try { + gate.close(); + } catch { + /* already closed by the test */ + } + } + rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + }); + return { + database: join(directory, "window.sqlite"), + open: (runId?: string): BoardAdmission => { + const gate = new BoardAdmission( + join(directory, "window.sqlite"), + plan, + specs, + runId ? { runId } : {}, + ); + opened.push(gate); + return gate; + }, + }; +} + +test("a pending external result survives a restart as pending, not as a guess", (t) => { + const { database, open } = scratch(t); + const gate = open("run-window"); + + // The check runs while the wait is pending: that is the concurrency the round exists for. + assert.equal(gate.issueCheck("A", "worker-one").taskId, "A"); + + // The wait cannot be closed by announcing it. There is no terminal evidence bound to this task, so + // the store refuses - which is the property, not the message. + assert.throws( + () => gate.externalReady("protocol-regression"), + /bound terminal evidence/, + "a result that nobody observed must not be recordable", + ); + assert.throws(() => gate.externalReady("invented"), /unknown external event/); + + // Simulate the crash: nothing was recorded, the handle goes away, a successor opens the store. + gate.close(); + const successor = open("run-window"); + const pending = rows( + database, + "SELECT external_ready, artifact FROM ooo_probe_task_view WHERE run_id='run-window' AND id='A'", + )[0]!; + assert.equal(pending.external_ready, 0, "an unrecorded result is not read back as success"); + assert.equal(pending.artifact, null, "and no artifact appeared while nobody was looking"); + assert.deepEqual(successor.accepted(), {}, "the crashed window accepts nothing"); + assert.equal(successor.cancelled(), null, "and the crash is not mistaken for a cancellation"); + + // The successor is no more able to close the wait by fiat than the first process was: pending has + // to stay pending until a check reports, which is the difference between an unknown result and a + // convenient one. It is still nameable, so pending does not mean lost. + assert.throws(() => successor.externalReady("protocol-regression"), /bound terminal evidence/); + assert.equal( + rows( + database, + "SELECT external_ready FROM ooo_probe_task_view WHERE run_id='run-window' AND id='A'", + )[0]!.external_ready, + 0, + "the wait is still the same pending wait after the restart", + ); +}); diff --git a/tests/integration/ooo-fusion-plan.test.ts b/tests/integration/ooo-fusion-plan.test.ts new file mode 100644 index 00000000..74e948cd --- /dev/null +++ b/tests/integration/ooo-fusion-plan.test.ts @@ -0,0 +1,173 @@ +/** + * Fusion planning: the move the online half makes, and the ceiling the offline half computes. + * + * The cases below pin the three properties the design claims - a move is a pure function of the plan + * and its facts, an offline graph prices the best case rather than a run's state, and a floor is a + * floor - plus the two refusals that keep a bound from turning into a schedule. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + chainCoverFloor, + fusionGraph, + listScheduleSessions, + nextSessionMove, + optimisticPlan, +} from "../../src/integration/ooo-fusion-plan.ts"; +import { + sharedSessionLegal, + type DispatchTask, + type SessionDeclaration, + type SessionPlan, +} from "../../src/integration/ooo-execution.ts"; + +function task(id: string, over: Partial = {}): DispatchTask { + return { + id, + effect: "isolated-artifact", + sourceVersion: "v1", + observedVersion: "v1", + dependencies: [], + accepted: true, + claimed: false, + externalReady: true, + ...over, + }; +} + +function declaration(over: Partial = {}): SessionDeclaration { + return { capability: "patch", authority: "host", visible: ["src/a.ts"], ...over }; +} + +/** Units named `one`, `two`... with identical declarations: the shape that fuses freely. */ +function plan(count: number, over: Partial = {}): SessionPlan { + const names = ["one", "two", "three", "four"].slice(0, count); + const tasks = names.map((name, index) => + task(name, index === 0 ? {} : { dependencies: [names[index - 1]!] }), + ); + return { + tasks, + declarations: Object.fromEntries(names.map((name) => [name, declaration()])), + ...over, + }; +} + +test("a linear plan is one chain, and its relation is transitive", () => { + const graph = fusionGraph(plan(3)); + assert.deepEqual(graph.units, ["one", "two", "three"]); + assert.equal(graph.transitive, true); + assert.deepEqual(graph.edges.get("one"), ["two", "three"]); + assert.deepEqual(graph.edges.get("two"), ["three"]); + assert.deepEqual(graph.edges.get("three"), []); +}); + +test("the offline graph prices the best case, not the run's state", () => { + const stateful = plan(2); + stateful.tasks[0]!.accepted = false; + stateful.tasks[1]!.cancelled = true; + // The run's own view refuses the pair - an unaccepted unit may not be continued. + assert.equal(sharedSessionLegal("one", "two", stateful), false); + // The projection makes the conditions that read run state vacuous, leaving condition 1. + assert.equal(sharedSessionLegal("one", "two", optimisticPlan(stateful)), true); + assert.deepEqual(fusionGraph(stateful).edges.get("one"), ["two"]); +}); + +test("a chain is a linear extension: a unit is never followed by what it depends on", () => { + const reversed: SessionPlan = { + tasks: [task("one", { dependencies: ["two"] }), task("two")], + declarations: { one: declaration(), two: declaration() }, + }; + // `two` cannot follow `one` because a chain follows plan order, and `one` cannot follow `two` + // because `one` depends on it: the pair is refused in both directions, each by one restriction. + assert.deepEqual(fusionGraph(reversed).edges.get("one"), []); + assert.deepEqual(fusionGraph(reversed).edges.get("two"), []); +}); + +test("a relation that is not transitive is reported as such, and still gets a floor", () => { + // one depends on three, so one may follow two and two may follow three, but one may not follow + // three - the dependency guard fires where plan order allows the pair. + const twisted: SessionPlan = { + tasks: [task("one", { dependencies: ["three"] }), task("two"), task("three")], + declarations: { one: declaration(), two: declaration(), three: declaration() }, + }; + const graph = fusionGraph(twisted); + assert.equal(graph.transitive, false); + assert.deepEqual(graph.edges.get("one"), ["two"]); + assert.deepEqual(graph.edges.get("two"), ["three"]); + assert.equal(chainCoverFloor(graph), 1); +}); + +test("the floor is the minimum chain cover: incommensurable units need their own sessions", () => { + const incompatible: SessionPlan = { + tasks: [task("one"), task("two"), task("three")], + declarations: { + one: declaration(), + two: declaration({ capability: "other" }), + three: declaration({ capability: "third" }), + }, + }; + assert.equal(chainCoverFloor(fusionGraph(incompatible)), 3); + assert.equal(chainCoverFloor(fusionGraph(plan(3))), 1); + assert.equal(chainCoverFloor(fusionGraph(plan(1))), 1); +}); + +test("list scheduling respects the cap and the legality edges", () => { + const graph = fusionGraph(plan(3)); + assert.deepEqual(listScheduleSessions(graph, 1), [["one"], ["two"], ["three"]]); + assert.deepEqual(listScheduleSessions(graph, 3), [["one", "two", "three"]]); + assert.deepEqual(listScheduleSessions(graph, 2), [["one", "two"], ["three"]]); +}); + +test("sessions never increase when the cap grows", () => { + for (const count of [1, 2, 3, 4]) { + const graph = fusionGraph(plan(count)); + const sizes = [1, 2, 3, 4].map((cap) => listScheduleSessions(graph, cap).length); + for (let index = 1; index < sizes.length; index += 1) { + assert.equal(sizes[index]! <= sizes[index - 1]!, true); + } + // The floor bounds every feasible schedule from below. + assert.equal(sizes[sizes.length - 1]! >= chainCoverFloor(graph), true); + } +}); + +test("list scheduling refuses a cap that is not a positive integer", () => { + const graph = fusionGraph(plan(2)); + assert.throws(() => listScheduleSessions(graph, 0), /positive integer/); + assert.throws(() => listScheduleSessions(graph, 1.5), /positive integer/); +}); + +test("the move admits the first legal successor on offer, in plan order", () => { + const move = nextSessionMove({ + plan: plan(3), + current: "one", + size: 1, + bound: 2, + onOffer: ["three", "two"], + }); + assert.deepEqual(move, { kind: "admit", unit: "two" }); +}); + +test("the move closes the session by name, never by guessing", () => { + assert.deepEqual( + nextSessionMove({ plan: plan(3), current: "one", size: 2, bound: 2, onOffer: ["two"] }), + { kind: "close", reason: "the declared bound is reached" }, + ); + assert.deepEqual( + nextSessionMove({ plan: plan(3), current: "one", size: 1, bound: 3, onOffer: [] }), + { kind: "close", reason: "no legal successor is on offer" }, + ); + // An illegal successor is not admitted even when the board offers it. + const incompatible: SessionPlan = { + tasks: [task("one"), task("two")], + declarations: { one: declaration(), two: declaration({ capability: "other" }) }, + }; + assert.deepEqual( + nextSessionMove({ plan: incompatible, current: "one", size: 1, bound: 2, onOffer: ["two"] }), + { kind: "close", reason: "no legal successor is on offer" }, + ); +}); + +test("the same plan and the same facts yield the same move", () => { + const input = { plan: plan(3), current: "one", size: 1, bound: 3, onOffer: ["two", "three"] }; + assert.deepEqual(nextSessionMove(input), nextSessionMove(input)); +}); diff --git a/tests/integration/ooo-fusion.test.ts b/tests/integration/ooo-fusion.test.ts new file mode 100644 index 00000000..f403cc31 --- /dev/null +++ b/tests/integration/ooo-fusion.test.ts @@ -0,0 +1,146 @@ +/** + * F4: fusion legality - the five conditions the design puts on reusing one Agent session across + * logical units, each refused by name, plus the candidate set a ranking policy may choose from. + * + * The rules live in `src/integration/ooo-execution.ts` beside the selection rules, because a fused + * successor answers the same kind of question as a selectable task: what the host is allowed to hand + * out. A pair is legal only when every condition holds, so each case below breaks exactly one. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + fusionCandidates, + fusionSuccessors, + sharedSessionLegal, + type DispatchTask, + type SessionDeclaration, + type SessionPlan, +} from "../../src/integration/ooo-execution.ts"; + +function task(id: string, over: Partial = {}): DispatchTask { + return { + id, + effect: "isolated-artifact", + sourceVersion: "v1", + observedVersion: "v1", + dependencies: [], + accepted: false, + claimed: false, + externalReady: true, + ...over, + }; +} + +function declaration(over: Partial = {}): SessionDeclaration { + return { capability: "patch", authority: "host", visible: ["src/a.ts"], ...over }; +} + +/** `before` accepted, `after` depending on it and passing every declaration check: the one plan the + * refusal cases each break in exactly one place. */ +function plan(over: Partial = {}): SessionPlan { + return { + tasks: [task("before", { accepted: true }), task("after", { dependencies: ["before"] })], + declarations: { before: declaration(), after: declaration() }, + ...over, + }; +} + +function legal(over: Partial = {}): boolean { + return sharedSessionLegal("before", "after", plan(over)); +} + +test("fusion lets a successor continue the session when all five conditions hold", () => { + assert.equal(legal(), true); +}); + +test("fusion refuses a unit that needs a different execution capability", () => { + const declarations = { before: declaration(), after: declaration({ capability: "snapshot" }) }; + assert.equal(legal({ declarations }), false); +}); + +test("fusion refuses a unit acting under a different authority", () => { + const declarations = { before: declaration(), after: declaration({ authority: "guest" }) }; + assert.equal(legal({ declarations }), false); +}); + +test("fusion refuses a successor whose visibility the session would widen", () => { + const declarations = { + before: declaration(), + after: declaration({ visible: ["src/a.ts", "src/secret.ts"] }), + }; + assert.equal(legal({ declarations }), false); +}); + +test("fusion refuses to continue from a unit whose verdict is not accepted", () => { + const tasks = [task("before"), task("after", { dependencies: ["before"] })]; + assert.equal(legal({ tasks }), false); +}); + +test("fusion refuses a successor whose dependency is delivered but not accepted", () => { + const tasks = [ + task("before", { accepted: true }), + task("upstream", { delivered: true }), + task("after", { dependencies: ["before", "upstream"] }), + ]; + const declarations = { ...plan().declarations, upstream: declaration() }; + assert.equal(legal({ tasks, declarations }), false); +}); + +test("fusion refuses a cancelled unit, before or after", () => { + const cancelledFirst = [ + task("before", { accepted: true, cancelled: true }), + task("after", { dependencies: ["before"] }), + ]; + assert.equal(legal({ tasks: cancelledFirst }), false); + const cancelledNext = [ + task("before", { accepted: true }), + task("after", { dependencies: ["before"], cancelled: true }), + ]; + assert.equal(legal({ tasks: cancelledNext }), false); +}); + +test("fusion ends the session at a declared external wait that is not ready", () => { + const waiting = [ + task("before", { accepted: true, externalEvent: "g", externalReady: false }), + task("after", { dependencies: ["before"] }), + ]; + assert.equal(legal({ tasks: waiting }), false); + const ready = [ + task("before", { accepted: true, externalEvent: "g", externalReady: true }), + task("after", { dependencies: ["before"] }), + ]; + assert.equal(legal({ tasks: ready }), true); +}); + +test("fusion never reuses the history across a fact whose branch is still pending", () => { + assert.equal(legal({ pendingBranches: ["after"] }), false); + assert.equal(legal({ pendingBranches: ["before"] }), false); + assert.equal(legal({ pendingBranches: ["elsewhere"] }), true); +}); + +test("fusion refuses a pair whose units or declarations are not in the plan", () => { + assert.equal(sharedSessionLegal("before", "before", plan()), false); + assert.equal(sharedSessionLegal("before", "absent", plan()), false); + assert.equal(legal({ declarations: { before: declaration() } }), false); +}); + +test("fusion lists the legal successors in plan order", () => { + const tasks = [ + task("before", { accepted: true }), + task("after", { dependencies: ["before"] }), + task("too-wide", { dependencies: ["before"] }), + ]; + const declarations = { + before: declaration(), + after: declaration(), + "too-wide": declaration({ visible: ["src/a.ts", "src/other.ts"] }), + }; + assert.deepEqual(fusionSuccessors("before", { tasks, declarations }), ["after"]); +}); + +test("fusion returns every legal pair, and only legal ones", () => { + const tasks = [task("before", { accepted: true }), task("after", { dependencies: ["before"] })]; + const declarations = plan().declarations; + assert.deepEqual(fusionCandidates({ tasks, declarations }), [["before", "after"]]); + assert.deepEqual(fusionCandidates({ tasks, declarations, pendingBranches: ["after"] }), []); +}); diff --git a/tests/integration/ooo-managed-adopt.test.ts b/tests/integration/ooo-managed-adopt.test.ts new file mode 100644 index 00000000..b145fc22 --- /dev/null +++ b/tests/integration/ooo-managed-adopt.test.ts @@ -0,0 +1,355 @@ +/** + * The binding of a logical task to the board entry that carries it. + * + * The board cannot keep this fact (an entry that a later attempt replaces must not lose the record + * of what it used to carry), so it lives in the run's appended facts - and it is the fact that makes + * an entry managed. These cases check what the store ends up holding: + * + * - binding is what turns a board entry into a managed one, and its writes then go through the + * run's coordinated transition rather than beside it; + * - the same task and attempt bound twice is a retry, not a second binding, and a different entry + * for that task and attempt is refused rather than silently dropped; + * - every refusal names a fact the store holds: an unregistered or cancelled run, a task the run + * never froze, an entry that is not on the channel the caller names, an entry that already + * carries something else; + * - the binding joins the transition that creates the entry, so there is never an entry on the + * board that no run manages. + */ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { NmgStore } from "../../src/core/store.ts"; +import { + ENTRY_BOUND_FACT, + RUN_CANCELLED_FACT, + bindRunEntry, + coordinatedEntryWrite, +} from "../../src/integration/task-coordinator.ts"; + +/** Windows can hold a handle to a just-closed store for a few milliseconds. */ +const REMOVE_TEMP_TREE = { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }; +const NEVER = new Date(Date.now() + 86_400_000).toISOString(); +const CHANNEL = "board"; + +function withStore(run: (store: NmgStore) => void): void { + const directory = mkdtempSync(join(tmpdir(), "nmg-managed-adopt-")); + const store = new NmgStore(join(directory, "test.sqlite")); + try { + run(store); + } finally { + store.close(); + rmSync(directory, REMOVE_TEMP_TREE); + } +} + +function publish(store: NmgStore, content: string, taskId = CHANNEL): string { + return store.putTaskBoardEntry({ + taskId, + agentId: "host", + kind: "handoff", + content, + expiresAt: NEVER, + }).id; +} + +function register(store: NmgStore, runId: string): void { + store.registerTaskRun({ + runId, + planDigest: "plan-a", + policy: "checks-a", + revision: "v1", + retention: "keep:evidence", + }); +} + +function freeze(store: NmgStore, runId: string, taskId: string, position = 0): void { + store.freezeTaskRunTask({ + runId, + taskId, + position, + revision: "v1", + input: `${taskId} input`, + dependencies: [], + effect: "isolated-artifact", + }); +} + +function bindings(store: NmgStore, runId: string) { + return store + .taskRunFacts(runId) + .filter((fact) => fact.kind === ENTRY_BOUND_FACT) + .map((fact) => ({ sequence: fact.sequence, taskId: fact.taskId, entryId: fact.entryId })); +} + +test("binding is what makes an entry managed, and its writes then go through the run", () => { + withStore((store) => { + register(store, "run-1"); + freeze(store, "run-1", "T1"); + const entryId = publish(store, "managed"); + + // Before the binding the entry is nobody's: the board verb lands directly. + store.claimTaskBoardEntry({ + taskId: CHANNEL, + entryId, + agentId: "worker-one", + leaseSeconds: 60, + }); + store.releaseTaskBoardEntry({ taskId: CHANNEL, entryId, agentId: "worker-one" }); + + const bound = bindRunEntry(store, { + runId: "run-1", + taskId: "T1", + boardTaskId: CHANNEL, + entryId, + }); + assert.equal(bound.recorded, true); + assert.deepEqual(bindings(store, "run-1"), [{ sequence: 1, taskId: "T1", entryId }]); + assert.deepEqual( + store.taskRunForEntry(entryId), + { runId: "run-1", kind: ENTRY_BOUND_FACT, taskId: "T1", attempt: 1 }, + "the binding is readable from the entry alone, which is what the fence reads", + ); + + assert.throws( + () => + store.claimTaskBoardEntry({ + taskId: CHANNEL, + entryId, + agentId: "worker-one", + leaseSeconds: 60, + }), + /go through the run's coordinated transition/, + "after the binding, the entry's lifecycle writes belong to the run", + ); + assert.equal(store.getTaskBoardEntryById(CHANNEL, entryId)!.claimedBy, null); + }); +}); + +test("a binding is idempotent for its task and attempt, and refuses a second entry", () => { + withStore((store) => { + register(store, "run-1"); + freeze(store, "run-1", "T1"); + const entryId = publish(store, "managed"); + const other = publish(store, "another", "other-channel"); + + const first = bindRunEntry(store, { + runId: "run-1", + taskId: "T1", + boardTaskId: CHANNEL, + entryId, + }); + const retry = bindRunEntry(store, { + runId: "run-1", + taskId: "T1", + boardTaskId: CHANNEL, + entryId, + }); + assert.deepEqual(retry, { sequence: first.sequence, recorded: false }); + assert.equal(bindings(store, "run-1").length, 1, "a retry is the same binding"); + + // Another entry for the same task and attempt is a disagreement, not a retry: the stored fact + // is keyed by task and attempt, so accepting it would silently keep the first entry and report + // the second as bound. + assert.throws( + () => + bindRunEntry(store, { + runId: "run-1", + taskId: "T1", + boardTaskId: "other-channel", + entryId: other, + }), + /already carries entry .*another entry is another attempt, not a rebinding/, + ); + + // A second attempt is its own binding, and the first attempt's entry keeps its own. + const attemptTwo = publish(store, "managed-again"); + const second = bindRunEntry(store, { + runId: "run-1", + taskId: "T1", + boardTaskId: CHANNEL, + entryId: attemptTwo, + attempt: 2, + }); + assert.equal(second.recorded, true); + assert.deepEqual(bindings(store, "run-1"), [ + { sequence: 1, taskId: "T1", entryId }, + { sequence: 2, taskId: "T1", entryId: attemptTwo }, + ]); + assert.equal(store.taskRunForEntry(entryId)!.attempt, 1); + assert.equal(store.taskRunForEntry(attemptTwo)!.attempt, 2); + }); +}); + +test("a binding refuses what the store does not hold", () => { + withStore((store) => { + const entryId = publish(store, "managed"); + + // An unregistered run: there is no run state for the binding to belong to. + assert.throws( + () => + bindRunEntry(store, { runId: "run-absent", taskId: "T1", boardTaskId: CHANNEL, entryId }), + /is not registered; a managed write needs the run it belongs to/, + ); + + register(store, "run-1"); + // A task the run never froze: a run cannot adopt an entry for work it never committed to. + assert.throws( + () => bindRunEntry(store, { runId: "run-1", taskId: "T9", boardTaskId: CHANNEL, entryId }), + /never froze task T9; there is no task to bind an entry to/, + ); + + freeze(store, "run-1", "T1"); + // An entry that is not on the channel the caller names: the binding names what the board holds. + assert.throws( + () => + bindRunEntry(store, { + runId: "run-1", + taskId: "T1", + boardTaskId: "other-channel", + entryId, + }), + /no board entry .* on other-channel/, + ); + + // An entry another run already carries: one entry carries one task. + bindRunEntry(store, { runId: "run-1", taskId: "T1", boardTaskId: CHANNEL, entryId }); + register(store, "run-2"); + freeze(store, "run-2", "T2"); + assert.throws( + () => bindRunEntry(store, { runId: "run-2", taskId: "T2", boardTaskId: CHANNEL, entryId }), + /already carries task T1 of run run-1; one entry carries one task/, + ); + + // A cancelled run binds nothing further (this run's only fact so far is the cancellation). + store.appendTaskRunFact({ runId: "run-2", kind: RUN_CANCELLED_FACT, taskId: "T2" }); + assert.throws( + () => bindRunEntry(store, { runId: "run-2", taskId: "T2", boardTaskId: CHANNEL, entryId }), + /was cancelled at sequence 1/, + ); + + assert.deepEqual(bindings(store, "run-1"), [{ sequence: 1, taskId: "T1", entryId }]); + assert.deepEqual(bindings(store, "run-2"), []); + }); +}); + +test("the binding joins the transition that creates the entry", () => { + withStore((store) => { + register(store, "run-1"); + freeze(store, "run-1", "T1"); + + // One transition: the entry exists and is a managed one, or neither happened. There is no + // moment where the board holds an entry the run does not. + const entryId = store.coordinateRunWrite("run-1", (port) => { + const entry = store.putTaskBoardEntry( + { + taskId: CHANNEL, + agentId: "host", + kind: "handoff", + content: "managed", + expiresAt: NEVER, + }, + port, + ); + bindRunEntry( + store, + { runId: "run-1", taskId: "T1", boardTaskId: CHANNEL, entryId: entry.id }, + port, + ); + return entry.id; + }); + assert.deepEqual(bindings(store, "run-1"), [{ sequence: 1, taskId: "T1", entryId }]); + + // And when the transition fails, neither lands: not the binding, and not the entry. + assert.throws( + () => + store.coordinateRunWrite("run-1", (port) => { + const entry = store.putTaskBoardEntry( + { + taskId: CHANNEL, + agentId: "host", + kind: "handoff", + content: "rolled back", + expiresAt: NEVER, + }, + port, + ); + bindRunEntry( + store, + { runId: "run-1", taskId: "T1", boardTaskId: CHANNEL, entryId: entry.id, attempt: 3 }, + port, + ); + throw new Error("the harness died before the round could record its own start"); + }), + /before the round could record its own start/, + ); + assert.deepEqual(bindings(store, "run-1"), [{ sequence: 1, taskId: "T1", entryId }]); + assert.equal( + store.readTaskBoard({ taskId: CHANNEL, limit: 50 }).entries.length, + 1, + "the rolled-back entry is not on the board either", + ); + }); +}); + +test("the routing rule sends a managed entry to its run and leaves an unmanaged one alone", () => { + withStore((store) => { + register(store, "run-1"); + freeze(store, "run-1", "T1"); + const managed = publish(store, "managed"); + const loose = publish(store, "loose", "other-channel"); + bindRunEntry(store, { runId: "run-1", taskId: "T1", boardTaskId: CHANNEL, entryId: managed }); + + const claimedLoose = coordinatedEntryWrite(store, { + verb: "claim", + entryId: loose, + actorId: "worker-one", + apply: () => + store.claimTaskBoardEntry({ + taskId: "other-channel", + entryId: loose, + agentId: "worker-one", + leaseSeconds: 60, + }), + }); + assert.equal(claimedLoose.claimedBy, "worker-one"); + assert.deepEqual( + store.taskRunFacts("run-1").map((fact) => fact.kind), + [ENTRY_BOUND_FACT], + "an unmanaged entry takes the path it always took and writes no run fact", + ); + + const claimed = coordinatedEntryWrite(store, { + verb: "claim", + entryId: managed, + actorId: "worker-two", + apply: () => + store.claimTaskBoardEntry({ + taskId: CHANNEL, + entryId: managed, + agentId: "worker-two", + leaseSeconds: 60, + }), + }); + assert.equal(claimed.claimedBy, "worker-two"); + const transition = store.taskRunFacts("run-1").at(-1)!; + assert.equal(transition.kind, "board-claim"); + assert.equal(transition.entryId, managed); + assert.equal(JSON.parse(transition.payload!).actorId, "worker-two"); + + // The routing rule reads the binding rather than trusting the caller: a caller that calls the + // board verb itself instead of going through it is refused by the store. + assert.throws( + () => + store.claimTaskBoardEntry({ + taskId: CHANNEL, + entryId: managed, + agentId: "worker-three", + leaseSeconds: 60, + }), + /go through the run's coordinated transition/, + ); + }); +}); diff --git a/tests/integration/ooo-managed-fence.test.ts b/tests/integration/ooo-managed-fence.test.ts new file mode 100644 index 00000000..11464ce8 --- /dev/null +++ b/tests/integration/ooo-managed-fence.test.ts @@ -0,0 +1,216 @@ +/** + * Two fencing properties the persistence contract asks for, tested through the public surface: + * + * - concurrent readers of the same ready task: exactly one gets a legal claim; + * - a generic board operation cannot move a managed round's own state. + * + * Both are checked by what the store ends up holding and what the round then reads, not by what + * a call returns. What is deliberately not asserted here is which task `next()` selects after a + * claim: that is the coordinator's scheduling policy, and the design puts dispatch order in the + * coordinator rather than in the fence. The second test also accepts either outcome of the + * generic call - a refusal is correct too - because the property is that the round's owner, + * attempt and acceptance do not change underneath it. + */ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import test from "node:test"; + +import { NmgStore } from "../../src/core/store.ts"; +import { + BoardAdmission, + type PatchTaskSpec, + type ProbePlan, +} from "../../src/integration/ooo-board.ts"; + +const plan: ProbePlan = [ + ["A", "", [], "isolated-artifact", null, null], + ["B", "", [], "isolated-artifact", null, null], +]; +const specs: Record = { + A: { + instruction: "A.", + files: { "a.ts": "export const a = 1;\n" }, + editable: ["a.ts"], + verify: async () => "accept", + }, + B: { + instruction: "B.", + files: { "b.ts": "export const b = 1;\n" }, + editable: ["b.ts"], + verify: async () => "accept", + }, +}; + +/** Raw rows, read with a separate handle: the fence is judged on what is in the file. */ +function rows(path: string, sql: string): Record[] { + const reader = new DatabaseSync(path, { readOnly: true }); + try { + return reader.prepare(sql).all() as Record[]; + } finally { + reader.close(); + } +} + +/** A scratch directory whose stores are closed before it is removed. One hook, not two: on + * Windows an open handle makes the removal fail instead of the assertion, and hooks run in + * registration order. */ +function scratch(t: { after: (fn: () => void) => void }) { + const directory = mkdtempSync(join(tmpdir(), "nmg-managed-fence-")); + const opened: BoardAdmission[] = []; + t.after(() => { + for (const gate of opened) gate.close(); + rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + }); + return { + directory, + open: (name: string, options: { runId?: string } = {}): BoardAdmission => { + const gate = new BoardAdmission(join(directory, name), plan, specs, options); + opened.push(gate); + return gate; + }, + }; +} + +test("one reader of the same ready task is given the claim, the second is refused", (t) => { + const { directory, open } = scratch(t); + const database = join(directory, "claim.sqlite"); + const gate = open("claim.sqlite", { runId: "run-claim" }); + + assert.equal(gate.claim("A", "worker-one").owner, "worker-one"); + const entryId = String( + rows( + database, + "SELECT entry_id FROM ooo_probe_task_view WHERE run_id='run-claim' AND id='A'", + )[0]!.entry_id, + ); + + // The second reader of the same ready task is refused rather than handed the same work. + assert.throws(() => gate.claim("A", "worker-two")); + assert.equal( + rows(database, "SELECT owner FROM ooo_probe_task_view WHERE run_id='run-claim' AND id='A'")[0]! + .owner, + "worker-one", + "the claim the store holds is the first one", + ); + + // And the refusal is not the round's own bookkeeping. A second reader that reaches the board + // directly - the shape another process has, and the one the round's own check cannot see - + // must lose the same CAS. Without this the property holds only because one caller asked + // nicely, and the store would hand the same live claim to two readers. + const store = new NmgStore(database); + try { + assert.throws( + () => store.claimTaskBoardEntry({ taskId: gate.channel, entryId, agentId: "worker-two" }), + /already claimed by worker-one/u, + "the store refuses a live claim held by another reader", + ); + assert.equal( + rows( + database, + "SELECT owner FROM ooo_probe_task_view WHERE run_id='run-claim' AND id='A'", + )[0]!.owner, + "worker-one", + "and the direct attempt did not reassign the round's own claim either", + ); + } finally { + store.close(); + } +}); + +test("a claim the board retires inside the verification window cannot be committed", async (t) => { + const { directory, open } = scratch(t); + const database = join(directory, "retired.sqlite"); + const gate = open("retired.sqlite", { runId: "run-retired" }); + const ticket = gate.claim("A", "worker-one"); + const entryId = String( + rows( + database, + "SELECT entry_id FROM ooo_probe_task_view WHERE run_id='run-retired' AND id='A'", + )[0]!.entry_id, + ); + + // The work is real and the host would accept it: the round is one transaction away from + // committing. What happens in between is another writer retiring the entry the round holds - + // verification is await-capable, which is exactly the window the design fences. + const artifact = JSON.stringify({ + digest: ticket.patch!.digest, + files: [{ path: "a.ts", content: "export const a = 2;\n" }], + }); + const result = gate.putTaskBoardEntry({ + taskId: gate.channel, + agentId: "worker-one", + kind: "result", + content: JSON.stringify({ ticket, artifact }), + expiresAt: new Date(gate.now + 86_400).toISOString(), + }); + gate.afterVerify = async () => { + const store = new NmgStore(database); + try { + store.resolveTaskBoardEntry({ + taskId: gate.channel, + entryId, + agentId: "outsider", + resolution: "this entry is mine to close", + }); + } finally { + store.close(); + } + }; + + // The commit is where the claim is re-checked. A generic write cannot make the round ACCEPT + // anything - the board's own verbs are how an Agent works a handoff, so the fence is not that + // the write is impossible; it is that the decision taken before the wait is not applied after + // it. The retired attempt fails closed instead, and nothing is accepted on its behalf. + assert.equal( + await gate.submit(result.id), + "stale", + "an attempt whose claim was retired while it verified is stale, not committed", + ); + assert.deepEqual(gate.accepted(), {}, "a retired attempt accepts nothing"); + const after = rows( + database, + "SELECT owner, attempt, artifact FROM ooo_probe_task_view WHERE run_id='run-retired' AND id='A'", + )[0]!; + assert.equal(after.owner, "worker-one", "and the round's own claim is still its own fact"); + assert.equal(after.attempt, 1, "the outsider's resolve opened no attempt either"); + assert.equal(after.artifact, null, "no artifact bytes were written by the retired attempt"); +}); + +test("a generic board claim on the round's entry leaves the round's own state alone", (t) => { + const { directory, open } = scratch(t); + const database = join(directory, "fence.sqlite"); + const gate = open("fence.sqlite", { runId: "run-fence" }); + gate.claim("A", "worker-one"); + + const entryId = String( + rows( + database, + "SELECT entry_id FROM ooo_probe_task_view WHERE run_id='run-fence' AND id='A'", + )[0]!.entry_id, + ); + assert.notEqual(entryId, "null", "the round's task holds a board entry to attack"); + + // The generic board is a second connection to the same file, which is also the multi-process + // shape: whichever it decides, the round must still be the owner of its own row. + const store = new NmgStore(database); + try { + try { + store.claimTaskBoardEntry({ taskId: gate.channel, entryId, agentId: "impostor" }); + } catch { + // A refusal is an acceptable answer; the assertions below are what the contract is about. + } + const after = rows( + database, + "SELECT owner, attempt FROM ooo_probe_task_view WHERE run_id='run-fence' AND id='A'", + )[0]!; + assert.equal(after.owner, "worker-one", "the round's claim is its own fact"); + assert.equal(after.attempt, 1, "the generic claim opened no attempt on the round's task"); + assert.deepEqual(gate.accepted(), {}, "no generic write makes the round accept anything"); + assert.equal(gate.cancelled(), null, "and nothing terminal was written on the round's behalf"); + } finally { + store.close(); + } +}); diff --git a/tests/integration/ooo-managed-write.test.ts b/tests/integration/ooo-managed-write.test.ts new file mode 100644 index 00000000..96370f8a --- /dev/null +++ b/tests/integration/ooo-managed-write.test.ts @@ -0,0 +1,338 @@ +/** + * The managed-entry write path: a board entry a run has adopted cannot be moved beside its run. + * + * Three properties, all of them checked by what the store ends up holding rather than by what a + * call returns: + * + * - a direct board verb on a managed entry is refused, and an entry no run manages takes the + * same path it always did; + * - a coordinated write lands the board transition and the run's own fact in one transaction, + * and a failure inside it leaves neither; + * - a cancelled run accepts no further lifecycle writes, and the entry it adopted does not move. + * + * The ordinary board is deliberately not re-tested here: `tests/core/task-board-*.test.ts` and + * `tests/cli/service.test.ts` drive the same verbs on unmanaged entries and are the regression + * evidence that the guard did not reach them. + */ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { NmgStore } from "../../src/core/store.ts"; +import { NmgService } from "../../src/cli/service.ts"; +import { + RUN_CANCELLED_FACT, + coordinatedBoardWrite, + managedTransitionKind, + managedWriteRefusal, +} from "../../src/integration/task-coordinator.ts"; +import { removeTempDirectory } from "../helpers/temp-directory.ts"; +import { stripProviderEnv } from "../helpers/test-env.ts"; + +// The daemon case below runs an in-process NmgService, which inherits process.env; keep recall +// lexical (tests/helpers/test-env.ts), as every daemon-spawning test must. +stripProviderEnv(); + +/** Windows can hold a handle to a just-closed store for a few milliseconds. */ +const REMOVE_TEMP_TREE = { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }; +const NEVER = new Date(Date.now() + 86_400_000).toISOString(); +const CHANNEL = "board"; + +function withStore(run: (store: NmgStore) => void): void { + const directory = mkdtempSync(join(tmpdir(), "nmg-managed-write-")); + const store = new NmgStore(join(directory, "test.sqlite")); + try { + run(store); + } finally { + store.close(); + rmSync(directory, REMOVE_TEMP_TREE); + } +} + +function publish(store: NmgStore, content: string, taskId = CHANNEL): string { + return store.putTaskBoardEntry({ + taskId, + agentId: "host", + kind: "handoff", + content, + expiresAt: NEVER, + }).id; +} + +/** Register a run, freeze one task, and adopt an entry as that task's own. */ +function adopt(store: NmgStore, runId: string, taskId: string, entryId: string): void { + store.registerTaskRun({ + runId, + planDigest: "plan-a", + policy: "checks-a", + revision: "v1", + retention: "keep:evidence", + }); + store.freezeTaskRunTask({ + runId, + taskId, + position: 0, + revision: "v1", + input: `${taskId} input`, + dependencies: [], + effect: "isolated-artifact", + }); + store.appendTaskRunFact({ runId, kind: "entry-bound", taskId, entryId }); +} + +function claim( + store: NmgStore, + entryId: string, + agentId = "worker-one", + taskId = CHANNEL, +): unknown { + return store.claimTaskBoardEntry({ taskId, entryId, agentId, leaseSeconds: 600 }); +} + +function runFacts(store: NmgStore, runId: string): Array<{ kind: string; sequence: number }> { + return store.taskRunFacts(runId).map((fact) => ({ + kind: fact.kind, + sequence: fact.sequence, + })); +} + +test("a direct board verb cannot move an entry a run has adopted", () => { + withStore((store) => { + const managed = publish(store, "managed"); + // A second handoff in the same channel waits for the first (reply-gated serial handoff), so the + // ordinary entry the guard must not touch lives in a channel of its own. + const ordinary = publish(store, "ordinary", "other"); + adopt(store, "run-1", "T1", managed); + + assert.throws( + () => claim(store, managed), + /go through the run's coordinated transition/, + "a managed entry's lifecycle write belongs to the run, not to a direct verb", + ); + const unchanged = store.getTaskBoardEntryById(CHANNEL, managed)!; + assert.equal(unchanged.claimedBy, null); + assert.equal(unchanged.status, "open"); + + // The same verb on an entry no run manages is the path it always was: no transaction, no + // lookup result, nothing to coordinate. + assert.equal( + (claim(store, ordinary, "worker-one", "other") as { claimedBy: string | null }).claimedBy, + "worker-one", + ); + assert.equal(store.getTaskBoardEntryById("other", ordinary)!.claimedBy, "worker-one"); + }); +}); + +test("a coordinated write lands the board transition and the run's fact together", () => { + withStore((store) => { + const entryId = publish(store, "managed"); + adopt(store, "run-1", "T1", entryId); + + const outcome = coordinatedBoardWrite(store, { + runId: "run-1", + entryId, + verb: "claim", + actorId: "worker-one", + apply: () => claim(store, entryId), + }); + assert.equal((outcome.entry as { claimedBy: string }).claimedBy, "worker-one"); + assert.equal(store.getTaskBoardEntryById(CHANNEL, entryId)!.claimedBy, "worker-one"); + + const fact = store + .taskRunFacts("run-1") + .find((f) => f.kind === managedTransitionKind("claim"))!; + assert.equal(fact.entryId, entryId, "the transition is recorded against the entry it moved"); + assert.equal(fact.taskId, "T1"); + assert.equal(fact.sequence, outcome.fact.sequence); + assert.deepEqual( + JSON.parse(fact.payload!), + { actorId: "worker-one", status: "open" }, + "the log keeps which agent moved it and to what state, since the board only keeps the state", + ); + + // A retry of the same transition on the same attempt is the same fact, not a second one. + const retry = coordinatedBoardWrite(store, { + runId: "run-1", + entryId, + verb: "claim", + actorId: "worker-one", + apply: () => claim(store, entryId), + }); + assert.deepEqual(retry.fact, { sequence: fact.sequence, recorded: false }); + assert.deepEqual(runFacts(store, "run-1"), [ + { kind: "entry-bound", sequence: 1 }, + { kind: "board-claim", sequence: 2 }, + ]); + }); +}); + +test("a failure inside the coordinated write leaves neither the transition nor the fact", () => { + withStore((store) => { + const entryId = publish(store, "managed"); + adopt(store, "run-1", "T1", entryId); + + assert.throws( + () => + coordinatedBoardWrite(store, { + runId: "run-1", + entryId, + verb: "claim", + actorId: "worker-one", + apply: () => { + claim(store, entryId); + // The board transition already happened on this connection; the transition as a whole + // still fails, and the store decides that nothing commits. + throw new Error("the check result arrived after the commit window closed"); + }, + }), + /after the commit window closed/, + ); + assert.equal( + store.getTaskBoardEntryById(CHANNEL, entryId)!.claimedBy, + null, + "the claim does not survive a transition that failed after it", + ); + assert.deepEqual(runFacts(store, "run-1"), [{ kind: "entry-bound", sequence: 1 }]); + }); +}); + +test("a cancelled run takes no further lifecycle writes on what it adopted", () => { + withStore((store) => { + const entryId = publish(store, "managed"); + adopt(store, "run-1", "T1", entryId); + store.appendTaskRunFact({ runId: "run-1", kind: RUN_CANCELLED_FACT, taskId: "T1" }); + + const refusal = managedWriteRefusal(store, "run-1"); + assert.match(refusal!, /was cancelled at sequence 2/); + + assert.throws( + () => + coordinatedBoardWrite(store, { + runId: "run-1", + entryId, + verb: "claim", + actorId: "worker-one", + apply: () => claim(store, entryId), + }), + /was cancelled at sequence 2; its managed entries take no further lifecycle writes/, + ); + assert.equal(store.getTaskBoardEntryById(CHANNEL, entryId)!.claimedBy, null); + assert.deepEqual(runFacts(store, "run-1"), [ + { kind: "entry-bound", sequence: 1 }, + { kind: RUN_CANCELLED_FACT, sequence: 2 }, + ]); + }); +}); + +test("a coordinated write refuses an entry that is not this run's", () => { + withStore((store) => { + const entryId = publish(store, "managed"); + const loose = publish(store, "unadopted"); + adopt(store, "run-1", "T1", entryId); + store.registerTaskRun({ + runId: "run-2", + planDigest: "plan-a", + policy: "checks-a", + revision: "v1", + retention: "keep:evidence", + }); + + assert.throws( + () => + coordinatedBoardWrite(store, { + runId: "run-2", + entryId, + verb: "claim", + actorId: "worker-one", + apply: () => claim(store, entryId), + }), + /belongs to run run-1, not run-2/, + ); + assert.throws( + () => + coordinatedBoardWrite(store, { + runId: "run-1", + entryId: loose, + verb: "claim", + actorId: "worker-one", + apply: () => claim(store, loose), + }), + /is not adopted by a run/, + ); + // A scope for a run this store cannot name is refused before anything is written. + assert.throws( + () => store.coordinateRunWrite("run-absent", () => claim(store, entryId)), + /is not registered; a managed write needs the run it belongs to/, + ); + assert.equal(store.getTaskBoardEntryById(CHANNEL, entryId)!.claimedBy, null); + assert.equal(store.getTaskBoardEntryById(CHANNEL, loose)!.claimedBy, null); + }); +}); + +test("a daemon board verb routes a managed entry through the run's transition", async () => { + const directory = mkdtempSync(join(tmpdir(), "nmg-managed-daemon-")); + const databasePath = join(directory, "nmg.sqlite"); + const service = new NmgService({ databasePath, environment: {} }); + try { + const written = await service.invoke("taskBoard", { + action: "put", + taskId: CHANNEL, + agentId: "host", + kind: "handoff", + content: "managed", + ttlSeconds: 3600, + }); + if (written.action !== "put") throw new Error("expected a put result"); + const entryId = written.entry.id; + + // Adopt the entry from another connection to the same file: the shape the coordinator has when + // it is not the daemon's own process. + const adopter = new NmgStore(databasePath); + try { + adopt(adopter, "run-1", "T1", entryId); + } finally { + adopter.close(); + } + + const claimed = await service.invoke("taskBoard", { + action: "claim", + taskId: CHANNEL, + entryId, + agentId: "worker-one", + leaseSeconds: 600, + }); + if (claimed.action !== "claim") throw new Error("expected a claim result"); + assert.equal(claimed.entry.claimedBy, "worker-one"); + + // The daemon's own store wrote the transition into the run's log, in the transaction that moved + // the board: the fact is there with nobody else having written it. + const reader = new NmgStore(databasePath); + try { + const fact = reader + .taskRunFacts("run-1") + .find((f) => f.kind === managedTransitionKind("claim")); + assert.equal(fact?.entryId, entryId); + assert.equal(JSON.parse(fact!.payload!).actorId, "worker-one"); + } finally { + reader.close(); + } + + // A managed entry's write is the coordinator's to make, and the daemon's protocol has no verb + // for it: the same file through the store directly is refused, which is what stops a client + // from moving the entry beside its run. + const bystander = new NmgStore(databasePath); + try { + assert.throws( + () => claim(bystander, entryId, "worker-two"), + /go through the run's coordinated transition/, + ); + } finally { + bystander.close(); + } + } finally { + service.close(); + removeTempDirectory(directory); + } +}); diff --git a/tests/integration/ooo-ordinary-failure.test.ts b/tests/integration/ooo-ordinary-failure.test.ts new file mode 100644 index 00000000..94a29188 --- /dev/null +++ b/tests/integration/ooo-ordinary-failure.test.ts @@ -0,0 +1,214 @@ +/** + * G3: failure, cancellation and the ordered fallback on the ordinary blackboard path. + * + * The design's minimal discriminating cases, on the path a caller reaches with the board's own verbs. + * Every expectation here was measured against the board first, because the honest version of these + * rules is stricter than a first guess: a refused deliverable is not re-selected on its own (the + * coordinator reopens it), a live claim keeps blocking selection, and a cancellation names the lease + * it revoked. The last case is a split refusal, because the design's split rules are host-declared + * and refused by name rather than guessed. + */ +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { + BoardAdmission, + type BoardTicket, + type PatchTaskSpec, + type ProbePlan, +} from "../../src/integration/ooo-board.ts"; +import { checkRefinement, compileTaskUnits } from "../../src/integration/task-semantics.ts"; + +const REV = "input-v1"; +const IMPL = "src/ordinary.ts"; +const TESTS = "src/ordinary.test.ts"; +const baseline = { [IMPL]: "export const a = 1;\n", [TESTS]: "test('a', () => {});\n" }; + +/** B runs first; A declares it as a dependency. */ +const plan: ProbePlan = [ + ["B", REV, [], "isolated-artifact", null, null], + ["A", REV, ["B"], "isolated-artifact", null, null], +]; + +/** What the host's verification says next. A rejection is one host decision, not a board rule. */ +let verdictA: "accept" | "reject" = "accept"; +let verdictB: "accept" | "reject" = "accept"; + +const specs: Record = { + A: { + instruction: "A works.", + files: { [IMPL]: baseline[IMPL] }, + editable: [IMPL], + verify: async () => verdictA, + }, + B: { + instruction: "B works.", + files: { [TESTS]: baseline[TESTS] }, + editable: [TESTS], + verify: async () => verdictB, + }, +}; + +function open(runId: string) { + const directory = mkdtempSync(join(tmpdir(), "nmg-ordinary-failure-")); + const gate = new BoardAdmission(join(directory, "round.sqlite"), plan, specs, { runId }); + const deliver = (ticket: BoardTicket, path: string, content: string) => + JSON.stringify({ digest: ticket.inputDigest, files: [{ path, content }] }); + const submitAs = (ticket: BoardTicket, agentId: string, artifact: string) => { + const entry = gate.putTaskBoardEntry({ + taskId: gate.channel, + agentId, + kind: "result", + content: JSON.stringify({ ticket, artifact }), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }); + return gate.submit(entry.id); + }; + return { gate, deliver, submitAs }; +} + +const B_DONE = "test('a', () => {}); test('b', () => {});\n"; + +test("a deliverable the host refuses neither accepts nor releases, and the coordinator reopens it", async (t) => { + const { gate, deliver, submitAs } = open("ordinary-rejection"); + t.after(() => gate.close()); + verdictB = "reject"; + + const ticket = gate.claim("B", "worker-one") as BoardTicket; + assert.equal( + await submitAs(ticket, "worker-one", deliver(ticket, TESTS, B_DONE)), + "rejected", + "the host's verification refuses this artifact", + ); + + assert.deepEqual(gate.accepted(), {}, "a refused artifact is not an acceptance"); + assert.equal( + gate.next(), + null, + "and it is not selected again on its own: the coordinator has to reopen it", + ); + assert.throws( + () => gate.claim("A", "worker-two"), + /unfulfilled dependencies/u, + "the dependent is not released by a refused deliverable", + ); + assert.throws( + () => gate.claim("B", "worker-one"), + /task already claimed/u, + "the refused attempt still holds its claim, so a retry waits for the reopen", + ); + + // The coordinator's recovery, then the same task with an artifact the host accepts. + gate.reopen("B"); + verdictB = "accept"; + assert.equal(gate.next(), "B", "the reopened task is selectable again"); + const retry = gate.claim("B", "worker-one") as BoardTicket; + assert.equal(await submitAs(retry, "worker-one", deliver(retry, TESTS, B_DONE)), "accepted"); + assert.equal(gate.next(), "A", "accepting the dependency releases the dependent"); +}); + +test("a cancellation names the lease it revokes, and the batch behind it is refused", async (t) => { + const { gate } = open("ordinary-cancellation"); + t.after(() => gate.close()); + verdictB = "accept"; + + gate.claim("B", "worker-one"); + assert.deepEqual( + gate.cancel("operator cancelled the round"), + ["B"], + "the cancellation names the lease it revoked", + ); + assert.equal(gate.cancelled(), "operator cancelled the round"); + assert.equal(gate.next(), null, "a cancelled round selects nothing"); + + // Every claim in the batch is refused, not only the one that happened to be first. + assert.throws(() => gate.claim("A", "worker-two"), /cancelled/u); + assert.throws(() => gate.claim("B", "worker-one"), /cancelled/u); + + // The first decision is the one that took effect: a later cancel does not rewrite the reason. + assert.deepEqual(gate.cancel("something else"), [], "cancelling twice withdraws nothing new"); + assert.equal(gate.cancelled(), "operator cancelled the round"); +}); + +test("with no fusion point the plan falls back to its declared order", async (t) => { + const { gate, deliver, submitAs } = open("ordinary-ordered"); + t.after(() => gate.close()); + verdictB = "accept"; + + // Both tasks are on files the same session would touch, which is the shape a fusion pass would look + // for. There is none here, so the baseline is one task at a time in declared order. + assert.equal(gate.next(), "B", "one task is selected, not a fused pair"); + const ticketB = gate.claim("B", "worker-one") as BoardTicket; + assert.equal(gate.next(), null, "and while its claim is live nothing else is selected"); + assert.equal(await submitAs(ticketB, "worker-one", deliver(ticketB, TESTS, B_DONE)), "accepted"); + + assert.equal(gate.next(), "A"); + const ticketA = gate.claim("A", "worker-two") as BoardTicket; + assert.equal( + await submitAs(ticketA, "worker-two", deliver(ticketA, IMPL, "export const a = 2;\n")), + "accepted", + ); + assert.deepEqual(Object.keys(gate.accepted()).sort(), ["A", "B"]); + assert.equal(gate.next(), null, "and then the plan is finished"); +}); + +test("a split that drops a parent obligation is refused by name, with its location", () => { + const compiled = compileTaskUnits({ plan, specs }); + assert.equal(compiled.legal, true, "the plan compiles: " + JSON.stringify(compiled.refusals)); + + const dropped = checkRefinement(compiled, { + parent: "B", + parts: ["A"], + join: "A", + obligations: {}, + }); + assert.ok(dropped.length > 0, "a split that maps no parent obligation is refused, not accepted"); + for (const refusal of dropped) + assert.ok( + refusal.task && refusal.field && refusal.reason, + "a refusal names its location: " + JSON.stringify(refusal), + ); + assert.ok( + dropped.some((refusal) => refusal.field.startsWith("obligations.")), + "the refusal points at the obligation the split dropped: " + JSON.stringify(dropped), + ); +}); + +test("a later refusal does not withdraw the prefix that was already accepted", async (t) => { + const { gate, deliver, submitAs } = open("ordinary-prefix"); + t.after(() => gate.close()); + verdictA = "accept"; + verdictB = "accept"; + + const ticketB = gate.claim("B", "worker-one") as BoardTicket; + assert.equal(await submitAs(ticketB, "worker-one", deliver(ticketB, TESTS, B_DONE)), "accepted"); + assert.deepEqual(Object.keys(gate.accepted()), ["B"], "B is the accepted prefix"); + + // The next deliverable is refused. The accepted prefix has to survive it: a later failure is not a + // reason to un-accept work the host already accepted, and the refused task waits for its reopen. + verdictA = "reject"; + const ticketA = gate.claim("A", "worker-two") as BoardTicket; + assert.equal( + await submitAs(ticketA, "worker-two", deliver(ticketA, IMPL, "export const a = 2;\n")), + "rejected", + ); + assert.deepEqual( + Object.keys(gate.accepted()), + ["B"], + "the accepted prefix survives the later refusal", + ); + assert.equal(gate.next(), null, "and the refused task is not re-selected on its own"); + + // Reopening it and accepting it completes the plan; the prefix was never lost along the way. + gate.reopen("A"); + verdictA = "accept"; + const retry = gate.claim("A", "worker-two") as BoardTicket; + assert.equal( + await submitAs(retry, "worker-two", deliver(retry, IMPL, "export const a = 2;\n")), + "accepted", + ); + assert.deepEqual(Object.keys(gate.accepted()).sort(), ["A", "B"]); +}); diff --git a/tests/integration/ooo-ordinary-handoff.test.ts b/tests/integration/ooo-ordinary-handoff.test.ts new file mode 100644 index 00000000..a7582ecc --- /dev/null +++ b/tests/integration/ooo-ordinary-handoff.test.ts @@ -0,0 +1,132 @@ +/** + * G1 and G2 on the ordinary path: a board handoff, walked with the board's own verbs and nothing + * else, reaches the shared semantics, runs work, and has that work accepted by an independent + * verdict - and accepting it releases the dependent that was waiting for it. + * + * Nothing here touches a dedicated OoO surface: `ooo_round` has left the product tool + * directory (ledger G5). The ordinary path below is what remains - a handoff, a deliverable and + * the board's own verdict. The point of G1 is that a caller who only knows the board and the + * shared semantics can get the whole thing. The selection is + * asserted through `next()`, which is now the compiler's answer, and cross-checked against + * `deriveStatus` over the same recorded facts, so the two views are held together rather than + * assumed to agree. + */ +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { + BoardAdmission, + type BoardTicket, + type PatchTaskSpec, + type ProbePlan, +} from "../../src/integration/ooo-board.ts"; +import { + compileTaskUnits, + deriveStatus, + type RecordedFacts, +} from "../../src/integration/task-semantics.ts"; + +const IMPL = "src/ordinary.ts"; +const TESTS = "src/ordinary.test.ts"; +const baseline = { [IMPL]: "export const a = 1;\n", [TESTS]: "test('a', () => {});\n" }; + +/** B first, and A declared as waiting on it: the dependency is the thing under test. */ +const REV = "input-v1"; +const plan: ProbePlan = [ + ["B", REV, [], "isolated-artifact", null, null], + ["A", REV, ["B"], "isolated-artifact", null, null], +]; + +const specs: Record = { + A: { + instruction: "A works.", + files: { [IMPL]: baseline[IMPL] }, + editable: [IMPL], + verify: async () => "accept", + }, + B: { + instruction: "B works.", + files: { [TESTS]: baseline[TESTS] }, + editable: [TESTS], + verify: async () => "accept", + }, +}; + +test("an ordinary handoff reaches the shared semantics, runs, is accepted, and releases its dependent", async () => { + const directory = mkdtempSync(join(tmpdir(), "nmg-ordinary-handoff-")); + const gate = new BoardAdmission(join(directory, "round.sqlite"), plan, specs, { + runId: "ordinary-handoff", + }); + try { + // The semantics' view of the same plan, for the cross-check at the end. + const { units, legal, refusals } = compileTaskUnits({ plan, specs }); + assert.equal(legal, true, `the plan compiles: ${JSON.stringify(refusals)}`); + + // Before anything is accepted, the dependent is not selectable: B is the only legal action, and + // A must not appear merely because it is declared. + assert.equal(gate.next(), "B"); + assert.deepEqual( + deriveStatus(units, {}).ready, + ["B"], + "the compiler agrees that B is the only thing to do", + ); + + // The worker is the caller's own code. Here it delivers a patch whose digest is the one the + // claim froze, which is the only envelope the board accepts for a patch task. + const delivery = (taskId: string, ticket: BoardTicket, path: string, content: string) => + JSON.stringify({ digest: ticket.inputDigest, files: [{ path, content }] }); + + const ticketB = gate.claim("B", "worker-one") as BoardTicket; + const entryB = gate.putTaskBoardEntry({ + taskId: gate.channel, + agentId: "worker-one", + kind: "result", + content: JSON.stringify({ + ticket: ticketB, + artifact: delivery("B", ticketB, TESTS, "test('a', () => {}); test('b', () => {});\n"), + }), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }); + assert.equal( + await gate.submit(entryB.id), + "accepted", + "the artifact is accepted by the board's own verdict", + ); + + // G2: the acceptance is what releases A. The compiler says so over the recorded facts, and the + // board - which now answers selection with that same rule - says so too. + const accepted = gate.accepted(); + const commitB = accepted.B; + assert.ok(commitB, "B's artifact is the value A is released by"); + const facts: RecordedFacts = { + artifacts: { B: commitB }, + verdicts: { B: { digest: commitB, verdict: "accepted" } }, + revisions: { B: REV }, + sourceRevisions: { B: REV }, + }; + assert.deepEqual(deriveStatus(units, facts).ready, ["A"], "accepting B releases A"); + assert.deepEqual(deriveStatus(units, facts).accepted, ["B"]); + assert.equal(gate.next(), "A", "and the board's selection is that same answer"); + + // The released dependent runs to completion through the same ordinary path. + const ticketA = gate.claim("A", "worker-two") as BoardTicket; + const entryA = gate.putTaskBoardEntry({ + taskId: gate.channel, + agentId: "worker-two", + kind: "result", + content: JSON.stringify({ + ticket: ticketA, + artifact: delivery("A", ticketA, IMPL, "export const a = 2;\n"), + }), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }); + assert.equal(await gate.submit(entryA.id), "accepted"); + assert.deepEqual(Object.keys(gate.accepted()).sort(), ["A", "B"]); + assert.equal(gate.next(), null, "and with both accepted there is nothing left to select"); + } finally { + gate.close(); + } +}); diff --git a/tests/integration/ooo-post-commit-notification.test.ts b/tests/integration/ooo-post-commit-notification.test.ts new file mode 100644 index 00000000..2a08ca25 --- /dev/null +++ b/tests/integration/ooo-post-commit-notification.test.ts @@ -0,0 +1,97 @@ +/** + * D9: the commit result and a notification failure are two different facts. + * + * `submit()` used to return through its post-commit notification, so a subscriber that could not be + * reached made a submission that had already landed read as a failed one - and a caller retrying on + * that reading submits the same work twice. The guard is the board's (`ooo-board.ts`), so the case + * belongs here rather than in the round that used to supply the harness: the round was retired in + * `docs/decisions/implemented/2026-09-18-retire-the-round-instrument.md`, and the property it pinned + * is the board's. + */ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { BoardAdmission, type PatchTaskSpec, type ProbePlan } from "../../src/integration/ooo-board.ts"; +import { preparePatchWork } from "../../src/integration/ooo-patch.ts"; + +const TASK = "A"; +const FILE = "src/check.ts"; +const baseline = { [FILE]: "export const a = 1;\n" }; +const plan: ProbePlan = [[TASK, "", [], "isolated-artifact", null, null]]; + +/** One board-level unit, claimed and submitted the way a host does it: the store decides the verdict + * from the candidate bytes, and the host never reads a worker's claim about itself. */ +async function submitOne(board: BoardAdmission): Promise { + const spec: PatchTaskSpec = { + instruction: "work on A", + files: baseline, + editable: [FILE], + verify: async () => "accept", + }; + board.installPatchTask(TASK, spec); + const ticket = board.claim(TASK, "post-commit-test"); + assert.ok(ticket.patch, "the plan declares a patch task"); + const frozen = preparePatchWork({ + taskId: ticket.patch.taskId, + attempt: ticket.attempt, + instruction: ticket.patch.instruction, + files: ticket.patch.files, + editable: ticket.patch.editable, + visible: ticket.patch.visible, + admittedConclusions: ticket.patch.admittedConclusions, + budget: ticket.patch.budget, + limits: ticket.patch.limits, + }); + // The worker's artifact is the wire shape the host validates (`{ digest, files: [{ path, content }] }`), + // not the store's `PatchSubmission`: the host re-derives the submission from its own frozen work. + const artifact = JSON.stringify({ + digest: frozen.digest, + files: [{ path: FILE, content: `${baseline[FILE]}// A\n` }], + }); + const entry = board.putTaskBoardEntry({ + taskId: board.channel, + agentId: "post-commit-test", + kind: "result", + content: JSON.stringify({ ticket, artifact }), + expiresAt: new Date(board.now + 86_400_000).toISOString(), + }); + return board.submit(entry.id); +} + +test("a notification that fails does not turn a landed commit into a failed submission", async () => { + // The control first: the same unit with a reachable notification reports nothing, so the failure + // below is the notification and not a field that is always set. + const quiet = new BoardAdmission(":memory:", plan, {}); + try { + assert.equal(await submitOne(quiet), "accepted", "the unit is accepted when nothing fails"); + assert.equal( + quiet.lastNotificationFailure(), + null, + "a notification that arrives is not a failure", + ); + } finally { + quiet.close(); + } + + const board = new BoardAdmission(":memory:", plan, {}); + // The board runs its own post-commit bookkeeping through `afterVerify`, so only the notification + // part of "after the commit" fails: the commit itself has already happened by then. + board.afterCommit = async () => { + throw new Error("subscriber unreachable"); + }; + try { + assert.equal( + await submitOne(board), + "accepted", + "the commit landed, and the caller is told so rather than told the submission failed", + ); + assert.deepEqual(Object.keys(board.accepted()), [TASK], "the artifact is the board's now"); + assert.match( + String(board.lastNotificationFailure()), + /subscriber unreachable/u, + "the failure is reported separately instead of thrown at a caller who would then resubmit", + ); + } finally { + board.close(); + } +}); diff --git a/tests/integration/ooo-publication-invariants.test.ts b/tests/integration/ooo-publication-invariants.test.ts new file mode 100644 index 00000000..698da413 --- /dev/null +++ b/tests/integration/ooo-publication-invariants.test.ts @@ -0,0 +1,432 @@ +/** + * The design's last offline acceptance, in two directions. + * + * The clean direction: over every legal interleaving of the design's script sets, no publication the + * derived view allows is unsupported by the facts the interleaving recorded - no completion whose + * verdict judged other bytes, no completion resting on a missing, drifted, cancelled or unverified + * input, no dispatch whose inputs are not closed. + * + * The other direction is the one that makes the first mean anything: each condition is deleted from + * the checker by a mutant in `tools/mutation-teeth.ts`, and the case below that names it must fail. + * A checker whose conditions cannot be broken is a description, not a check. + */ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + DESIGN_PLAN, + DESIGN_SCRIPT_SETS, + MAX_INTERLEAVING_EVENTS, + MAX_INTERLEAVING_UNITS, + checkBudget, + checkPublications, + enumerateInterleavings, + enumerateTable, + type InterleavingEventKind, + type UnitScript, +} from "../../src/integration/task-semantics-interleavings.ts"; +import { + compileTaskUnits, + deriveStatus, + type RecordedFacts, +} from "../../src/integration/task-semantics.ts"; +import { nextTask, type DispatchTask } from "../../src/integration/ooo-execution.ts"; +import type { PatchTaskSpec } from "../../src/integration/ooo-board.ts"; + +const spec = (): PatchTaskSpec => ({ + instruction: "Rename byId to planIndex in nextTask only.", + files: { "src/integration/ooo-execution.ts": "export const planIndex = 1;\n" }, + editable: ["src/integration/ooo-execution.ts"], + verify: async () => "accept", +}); + +const compiled = compileTaskUnits({ plan: DESIGN_PLAN, specs: { P: spec() } }); +const factsWith = (overrides: Partial): RecordedFacts => ({ + artifacts: {}, + verdicts: {}, + revisions: {}, + sourceRevisions: {}, + cancellations: [], + ...overrides, +}); + +test("no publication over the design's interleavings is unsupported by its own facts", () => { + const report = enumerateTable({ + plan: DESIGN_PLAN, + specs: { P: spec() }, + sets: DESIGN_SCRIPT_SETS, + }); + assert.equal(report.refused, undefined); + // The enumeration is non-vacuous: it really walked the merges, and the sets really publish things. + assert.equal(report.sets, DESIGN_SCRIPT_SETS.length); + assert.ok(report.interleavings > 20, `expected many interleavings, got ${report.interleavings}`); + assert.ok(report.completions > 0, "some interleaving publishes a completed unit"); + assert.ok(report.dispatches > 0, "some interleaving dispatches the dependent"); + assert.deepEqual( + report.findings.map( + (finding) => `${finding.publication}:${finding.unit}:${finding.obligation}`, + ), + [], + "every publication is supported by the recorded facts", + ); + assert.deepEqual( + report.budgetFindings.map( + (finding) => `${finding.property}:${finding.budget}:${finding.unit}`, + ), + [], + "every declared budget published a view its own budget allows", + ); + assert.deepEqual(report.budgets, [1, 2], "the design's shapes are walked at one slot and at two"); +}); + +test("a declared budget publishes more than one slot can, and never a claimed task", () => { + // Two independent units plus a dependent: the second slot is what a one-slot run cannot offer. + const wide: CompileInput["plan"] = [ + ["P", "rev-1", [], "isolated-artifact", null, null], + ["Q", "rev-1", [], "isolated-artifact", null, null], + ["D", "rev-1", ["P"], "read-only", null, null], + ]; + const scripts: readonly UnitScript[] = [ + { unit: "P", events: ["claim"] }, + { unit: "Q", events: ["deliver", "judge-accept"] }, + ]; + const report = enumerateInterleavings({ plan: wide, specs: { P: spec() }, scripts }); + assert.equal(report.refused, undefined); + assert.deepEqual(report.budgetFindings, [], "no budget offered a claimed task or dropped a candidate"); + assert.ok( + report.widened > 0, + "the two-slot view published a task the one-slot view did not, which is what the budget buys", + ); + assert.deepEqual( + report.findings.map((finding) => `${finding.unit}:${finding.obligation}`), + [], + "and what it published is still supported by the recorded facts", + ); + // The claimed unit is never one of them: at the prefix where P is claimed, only Q is on offer. + const oneSlot = enumerateInterleavings({ + plan: wide, + specs: { P: spec() }, + scripts, + budgets: [1], + }); + assert.equal(oneSlot.widened, 0, "a single budget has nothing to widen against"); + assert.ok(report.dispatches > oneSlot.dispatches, "the second budget really published something"); +}); + +test("the budget properties fire on a hand-built view, so deleting them cannot pass quietly", () => { + const claimed = checkBudget({ budget: 2, ready: ["P", "Q"], claimed: ["P"], atStep: 3 }); + assert.deepEqual( + claimed.map((finding) => `${finding.property}:${finding.unit}:${finding.atStep}`), + ["claimed-task-is-not-startable:P:3"], + "a claimed task in the startable set is the property a second worker would break", + ); + const dropped = checkBudget({ budget: 2, ready: [], claimed: [], smallerReady: ["Q"] }); + assert.deepEqual( + dropped.map((finding) => `${finding.property}:${finding.unit}`), + ["a-bigger-budget-keeps-the-smaller-candidates:Q"], + "a bigger budget adds candidates, it does not replace them", + ); + assert.deepEqual(checkBudget({ budget: 2, ready: ["Q"], claimed: ["P"], smallerReady: [] }), []); + const refused = enumerateInterleavings({ plan: DESIGN_PLAN, specs: { P: spec() }, scripts: [], budgets: [0] }); + assert.match(refused.refused!, /budget 0 is not a positive integer/); +}); + +test("the merge enumerates every legal order, not one of them", () => { + // P has three events in this order and D two: the merges are the multinomial 5!/(3!2!) = 10. + const report = enumerateInterleavings({ + plan: DESIGN_PLAN, + specs: { P: spec() }, + scripts: [ + { unit: "P", events: ["deliver", "judge-accept", "redeliver"] }, + { unit: "D", events: ["deliver", "judge-accept"] }, + ], + }); + assert.equal(report.refused, undefined); + assert.equal(report.interleavings, 10); + assert.equal(report.steps, 10 * 5, "every prefix of every interleaving is checked"); +}); + +test("the enumeration refuses what it cannot bound, rather than growing", () => { + const many = [ + ["P", "rev-1", [], "isolated-artifact", null, null], + ["Q", "rev-1", [], "isolated-artifact", null, null], + ["R", "rev-1", [], "isolated-artifact", null, null], + ["S", "rev-1", [], "isolated-artifact", null, null], + ["T", "rev-1", [], "isolated-artifact", null, null], + ] as const; + const over = enumerateInterleavings({ + plan: [...many], + specs: { P: spec() }, + scripts: [{ unit: "P", events: ["deliver"] }], + }); + assert.match(over.refused!, new RegExp(`cap of ${MAX_INTERLEAVING_UNITS}`)); + + const tooManyEvents = enumerateInterleavings({ + plan: DESIGN_PLAN, + specs: { P: spec() }, + scripts: [ + { unit: "P", events: ["deliver", "judge-accept", "redeliver", "revision-drift"] }, + { unit: "D", events: ["deliver", "judge-accept", "redeliver"] }, + ], + }); + assert.match(tooManyEvents.refused!, new RegExp(`cap of ${MAX_INTERLEAVING_EVENTS}`)); + + const twoScripts = enumerateInterleavings({ + plan: DESIGN_PLAN, + specs: { P: spec() }, + scripts: [ + { unit: "P", events: ["deliver"] }, + { unit: "P", events: ["judge-accept"] }, + ], + }); + assert.match(twoScripts.refused!, /two scripts/); + + const unknown = enumerateInterleavings({ + plan: DESIGN_PLAN, + specs: { P: spec() }, + scripts: [{ unit: "Z", events: ["deliver"] }], + }); + assert.match(unknown.refused!, /names a unit the plan does not have/); +}); + +/** + * One case per condition in the checker. Each is a fact set a run can really record, and what the + * checker has to say about publishing it. + */ +const violationCases: ReadonlyArray<{ + name: string; + facts: RecordedFacts; + publications: { dispatches?: readonly string[]; completions?: readonly string[] }; + obligation: string; + publication: "dispatch" | "completion"; +}> = [ + { + name: "a completion with no verdict", + facts: factsWith({ artifacts: { P: "P-artifact-v0" } }), + publications: { completions: ["P"] }, + obligation: "explicit-acceptance", + publication: "completion", + }, + { + name: "a completion whose verdict judged other bytes", + facts: factsWith({ + artifacts: { P: "P-artifact-v0" }, + verdicts: { P: { digest: "P-artifact-v1", verdict: "accepted" } }, + }), + publications: { completions: ["P"] }, + obligation: "explicit-acceptance", + publication: "completion", + }, + { + name: "a completion with no bytes", + facts: factsWith({ verdicts: { P: { digest: "P-artifact-v0", verdict: "accepted" } } }), + publications: { completions: ["P"] }, + obligation: "artifact-handoff", + publication: "completion", + }, + { + name: "a completion of a cancelled unit", + facts: factsWith({ + artifacts: { P: "P-artifact-v0" }, + verdicts: { P: { digest: "P-artifact-v0", verdict: "accepted" } }, + cancellations: ["P"], + }), + publications: { completions: ["P"] }, + obligation: "stoppable-voidable", + publication: "completion", + }, + { + name: "a completion resting on a cancelled input", + facts: factsWith({ + artifacts: { P: "P-artifact-v0", D: "D-artifact-v0" }, + verdicts: { + P: { digest: "P-artifact-v0", verdict: "accepted" }, + D: { digest: "D-artifact-v0", verdict: "accepted" }, + }, + cancellations: ["P"], + }), + publications: { completions: ["D"] }, + obligation: "stoppable-voidable", + publication: "completion", + }, + { + name: "a completion resting on an input that has no bytes", + facts: factsWith({ + artifacts: { D: "D-artifact-v0" }, + verdicts: { D: { digest: "D-artifact-v0", verdict: "accepted" } }, + }), + publications: { completions: ["D"] }, + obligation: "input-closure", + publication: "completion", + }, + { + name: "a completion resting on an input that drifted", + facts: factsWith({ + artifacts: { P: "P-artifact-v0", D: "D-artifact-v0" }, + verdicts: { + P: { digest: "P-artifact-v0", verdict: "accepted" }, + D: { digest: "D-artifact-v0", verdict: "accepted" }, + }, + revisions: { P: "rev-1" }, + sourceRevisions: { P: "rev-2" }, + }), + publications: { completions: ["D"] }, + obligation: "input-closure", + publication: "completion", + }, + { + name: "a completion resting on an input whose verdict is stale", + facts: factsWith({ + artifacts: { P: "P-artifact-v1", D: "D-artifact-v0" }, + verdicts: { + P: { digest: "P-artifact-v0", verdict: "accepted" }, + D: { digest: "D-artifact-v0", verdict: "accepted" }, + }, + }), + publications: { completions: ["D"] }, + obligation: "input-closure", + publication: "completion", + }, + { + name: "a dispatch whose input is not accepted", + facts: factsWith({ + artifacts: { P: "P-artifact-v0" }, + verdicts: { P: { digest: "P-artifact-v0", verdict: "rejected" } }, + }), + publications: { dispatches: ["D"] }, + obligation: "input-closure", + publication: "dispatch", + }, + { + name: "a dispatch whose input has no bytes", + facts: factsWith({}), + publications: { dispatches: ["D"] }, + obligation: "input-closure", + publication: "dispatch", + }, + { + name: "a dispatch of a cancelled unit", + facts: factsWith({ cancellations: ["D"] }), + publications: { dispatches: ["D"] }, + obligation: "stoppable-voidable", + publication: "dispatch", + }, +]; + +for (const testCase of violationCases) { + test(`the checker reports ${testCase.name}`, () => { + const violations = checkPublications(compiled.units, testCase.facts, testCase.publications, 3); + const match = violations.find( + (violation) => + violation.obligation === testCase.obligation && + violation.publication === testCase.publication, + ); + assert.ok( + match, + `expected ${testCase.obligation} on a ${testCase.publication}; got ${JSON.stringify(violations)}`, + ); + assert.equal(match.atStep, 3, "the finding says which prefix it was published at"); + }); +} + +test("the checker accepts what the interleavings actually record", () => { + // The mirror of the cases above: the same shapes, recorded in an order that supports them. + const facts = factsWith({ + artifacts: { P: "P-artifact-v0", D: "D-artifact-v0" }, + verdicts: { + P: { digest: "P-artifact-v0", verdict: "accepted" }, + D: { digest: "D-artifact-v0", verdict: "accepted" }, + }, + revisions: { P: "rev-1", D: "rev-1" }, + sourceRevisions: { P: "rev-1", D: "rev-1" }, + externalReady: [], + claimed: ["D"], + }); + assert.deepEqual( + checkPublications(compiled.units, facts, { dispatches: ["D"], completions: ["P", "D"] }), + [], + "a cancelled-free, current, verdict-bound record publishes cleanly", + ); +}); + +test("a cancelled task is not dispatched, and nothing reads one as a closed input", () => { + // The enumerator above found this: acceptance already refused a cancelled task, while the + // eligibility rule could not see the cancellation at all, so a cancelled task stayed selectable. + const cancelledDependency = factsWith({ cancellations: ["P"] }); + assert.deepEqual( + deriveStatus(compiled.units, cancelledDependency).ready, + [], + "a cancelled unit is not handed out", + ); + const dependencyAccepted = factsWith({ + artifacts: { P: "P-artifact-v0" }, + verdicts: { P: { digest: "P-artifact-v0", verdict: "accepted" } }, + revisions: { P: "rev-1" }, + sourceRevisions: { P: "rev-1" }, + cancellations: ["P"], + }); + assert.deepEqual( + deriveStatus(compiled.units, dependencyAccepted).ready, + [], + "a cancelled input is not a closed input, even when its bytes carry a verdict", + ); + const live = factsWith({ + artifacts: { P: "P-artifact-v0" }, + verdicts: { P: { digest: "P-artifact-v0", verdict: "accepted" } }, + revisions: { P: "rev-1" }, + sourceRevisions: { P: "rev-1" }, + }); + assert.deepEqual(deriveStatus(compiled.units, live).ready, ["D"], "the same facts without it do"); +}); + +test("nextTask refuses a task marked cancelled, whatever else the caller set", () => { + // `dispatchTasks` fills both fields from the same recorded fact, so these states do not arise from + // it - but eligibility is the public contract of this function, and a caller that builds the + // dispatch view itself must not get a cancelled task handed out either. + const task = (id: string, extra: Partial = {}): DispatchTask => ({ + id, + effect: "read-only", + sourceVersion: "rev-1", + observedVersion: "rev-1", + dependencies: [], + accepted: false, + claimed: false, + externalReady: false, + ...extra, + }); + assert.equal( + nextTask([task("A", { cancelled: true })]), + null, + "a cancelled task is not selected", + ); + assert.equal( + nextTask([task("A", { accepted: true, cancelled: true })]), + null, + "not even when the caller also marked it accepted", + ); + assert.equal( + nextTask([task("P", { accepted: true, cancelled: true }), task("D", { dependencies: ["P"] })]), + null, + "and a cancelled task is not a closed input for a dependent", + ); + assert.equal(nextTask([task("A")]), "A", "an otherwise identical uncancelled task is selected"); +}); + +test("every event kind the scripts use is one the recorder knows", () => { + const used = new Set( + DESIGN_SCRIPT_SETS.flatMap((set) => set.flatMap((script) => [...script.events])), + ); + // A silently ignored kind would make an interleaving cheaper than it is. + assert.deepEqual([...used].sort(), [ + "cancel", + "claim", + "deliver", + "external-ready", + "judge-accept", + "judge-reject", + "judge-stale-digest", + "judge-undecidable", + "redeliver", + "revision-drift", + ]); +}); diff --git a/tests/integration/ooo-read-paths-agree.test.ts b/tests/integration/ooo-read-paths-agree.test.ts new file mode 100644 index 00000000..e04090c5 --- /dev/null +++ b/tests/integration/ooo-read-paths-agree.test.ts @@ -0,0 +1,138 @@ +/** + * The design's D5: the owner's borrowed view and the offline read-only path must agree on the same + * facts at the same evaluation time. Each path is covered for its own properties elsewhere; what + * this file adds is that they do not disagree, so "status says accepted" and "the dependency view + * says accepted" cannot come from two different rules or two different moments. + */ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { + BoardAdmission, + openRoundQuery, + type BoardTicket, + type PatchTaskSpec, + type ProbePlan, +} from "../../src/integration/ooo-board.ts"; + +const plan: ProbePlan = [ + ["A", "", [], "isolated-artifact", null, null], + ["B", "", [], "isolated-artifact", null, null], +]; +const specs: Record = { + A: { + instruction: "A.", + files: { "a.ts": "export const a = 1;\n" }, + editable: ["a.ts"], + verify: async () => "accept", + }, + B: { + instruction: "B.", + files: { "b.ts": "export const b = 1;\n" }, + editable: ["b.ts"], + verify: async () => "accept", + }, +}; + +function scratch(t: { after: (fn: () => void) => void }) { + const directory = mkdtempSync(join(tmpdir(), "nmg-read-paths-")); + const opened: BoardAdmission[] = []; + t.after(() => { + for (const gate of opened) gate.close(); + rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + }); + return { + directory, + open: (name: string, options: { runId?: string } = {}): BoardAdmission => { + const gate = new BoardAdmission(join(directory, name), plan, specs, options); + opened.push(gate); + return gate; + }, + }; +} + +/** The only way an artifact becomes accepted: the claimed owner's own board entry, verified by the + * host. The two paths are compared on a NON-EMPTY answer, because two readers that both say + * "nothing accepted" agree without either of them reading anything. */ +async function accept( + gate: BoardAdmission, + ticket: BoardTicket, + task: string, + agent: string, +): Promise { + const artifact = JSON.stringify({ + digest: ticket.patch!.digest, + files: Object.entries(specs[task]!.files).map(([path, content]) => ({ + path, + content: `${content}// candidate for ${task}\n`, + })), + }); + const entry = gate.putTaskBoardEntry({ + taskId: gate.channel, + agentId: agent, + kind: "result", + content: JSON.stringify({ ticket, artifact }), + expiresAt: new Date(gate.now + 86_400_000).toISOString(), + }); + assert.equal(await gate.submit(entry.id), "accepted"); +} + +test("the owner's view and the offline reader report the same facts", async (t) => { + const { directory, open } = scratch(t); + const database = join(directory, "agree.sqlite"); + const gate = open("agree.sqlite", { runId: "run-agree" }); + const ticketA = gate.claim("A", "worker-one"); + + // While the round is live: nothing is accepted on either path, and neither reports a terminal + // decision. An artifact that exists is not acceptance, and a claim is not a verdict. + const live = openRoundQuery(database, "run-agree"); + try { + assert.deepEqual(gate.accepted(), {}, "nothing is accepted yet"); + assert.deepEqual(live.port.accepted(), gate.accepted(), "both paths agree while live"); + assert.equal(live.port.cancelled(), null, "no terminal reason yet"); + assert.equal(live.port.cancelled(), gate.cancelled(), "both paths agree on that too"); + } finally { + live.close(); + } + + // The accepting direction, so agreement is not the agreement of two empty answers: the owner's + // view and a freshly opened read-only one must both report A, and the offline reader is the one + // an outside process would ask. + await accept(gate, ticketA, "A", "worker-one"); + assert.deepEqual( + Object.keys(gate.accepted()), + ["A"], + "the owner's view reports the accepted task, so the comparison below is not vacuous", + ); + const reader = openRoundQuery(database, "run-agree"); + try { + assert.deepEqual( + reader.port.accepted(), + gate.accepted(), + "both paths report the same acceptance", + ); + assert.equal( + reader.port.accepted()["A"], + gate.accepted()["A"], + "and the same artifact bytes, not merely the same task ids", + ); + } finally { + reader.close(); + } + + // After a terminal decision: the read-only path is opened fresh, so its answer is the one an + // outside process would get right now - and it has to be the same answer. + gate.cancel("operator stopped this one"); + const after = openRoundQuery(database, "run-agree"); + try { + assert.equal(after.port.cancelled(), "operator stopped this one"); + assert.equal(after.port.cancelled(), gate.cancelled(), "the terminal reason is the same one"); + assert.deepEqual(after.port.accepted(), gate.accepted(), "and so is acceptance"); + assert.deepEqual(after.port.accepted(), {}, "a cancelled round accepts nothing on either path"); + } finally { + after.close(); + } +}); diff --git a/tests/integration/ooo-round-query.test.ts b/tests/integration/ooo-round-query.test.ts new file mode 100644 index 00000000..1fc36b56 --- /dev/null +++ b/tests/integration/ooo-round-query.test.ts @@ -0,0 +1,106 @@ +// The status read path is borrowed. It must not migrate the store, must not publish, and must not give +// the caller any way to write - so it goes through the owner's narrow port rather than the board. +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import test from "node:test"; +import { BoardAdmission, openRoundQuery } from "../../src/integration/ooo-board.ts"; + +const hash = (path: string) => createHash("sha256").update(readFileSync(path)).digest("hex"); +const entries = (path: string): number => { + const db = new DatabaseSync(path, { readOnly: true }); + const count = db.prepare("SELECT COUNT(*) AS c FROM task_board_entries").get() as { c: number }; + db.close(); + return count.c; +}; + +test("the terminal decision outlives the host that made it, and still refuses new work", () => { + const dir = mkdtempSync(join(tmpdir(), "nmg-round-cancel-")); + test.after(() => rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 })); + const database = join(dir, "cancelled.sqlite"); + + // One host claims a task and then decides to stop the round. Cancel is reachable through the + // ordinary owner, with no dedicated surface: it is the round's own lifecycle operation. + const first = new BoardAdmission(database, undefined, {}, { runId: "cancel-run" }); + const task = first.next()!; + first.claim(task, "worker-one"); + assert.deepEqual( + first.cancel("operator stopped the round"), + [task], + "the cancellation names the lease it revoked", + ); + first.close(); + + // A second host opens the same store. The decision has to be a fact in the store rather than in + // the memory of the process that made it, or a restart would silently resume a stopped round. + const second = new BoardAdmission(database, undefined, {}, { runId: "cancel-run" }); + try { + assert.equal( + second.cancelled(), + "operator stopped the round", + "the terminal reason survives the host that recorded it", + ); + assert.throws(() => second.claim(task, "worker-two"), /cancelled/u); + assert.deepEqual(second.accepted(), {}, "and nothing is accepted after the decision"); + const fenced = new DatabaseSync(database, { readOnly: true }); + try { + const row = fenced + .prepare("SELECT owner, artifact, attempt FROM ooo_probe_facts WHERE run_id=? AND id=?") + .get("cancel-run", task) as { + owner: string | null; + artifact: string | null; + attempt: number; + }; + assert.equal(row.owner, null, "the revoked claim is gone, not merely refused"); + assert.equal(row.artifact, null, "and the attempt holds no artifact"); + assert.equal( + row.attempt, + 2, + "the fence advanced the attempt past the claim it revoked, so a late delivery is stale", + ); + } finally { + fenced.close(); + } + } finally { + second.close(); + } +}); + +test("the query port reads a round without migrating, publishing or exposing a write", () => { + const dir = mkdtempSync(join(tmpdir(), "nmg-round-view-")); + test.after(() => rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 })); + const database = join(dir, "store.sqlite"); + const owner = new BoardAdmission(database); + owner.close(); + const before = hash(database); + const beforeEntries = entries(database); + + const view = openRoundQuery(database); + try { + assert.deepEqual( + Object.keys(view.port).sort(), + ["accepted", "cancelled"], + "the port is reads only", + ); + const asRecord = view.port as unknown as Record; + for (const forbidden of ["close", "claim", "cancel", "deliver", "judge", "put", "db"]) + assert.equal(asRecord[forbidden], undefined, `the port exposes ${forbidden}`); + assert.equal(view.port.cancelled(), null, "an uncancelled round has no reason"); + assert.deepEqual(view.port.accepted(), {}, "nothing is accepted yet"); + } finally { + view.close(); + } + + assert.equal(hash(database), before, "the read path wrote to the store"); + assert.equal(entries(database), beforeEntries, "the read path published something"); + + // Each refusal names its own reason: nothing is created, migrated or guessed. + assert.throws(() => openRoundQuery(join(dir, "missing.sqlite")), /does not exist/u); + const bare = join(dir, "bare.sqlite"); + new DatabaseSync(bare).close(); + assert.throws(() => openRoundQuery(bare), /no round schema/u); + assert.throws(() => openRoundQuery(database, "no-such-run"), /holds no run/u); +}); diff --git a/tests/integration/ooo-run-namespace.test.ts b/tests/integration/ooo-run-namespace.test.ts new file mode 100644 index 00000000..8d34b3f9 --- /dev/null +++ b/tests/integration/ooo-run-namespace.test.ts @@ -0,0 +1,225 @@ +/** + * The round's storage is namespaced by run, so one store can hold several rounds without them + * colliding — and a store built before that existed is migrated rather than discarded. + * + * What these tests are for: the design's persistence bullet "two rounds with the same taskId must + * not collide" was untestable while a store held exactly one run (`ooo_probe_meta` had + * `CHECK(id=1)`, the task table was keyed by `id` alone, and the board channel was a constant). + * The first test here is that bullet; the last is that a store this code wrote earlier still + * opens, with its run, its rows and its terminal decision intact. + */ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import test from "node:test"; + +import { + BoardAdmission, + roundChannel, + type PatchTaskSpec, + type ProbePlan, +} from "../../src/integration/ooo-board.ts"; + +const plan: ProbePlan = [ + ["A", "", [], "isolated-artifact", null, null], + ["B", "", [], "isolated-artifact", null, null], + ["C", "", ["A", "B"], "isolated-artifact", null, null], +]; +const specs: Record = { + A: { + instruction: "A.", + files: { "a.ts": "export const a = 1;\n" }, + editable: ["a.ts"], + verify: async () => "accept", + }, + B: { + instruction: "B.", + files: { "b.ts": "export const b = 1;\n" }, + editable: ["b.ts"], + verify: async () => "accept", + }, +}; + +/** Raw rows, read with a separate handle: a migration is judged on what is in the file. */ +function rows(path: string, sql: string): Record[] { + const reader = new DatabaseSync(path, { readOnly: true }); + try { + return reader.prepare(sql).all() as Record[]; + } finally { + reader.close(); + } +} + +/** A scratch directory whose stores are closed before it is removed. One hook, not two: on + * Windows an open handle makes the removal fail rather than the assertion, and hooks run in + * registration order, so a cleanup registered before the stores are opened deletes first. */ +function scratch(t: { after: (fn: () => void) => void }) { + const directory = mkdtempSync(join(tmpdir(), "nmg-run-namespace-")); + const opened: BoardAdmission[] = []; + const adopt = (gate: BoardAdmission): BoardAdmission => { + opened.push(gate); + return gate; + }; + const open = (name: string, options: { runId?: string } = {}): BoardAdmission => + adopt(new BoardAdmission(join(directory, name), plan, specs, options)); + t.after(() => { + for (const gate of opened) gate.close(); + rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + }); + return { directory, open, adopt }; +} + +test("two runs in one store do not collide, do not see each other, and cancel separately", (t) => { + const { directory, open, adopt } = scratch(t); + const first = open("shared.sqlite", { runId: "run-one" }); + const second = open("shared.sqlite", { runId: "run-two" }); + + // Same task ids, different runs, different channels: the collision the design asked about. + assert.notEqual(first.channel, second.channel); + assert.equal(first.channel, roundChannel("run-one")); + assert.equal(second.runId, "run-two"); + + const database = join(directory, "shared.sqlite"); + assert.equal(first.claim("A", "worker-one").owner, "worker-one"); + // The second run's A is untouched by the first run's claim and is still the next task to work + // on there: one run's live claim is not another run's blocked task, and claiming it there does + // not write the first run's row. + assert.equal(second.next(), "A"); + assert.equal(second.accepted()["A"], undefined); + assert.equal(second.claim("A", "worker-two").owner, "worker-two"); + assert.equal( + rows(database, "SELECT owner FROM ooo_probe_task_view WHERE run_id='run-one' AND id='A'")[0]! + .owner, + "worker-one", + "the other run's row for the same task id belongs to the other run", + ); + + // A terminal decision is per run: cancelling one leaves the other alive. + second.cancel("operator stopped this one"); + assert.equal(second.cancelled(), "operator stopped this one"); + assert.equal(first.cancelled(), null); + + // A store holding several runs can still be reopened by name. + first.observeRevision("A", "input-v1"); + const reopened = adopt( + new BoardAdmission(join(directory, "shared.sqlite"), plan, specs, { runId: "run-one" }), + ); + assert.equal(reopened.runId, "run-one"); + assert.equal(reopened.cancelled(), null, "the other run's cancellation is not this run's"); +}); + +test("a store with several runs refuses to guess which one was meant", (t) => { + const { directory, open } = scratch(t); + const database = join(directory, "shared.sqlite"); + open("shared.sqlite", { runId: "run-one" }); + open("shared.sqlite", { runId: "run-two" }); + assert.throws( + () => new BoardAdmission(database, plan, specs), + /holds 2 runs; name the one to open/u, + "adopting the newest run and starting another are both silent answers", + ); +}); + +test("a store with one run is still continued by opening it", (t) => { + const { open } = scratch(t); + const runId = open("single.sqlite").runId; + assert.equal(runId.length > 0, true); + const reopened = open("single.sqlite"); + assert.equal(reopened.runId, runId, "a one-run store keeps behaving as it did before"); + assert.equal(reopened.channel, roundChannel(runId)); +}); + +test("a pre-namespace store is migrated in place, keeping its run, rows and terminal decision", (t) => { + const { directory, open } = scratch(t); + const database = join(directory, "legacy.sqlite"); + // The policy string this code writes for this plan, read from a store it wrote, so the fixture + // is a store this code could have produced rather than one shaped to pass. + const reference = open("reference.sqlite", { runId: "reference" }); + const holder = reference as unknown as { + db: { prepare: (sql: string) => { get: () => { policy: string } } }; + }; + const policy = holder.db.prepare("SELECT policy FROM ooo_probe_runs").get().policy; + + const runId = "legacy-run"; + // The shape this code wrote before runs existed, built directly so the migration is tested + // against real bytes rather than against this version's schema. + const legacy = new DatabaseSync(database); + legacy.exec(` + CREATE TABLE ooo_probe_meta (id INTEGER PRIMARY KEY CHECK(id=1), run_id TEXT NOT NULL, policy TEXT NOT NULL, + cancel_reason TEXT, cancelled_at TEXT); + CREATE TABLE ooo_probe_checks ( + task_id TEXT PRIMARY KEY, ticket TEXT NOT NULL, terminal TEXT, cancelled INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE ooo_probe_tasks ( + id TEXT PRIMARY KEY, revision TEXT NOT NULL, input TEXT NOT NULL, dependencies TEXT NOT NULL, + entry_id TEXT UNIQUE, attempt INTEGER NOT NULL DEFAULT 0, owner TEXT, claim_time TEXT, + input_digest TEXT, artifact TEXT, position INTEGER NOT NULL, effect TEXT NOT NULL, + source_revision TEXT NOT NULL, observed_revision TEXT NOT NULL, + wait_event TEXT, external_ready INTEGER NOT NULL, operation TEXT NOT NULL, + kind TEXT NOT NULL DEFAULT 'snapshot', patch_files TEXT, patch_editable TEXT, + accepted_entry_id TEXT + ); + `); + legacy + .prepare("INSERT INTO ooo_probe_meta (id, run_id, policy, cancel_reason) VALUES (1, ?, ?, ?)") + .run(runId, policy, "operator stopped it"); + const insert = legacy.prepare( + `INSERT INTO ooo_probe_tasks (id, revision, input, dependencies, attempt, owner, artifact, position, + effect, source_revision, observed_revision, wait_event, external_ready, operation, kind) + VALUES (?, 'v1', ?, ?, ?, ?, ?, ?, ?, 'input-v1', 'input-v1', NULL, 1, '', 'snapshot')`, + ); + insert.run("A", "A", "[]", 1, "worker-one", "commit-a", 0, "isolated-artifact"); + insert.run("C", "C", '["A"]', 0, null, null, 2, "isolated-artifact"); + legacy + .prepare( + "INSERT INTO ooo_probe_checks (task_id, ticket, terminal, cancelled) VALUES (?, ?, ?, 0)", + ) + .run("A", JSON.stringify({ taskId: "A", attempt: 1 }), "accepted"); + legacy.close(); + + const gate = open("legacy.sqlite"); + assert.equal(gate.runId, runId, "the stored run identity is kept, not replaced"); + assert.equal(gate.channel, roundChannel(runId)); + assert.equal(gate.cancelled(), "operator stopped it", "the terminal decision survives"); + + // The work state survives: the artifact, the attempt counted, the owner, and the dependency + // list of the task that was waiting on it. Rows come back with a null prototype, so they are + // copied before being compared as objects. + const worked = rows( + database, + "SELECT artifact, attempt, owner FROM ooo_probe_task_view WHERE id='A'", + )[0]!; + assert.deepEqual( + { ...worked }, + { artifact: "commit-a", attempt: 1, owner: "worker-one" }, + "the artifact, the attempt and the owner are the stored ones", + ); + assert.equal( + rows(database, "SELECT dependencies FROM ooo_probe_task_view WHERE id='C'")[0]!.dependencies, + '["A"]', + ); + assert.equal( + rows(database, "SELECT terminal FROM ooo_probe_checks WHERE task_id='A'")[0]!.terminal, + "accepted", + ); + // And the decision it kept is enforced, not merely readable: a cancelled run fences work. + assert.throws(() => gate.claim("B", "worker-two"), /round cancelled/u); + // Acceptance, though, is not restored from the old column. A pre-namespace store recorded the + // round's own verdict, which is what the board's independent verdict replaced, so the migration + // must not promote it: the artifact is present and unaccepted, not silently accepted. + gate.observeRevision("A", "input-v1"); + assert.deepEqual(gate.accepted(), {}); + + const names = rows(database, "SELECT name FROM sqlite_master WHERE type='table'").map((row) => + String(row.name), + ); + assert.equal(names.includes("ooo_probe_runs"), true); + assert.equal( + names.includes("ooo_probe_meta"), + false, + "the single-row table is gone, not copied forward", + ); + assert.equal(names.includes("ooo_probe_tasks_pre_split"), false); +}); diff --git a/tests/integration/ooo-speculation.test.ts b/tests/integration/ooo-speculation.test.ts new file mode 100644 index 00000000..4590b303 --- /dev/null +++ b/tests/integration/ooo-speculation.test.ts @@ -0,0 +1,127 @@ +/** + * F5: the speculation lifecycle - one declared, finite-valued fact guessed ahead of its evidence, and + * the three outcomes the design names (true publishes, false discards the candidate and closes its + * branch session, unknown waits). + * + * The rules live in `src/integration/ooo-execution.ts` beside the fusion conditions, because a guessed + * branch and a fused session answer the same kind of question: what the host may hand out, and what it + * may keep. Each case below breaks exactly one thing, and the last one pins the rule that joins the two + * halves - an invalidated branch is not reusable for the real path. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + isBoundedSpeculation, + sharedSessionLegal, + speculationOutcome, + type DispatchTask, + type ResolvedPredicate, + type SessionDeclaration, + type SessionPlan, + type SpeculationCandidate, +} from "../../src/integration/ooo-execution.ts"; + +function candidate(over: Partial = {}): SpeculationCandidate { + return { + taskId: "prepare", + assumptions: [{ predicateId: "gate", version: "v1", expected: "true" }], + speculativeSuccessors: [], + irreversibleOperations: [], + ...over, + }; +} + +function predicate(over: Partial = {}): ResolvedPredicate { + return { predicateId: "gate", version: "v1", value: "true", authoritative: true, ...over }; +} + +test("a guess the evidence confirms publishes, and its session stays reusable", () => { + const decision = speculationOutcome(candidate(), [predicate()]); + assert.equal(decision.outcome, "publish"); + assert.equal(decision.sessionReusable, true); + assert.match(decision.reason, /gate/); +}); + +test("a guess the evidence contradicts is discarded, and its branch session is closed", () => { + const decision = speculationOutcome(candidate(), [predicate({ value: "false" })]); + assert.equal(decision.outcome, "discard"); + assert.equal( + decision.sessionReusable, + false, + "the model has already seen the guess: an answer taken from that session is not the real answer", + ); +}); + +test("no evidence waits: an unknown fact never becomes a silent publish", () => { + const decision = speculationOutcome(candidate(), []); + assert.equal(decision.outcome, "wait"); + assert.equal(decision.sessionReusable, true, "waiting ends nothing"); +}); + +test("a reading nobody attested is not evidence", () => { + const decision = speculationOutcome(candidate(), [predicate({ authoritative: false })]); + assert.equal(decision.outcome, "wait"); + assert.match(decision.reason, /not authoritative/); +}); + +test("evidence about another version is not evidence about this fact", () => { + const decision = speculationOutcome(candidate(), [predicate({ version: "v2" })]); + assert.equal(decision.outcome, "wait"); + assert.match(decision.reason, /v2/); +}); + +test("the first experiment allows one pending fact, and a second is refused by name", () => { + const two = candidate({ + assumptions: [ + { predicateId: "gate", version: "v1", expected: "true" }, + { predicateId: "other", version: "v1", expected: "1" }, + ], + }); + assert.equal(isBoundedSpeculation(two), false); + assert.throws(() => speculationOutcome(two, []), /not a bounded speculation candidate/); +}); + +test("nothing is prepared from the guess: a speculative successor or an irreversible write is refused", () => { + assert.equal(isBoundedSpeculation(candidate({ speculativeSuccessors: ["after"] })), false); + assert.equal(isBoundedSpeculation(candidate({ irreversibleOperations: ["publish"] })), false); +}); + +test("a candidate that guesses nothing is not a speculative candidate at all", () => { + assert.equal(isBoundedSpeculation(candidate({ assumptions: [] })), false); +}); + +test("an invalidated branch is not reusable for the real path: fusion refuses the pair", () => { + const tasks: DispatchTask[] = [ + { ...base("prepare"), accepted: true }, + base("real", { dependencies: ["prepare"] }), + ]; + const declarations: Record = { + prepare: { capability: "patch", authority: "host", visible: ["src/a.ts"] }, + real: { capability: "patch", authority: "host", visible: ["src/a.ts"] }, + }; + const plan: SessionPlan = { tasks, declarations }; + assert.equal( + sharedSessionLegal("prepare", "real", plan), + true, + "the pair is legal while the branch is undecided", + ); + assert.equal( + sharedSessionLegal("prepare", "real", { ...plan, pendingBranches: ["prepare"] }), + false, + "a discarded branch's history may not carry the real path", + ); +}); + +function base(id: string, over: Partial = {}): DispatchTask { + return { + id, + effect: "isolated-artifact", + sourceVersion: "v1", + observedVersion: "v1", + dependencies: [], + accepted: false, + claimed: false, + externalReady: true, + ...over, + }; +} diff --git a/tests/integration/ooo-task-tables.test.ts b/tests/integration/ooo-task-tables.test.ts new file mode 100644 index 00000000..91698798 --- /dev/null +++ b/tests/integration/ooo-task-tables.test.ts @@ -0,0 +1,227 @@ +/** + * A task row used to hold three kinds of fact at once: what the run froze, what the round + * appended, and what can be recomputed. The split is what makes the third kind cheap to distrust — + * the design's recomputation requirement ("delete every derived cache and get the same view") has + * to be a real check, not a claim about the schema. + * + * So these tests are about *which* table decides: the manifest is never written after the plan is + * installed, the candidate bytes outlive a wiped cache because they are facts, and whatever the + * cache holds is overwritten by the sources rather than trusted. + */ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import test, { type TestContext } from "node:test"; + +import { + BoardAdmission, + type PatchTaskSpec, + type ProbePlan, +} from "../../src/integration/ooo-board.ts"; + +const plan: ProbePlan = [ + ["A", "", [], "isolated-artifact", null, null], + ["B", "", ["A"], "isolated-artifact", null, null], +]; +// A patch task's candidate must differ from the frozen input, so the frozen file and the +// submitted candidate are two different strings here rather than the same one twice. +const FROZEN = "export const a = 1;\n"; +const specs: Record = { + A: { + instruction: "A.", + files: { "a.ts": FROZEN }, + editable: ["a.ts"], + verify: async () => "accept", + }, + B: { + instruction: "B.", + files: { "b.ts": "export const b = 1;\n" }, + editable: ["b.ts"], + verify: async () => "accept", + }, +}; + +type Rows = Record[]; + +/** Raw rows, read through a second handle: these tests are about what is in the file. */ +function rows(path: string, sql: string): Rows { + const reader = new DatabaseSync(path, { readOnly: true }); + try { + return reader.prepare(sql).all() as Rows; + } finally { + reader.close(); + } +} + +/** A second, writable handle: these tests play the outsider that wipes or corrupts the cache. */ +function write(path: string, sql: string): void { + const writer = new DatabaseSync(path); + try { + writer.exec(sql); + } finally { + writer.close(); + } +} + +function fixture(t: TestContext) { + const directory = mkdtempSync(join(tmpdir(), "nmg-task-tables-")); + const database = join(directory, "round.sqlite"); + const gate = new BoardAdmission(database, plan, specs, { runId: "split-run" }); + t.after(() => { + gate.close(); + rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + }); + /** The only way an artifact reaches the coordinator: the claimed owner's own board entry. */ + const deliver = async (task: string, agent: string): Promise => { + const ticket = gate.claim(task, agent); + // Every task's candidate is built from its own frozen files: a patch task's candidate must + // differ from the frozen input, and a helper that hard-codes one task's path only works for + // that one task. + const spec = specs[task]!; + const artifact = JSON.stringify({ + digest: ticket.patch!.digest, + files: Object.entries(spec.files).map(([path, content]) => ({ + path, + content: `${content}// candidate for ${task} +`, + })), + }); + const entry = gate.putTaskBoardEntry({ + taskId: gate.channel, + agentId: agent, + kind: "result", + content: JSON.stringify({ ticket, artifact }), + expiresAt: new Date(gate.now + 86_400_000).toISOString(), + }); + assert.equal(await gate.submit(entry.id), "accepted"); + }; + return { gate, database, deliver }; +} + +/** The one derived column. Who claimed is a fact, and the board stops reporting it once the round + * resolves the entry, so a rebuild cannot re-derive it - that is why it is not here. */ +const cache = (database: string, id = "A"): Rows => + rows(database, `SELECT input_digest FROM ooo_probe_derived WHERE id='${id}' ORDER BY id`); + +test("the three tables hold the three kinds of fact, and a write lands in the right one", (t) => { + const { gate, database } = fixture(t); + const manifestBefore = rows( + database, + "SELECT * FROM ooo_probe_manifest WHERE run_id='split-run' AND id='A'", + ); + assert.equal(manifestBefore.length, 1, "the plan is installed as a manifest row"); + assert.equal(rows(database, "SELECT * FROM ooo_probe_facts WHERE id='A'").length, 1); + assert.equal( + cache(database).length, + 0, + "no derived row is written for a task nobody has claimed: it is a cache, not a record", + ); + + gate.claim("A", "worker-one"); + assert.deepEqual( + rows(database, "SELECT * FROM ooo_probe_manifest WHERE run_id='split-run' AND id='A'")[0], + manifestBefore[0], + "a claim writes no part of the manifest", + ); + assert.equal( + rows(database, "SELECT attempt FROM ooo_probe_facts WHERE id='A'")[0]!.attempt, + 1, + "the attempt is a fact", + ); + assert.equal( + rows(database, "SELECT owner FROM ooo_probe_facts WHERE id='A'")[0]!.owner, + "worker-one", + "the claim holder is a fact: the board will forget it when the entry is resolved", + ); + assert.equal( + cache(database)[0]!.input_digest !== null, + true, + "and the digest of the input this attempt was claimed against is the cache", + ); + + // The projection is what reads use, and it joins the three. + const view = rows( + database, + "SELECT id, owner, attempt, artifact FROM ooo_probe_task_view WHERE id='A'", + )[0]!; + assert.equal(view.owner, "worker-one"); + assert.equal(view.attempt, 1); + assert.equal(view.artifact, null); +}); + +test("the frozen plan has one owner, and a second, different plan is refused", (t) => { + const { database } = fixture(t); + const frozen = (): Rows => + rows(database, "SELECT * FROM ooo_probe_manifest WHERE run_id='split-run' ORDER BY id"); + const before = frozen(); + assert.equal(before.length, 2, "both tasks of the plan are frozen"); + + // The same two tasks in the other order are a different plan: the declared order is what the + // fallback dispatches by, so adopting it would silently replace the run's input. + const reordered: ProbePlan = [plan[1]!, plan[0]!]; + assert.throws( + () => new BoardAdmission(database, reordered, specs, { runId: "split-run" }), + /probe policy changed/u, + "a second plan for a run that already has one is refused", + ); + assert.deepEqual( + frozen(), + before, + "and the refused plan wrote nothing over the plan the run froze", + ); + + // The same plan is not a second one: reopening the run the store already holds is how a host + // continues a round, and it must not look like a new input. + const again = new BoardAdmission(database, plan, specs, { runId: "split-run" }); + try { + assert.deepEqual(frozen(), before, "reopening with the same plan changes no frozen row"); + } finally { + again.close(); + } +}); + +test("deleting the derived cache and rebuilding it yields the same view", async (t) => { + const { gate, database, deliver } = fixture(t); + await deliver("A", "worker-one"); + const before = cache(database); + assert.equal(before[0]!.input_digest !== null, true); + + write(database, "DELETE FROM ooo_probe_derived"); + assert.equal(cache(database).length, 0); + // The cache really did carry an answer the view needs, so this is not a vacuous comparison. + assert.equal( + rows(database, "SELECT input_digest FROM ooo_probe_task_view WHERE id='A'")[0]!.input_digest, + null, + ); + + assert.equal(gate.refreshDerived(), 2, "every manifest row is rebuilt"); + assert.deepEqual(cache(database), before, "the rebuilt cache is the same cache"); + assert.equal( + rows(database, "SELECT artifact FROM ooo_probe_facts WHERE id='A'")[0]!.artifact !== null, + true, + "the candidate bytes were never in the cache: wiping it cannot lose them", + ); + // And the rebuilt value is usable, not merely equal: the next task in the round is still + // deliverable, which needs the digest the rebuild just recomputed. + await deliver("B", "worker-two"); + assert.deepEqual(Object.keys(gate.accepted()).sort(), ["A", "B"]); +}); + +test("the sources overwrite the cache rather than trusting it", async (t) => { + const { gate, database, deliver } = fixture(t); + await deliver("A", "worker-one"); + const honest = cache(database)[0]!; + write(database, "UPDATE ooo_probe_derived SET input_digest='bogus' WHERE id='A'"); + assert.equal(cache(database)[0]!.input_digest, "bogus"); + + gate.refreshDerived(); + assert.notEqual(cache(database)[0]!.input_digest, "bogus"); + assert.deepEqual(cache(database), [honest], "and the rebuilt digest is the frozen one"); + // The claim the cache was corrupted beside is untouched: it is a fact. + assert.equal( + rows(database, "SELECT owner FROM ooo_probe_facts WHERE id='A'")[0]!.owner, + "worker-one", + ); +}); diff --git a/tests/integration/ooo-transition-atomicity.test.ts b/tests/integration/ooo-transition-atomicity.test.ts new file mode 100644 index 00000000..f34c78a8 --- /dev/null +++ b/tests/integration/ooo-transition-atomicity.test.ts @@ -0,0 +1,81 @@ +/** + * A round's transition is one state change, so a board write inside it is part of it: either the + * publication and the run facts land together, or neither does. The design's contract requires + * that the composed write join the open transition through its port instead of opening a second + * BEGIN, and these tests are what makes that checkable rather than a convention. + */ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import test, { type TestContext } from "node:test"; + +import { BoardAdmission, type ProbePlan } from "../../src/integration/ooo-board.ts"; + +const plan: ProbePlan = [["A", "", [], "isolated-artifact", null, null]]; + +function fixture(t: TestContext) { + const directory = mkdtempSync(join(tmpdir(), "nmg-atomic-")); + const database = join(directory, "round.sqlite"); + const gate = new BoardAdmission(database, plan, {}, { runId: "atomic-run" }); + t.after(() => { + gate.close(); + rmSync(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + }); + const published = (content: string): number => { + const reader = new DatabaseSync(database, { readOnly: true }); + try { + return ( + reader + .prepare("SELECT COUNT(*) AS count FROM task_board_entries WHERE content = ?") + .get(content) as { count: number } + ).count; + } finally { + reader.close(); + } + }; + /** The round's own publication path, which is private: this is the call the round makes. */ + const publish = (content: string, port: unknown): string => + ( + gate as unknown as { publish: (kind: string, content: string, port?: unknown) => string } + ).publish("decision", content, port); + return { gate, published, publish }; +} + +test("a board write inside a round transition commits with it", (t) => { + const { gate, publish, published } = fixture(t); + gate.writeTransaction((port) => { + publish("committed with the transition", port); + }); + assert.equal(published("committed with the transition"), 1); +}); + +test("the round's own publication rolls back with the transition that made it", (t) => { + const { gate, publish, published } = fixture(t); + assert.throws( + () => + gate.writeTransaction((port) => { + publish("rolled back with the transition", port); + throw new Error("the transition failed after the publication"); + }), + /the transition failed after the publication/u, + ); + assert.equal( + published("rolled back with the transition"), + 0, + "a publication that opened its own transaction would have survived this", + ); +}); + +test("a publication without a port is refused inside a transition, not nested", (t) => { + const { gate, publish, published } = fixture(t); + assert.throws( + () => + gate.writeTransaction(() => { + publish("published the wrong way", undefined); + }), + /already open: join it with the port it issued/u, + ); + assert.equal(published("published the wrong way"), 0); +}); diff --git a/tests/integration/ooo-verification.test.ts b/tests/integration/ooo-verification.test.ts index f154bbbc..f2c8ea24 100644 --- a/tests/integration/ooo-verification.test.ts +++ b/tests/integration/ooo-verification.test.ts @@ -1,6 +1,5 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { readFileSync } from "node:fs"; import { ARTIFACT_TOOL, artifactEnvelope, @@ -10,8 +9,9 @@ import { piCompletionAllowed, snapshotText, } from "../../.pi/extensions/nmg/ooo-execution.ts"; -import { patchPrompt, preparePatchWork } from "../../src/integration/ooo-patch.ts"; -import { expectedRename, verifyRenameCandidate } from "../../src/integration/ooo-verifier.ts"; +import { patchCandidate, patchPrompt, preparePatchWork } from "../../src/integration/ooo-patch.ts"; +import { verifyRenameCandidate } from "../../src/integration/check-runner.ts"; +import { expectedRenameOf, renameSource } from "../../evals/ooo-execution/rename-probe.ts"; import { mutate } from "../../src/integration/ooo-mutation.ts"; const patchWorkFields = () => ({ @@ -98,7 +98,7 @@ test("safety: the worker's check tool validates proposed files through the share "src/check.test.ts": "new\n", }); for (const files of [ - [{ path: "src/integration/ooo-check.ts", content: "injected\n" }], + [{ path: "src/integration/check-ticket.ts", content: "injected\n" }], [{ path: "../outside.ts", content: "injected\n" }], [{ path: "src/check.test.ts", content: "old\n" }], [], @@ -142,41 +142,40 @@ test("contract: the worker's check budget is host-limited, not model-chosen", () }); test("contract: candidate directory check reports a real terminal event without modifying source", async () => { - const path = new URL("../../src/integration/ooo-execution.ts", import.meta.url); - const source = readFileSync(path, "utf8"); - const result = await verifyRenameCandidate(source, expectedRename(source)); + // The probe's frozen target rather than a live product file: the oracle needs a file whose shape it + // requires (one exported `nextTask`, the `byId` map inside it, no further export), and freezing it + // keeps this test's own bounds from moving with a file it does not test. See `rename-probe.ts`. + const source = renameSource(); + const result = await verifyRenameCandidate(source, expectedRenameOf(source)); assert.equal(result.verdict, "accept"); assert.ok(result.checkId); assert.ok(result.startedAt && result.finishedAt && result.finishedAt >= result.startedAt); - assert.equal(readFileSync(path, "utf8"), source); + assert.equal(renameSource(), source, "the frozen target is not modified by a candidate check"); assert.equal( - (await verifyRenameCandidate(source, expectedRename(source) + "\n//extra")).verdict, + (await verifyRenameCandidate(source, expectedRenameOf(source) + "\n//extra")).verdict, "reject", ); const invalid = source + "\nfunction broken( {"; - assert.equal((await verifyRenameCandidate(invalid, expectedRename(invalid))).verdict, "reject"); + assert.equal((await verifyRenameCandidate(invalid, expectedRenameOf(invalid))).verdict, "reject"); }); test("safety: rename oracle rejects unrelated changes which passed the old substring check", () => { - const source = readFileSync( - new URL("../../src/integration/ooo-execution.ts", import.meta.url), - "utf8", - ); - const expected = expectedRename(source); + const source = renameSource(); + const expected = expectedRenameOf(source); assert.notEqual(expected, source); for (const candidate of [ expected + "\n// unrelated", expected.replace("return null;", "return 'unsafe';"), - expected.replace("export interface SnapshotWork", "interface SnapshotWork"), + expected.replace("export interface FrozenDispatchTask", "interface FrozenDispatchTask"), ]) { assert.ok(candidate.includes("planIndex") && !candidate.includes("byId")); assert.notEqual(candidate, expected); } const prefix = "// byId outside the target stays unchanged\n"; - assert.equal(expectedRename(prefix + source), prefix + expected); - assert.throws(() => expectedRename("missing function")); - assert.throws(() => expectedRename(expected)); - assert.throws(() => expectedRename(source + "\nexport const another = 1;")); + assert.equal(expectedRenameOf(prefix + source), prefix + expected); + assert.throws(() => expectedRenameOf("missing function")); + assert.throws(() => expectedRenameOf(expected)); + assert.throws(() => expectedRenameOf(source + "\nexport const another = 1;")); }); test("scope: the snapshot carries the readable subset only, and names what is hidden", () => { @@ -185,17 +184,17 @@ test("scope: the snapshot carries the readable subset only, and names what is hi attempt: 1, instruction: "Repair the check.", files: { - "src/integration/ooo-check.ts": "export const a = 1;\n", + "src/integration/check-ticket.ts": "export const a = 1;\n", "src/integration/ooo-patch.ts": "export const big = 1;\n", }, - editable: ["src/integration/ooo-check.ts"], - visible: ["src/integration/ooo-check.ts"], + editable: ["src/integration/check-ticket.ts"], + visible: ["src/integration/check-ticket.ts"], }); const snapshot = JSON.parse(snapshotText(frozen)) as { files: Record; hidden?: string[]; }; - assert.deepEqual(Object.keys(snapshot.files), ["src/integration/ooo-check.ts"]); + assert.deepEqual(Object.keys(snapshot.files), ["src/integration/check-ticket.ts"]); assert.deepEqual(snapshot.hidden, ["src/integration/ooo-patch.ts"]); const prompt = patchPrompt(frozen, ARTIFACT_TOOL); // What is frozen but not shown is stated, so narrowing the view is never a hidden rule. @@ -206,8 +205,8 @@ test("scope: the snapshot carries the readable subset only, and names what is hi taskId: "run-1:B", attempt: 1, instruction: "Add a regression.", - files: { "src/integration/ooo-check.ts": "export const a = 1;\n" }, - editable: ["src/integration/ooo-check.ts"], + files: { "src/integration/check-ticket.ts": "export const a = 1;\n" }, + editable: ["src/integration/check-ticket.ts"], }); assert.ok(!patchPrompt(whole, ARTIFACT_TOOL).includes("Frozen but not shown")); assert.ok(snapshotText(whole).includes('"hidden"') === false); @@ -321,3 +320,26 @@ test("contract: a patch attempt is told to answer through the artifact tool, not const snapshotMode = resourceLoader(false).getSystemPrompt() ?? ""; assert.match(snapshotMode, /must begin with '\{'/); }); + +test("contract: an artifact is read by its kind, and the patch reader refuses a conclusion", () => { + const frozen = patchWork(); + const conclusion = artifactEnvelope(frozen, { + digest: frozen.digest, + conclusion: "no-change-needed", + summary: "already covered", + evidence: "cited title exists", + }); + assert.equal( + conclusion.ok, + true, + "a conclusion is a legitimate artifact for a task whose rule admits one", + ); + // Two readers exist and the kind decides which one may read the bytes. A harness that fed a + // conclusion to the patch reader would report the reader's complaint as the candidate's quality, + // which is exactly what the E arm's first run did (see the arms record). + assert.throws( + () => patchCandidate(frozen, conclusion.ok ? conclusion.json : "{}"), + /invalid patch structure/, + "the patch reader must refuse a conclusion by name rather than reading half of it", + ); +}); diff --git a/tests/integration/task-semantics.test.ts b/tests/integration/task-semantics.test.ts index 91a7093e..8034b677 100644 --- a/tests/integration/task-semantics.test.ts +++ b/tests/integration/task-semantics.test.ts @@ -245,6 +245,32 @@ test("an unsatisfied external wait keeps a unit out of ready", () => { assert.deepEqual(ready.ready, ["W"]); }); +test("a run that declares more slots reports the tasks it may start, not just the head", () => { + // Two independent read-only units, nothing claimed: one slot sees one, a declared budget sees both. + const two = compileTaskUnits( + input({ + plan: [ + ["W", "1", [], "read-only", "check-event", null], + ["X", "1", [], "read-only", "check-event", null], + ] as unknown as ProbePlan, + specs: {}, + }), + ); + const facts = { externalReady: ["W", "X"] }; + assert.deepEqual(deriveStatus(two.units, facts).ready, ["W"], "the default budget is one task"); + assert.deepEqual(deriveStatus(two.units, facts, 2).ready, ["W", "X"]); + // A claim in flight spends a slot, and the claimed unit leaves `ready` with it. + assert.deepEqual(deriveStatus(two.units, { ...facts, claimed: ["W"] }, 2).ready, ["X"]); + assert.deepEqual(deriveStatus(two.units, { ...facts, claimed: ["W"] }, 1).ready, []); + const spent = deriveStatus(two.units, { ...facts, claimed: ["W", "X"] }, 2); + assert.deepEqual(spent.ready, [], "a spent budget is an empty answer, not a blocked one"); + assert.deepEqual( + spent.blocked.map((entry) => entry.id).sort(), + ["W", "X"], + "a claimed unit is still not accepted, so it is reported where it is not ready", + ); +}); + test("a refinement must carry every parent obligation and may not widen the write set", () => { const parent = compileTaskUnits(input()).units.find((unit) => unit.id === "P")!; const parts = compileTaskUnits( diff --git a/tests/tools/agent-verify.test.ts b/tests/tools/agent-verify.test.ts index 65801a71..f402f12b 100644 --- a/tests/tools/agent-verify.test.ts +++ b/tests/tools/agent-verify.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -10,6 +10,7 @@ import { buildVerificationPlan, discoverApplicableRcpContract, executeVerificationPlan, + narrowReason, type VerificationCommandResult, } from "../../tools/agent-verify.ts"; import type { AgentContextReport } from "../../tools/repo-context.ts"; @@ -190,6 +191,117 @@ test("CLI dry-run emits a machine-readable plan without running checks", () => { ); }); +test("a route that declines the shared checks plans only its own tests", () => { + // `.gitignore` is the surface the repository-tooling route declares the always-run shared checks + // not applicable to; its own tests still run, and they are where the one assertion that reads + // ignore rules lives (`tests/tools/complexity-gate-base.test.ts`). + const script = fileURLToPath(new URL("../../tools/agent-verify.ts", import.meta.url)); + const root = fileURLToPath(new URL("../..", import.meta.url)); + // Evidence goes to a temp path: this case roots the CLI at the repository itself, and a + // test must not overwrite the evidence a real verification left in `.nmg/verification`. + const evidence = join(mkdtempSync(join(tmpdir(), "nmg-agent-verify-plan-")), "plan.json"); + const result = spawnSync( + process.execPath, + [ + "--experimental-strip-types", + script, + "--root", + root, + "--scope", + ".gitignore", + "--dry-run", + "--output", + evidence, + "--json", + ], + { encoding: "utf8", windowsHide: true }, + ); + assert.equal(result.status, 0, result.stderr); + const payload = JSON.parse(result.stdout) as { + ok: boolean; + results: VerificationCommandResult[]; + }; + assert.equal(payload.ok, true); + assert.deepEqual( + payload.results.map(({ command, status }) => ({ command, status })), + [{ command: "node --test (repository-tooling)", status: "skipped" }], + ); +}); + +test("a declining route's narrow run verifies on its own tests and nothing else", () => { + // The dry-run case above reads the printed plan; this one runs the whole path, so the claim is + // about what executed. The fixture declares no shared script at all, so a run that touched the + // floor would fail on a missing script rather than quietly pass, and the receipt is asserted to + // record the declaration, which is the auditable half of it. + const root = mkdtempSync(join(tmpdir(), "nmg-agent-verify-declined-")); + mkdirSync(join(root, "docs"), { recursive: true }); + mkdirSync(join(root, "tests"), { recursive: true }); + writeFileSync(join(root, "docs", "owner.md"), "# Owner\n"); + writeFileSync( + join(root, "tests", "fixture.test.ts"), + 'import test from "node:test";\ntest("the route can check its own surface", () => {});\n', + ); + writeFileSync( + join(root, "package.json"), + JSON.stringify({ name: "fixture", version: "1.0.0", scripts: {} }), + ); + writeFileSync( + join(root, "agent-context.yaml"), + [ + "version: 1", + "routes:", + " - id: fixture", + " paths: [docs/**]", + " owners: [docs/owner.md]", + " tests: [tests/fixture.test.ts]", + " verify:", + " blocking: [check]", + " advisory: []", + " sharedChecks: none", + "", + ].join("\n"), + ); + const git = (args: string[]) => + spawnSync("git", args, { cwd: root, encoding: "utf8", windowsHide: true }); + assert.equal(git(["init", "--quiet"]).status, 0); + assert.equal(git(["config", "user.email", "verify@example.invalid"]).status, 0); + assert.equal(git(["config", "user.name", "Verify Test"]).status, 0); + assert.equal(git(["add", "."]).status, 0); + assert.equal(git(["commit", "--quiet", "-m", "fixture"]).status, 0); + writeFileSync(join(root, "docs", "owner.md"), "# Owner\n\nChanged.\n"); + + const script = fileURLToPath(new URL("../../tools/agent-verify.ts", import.meta.url)); + const result = spawnSync( + process.execPath, + ["--experimental-strip-types", script, "--root", root, "--json"], + { encoding: "utf8", windowsHide: true }, + ); + assert.equal(result.status, 0, result.stderr || result.stdout); + const payload = JSON.parse(result.stdout) as { + ok: boolean; + results: VerificationCommandResult[]; + rcp?: { status: string; receiptPath?: string }; + }; + assert.equal(payload.ok, true); + assert.equal(payload.rcp?.status, "verified"); + assert.deepEqual( + payload.results.map(({ command, status }) => ({ command, status })), + [{ command: "node-test:fixture", status: "passed" }], + "the route's own tests ran, and the always-run shared checks did not", + ); + const receipt = JSON.parse(readFileSync(payload.rcp!.receiptPath!, "utf8")) as { + gate: { mode: string; reason?: string }; + }; + assert.equal(receipt.gate.mode, "narrow"); + assert.match(receipt.gate.reason!, /declares the always-run shared checks not applicable/); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("the receipt reason records a route's decision to decline the shared checks", () => { + assert.match(narrowReason("repository-tooling", 0), /declares the always-run shared checks/); + assert.equal(narrowReason("documentation", 7), "cleanly owned by route documentation"); +}); + test("CLI executes npm scripts through a cross-platform child process", () => { const root = mkdtempSync(join(tmpdir(), "nmg-agent-verify-")); mkdirSync(join(root, "docs"), { recursive: true }); @@ -659,4 +771,113 @@ test("route tests that are only skipped do not pass", () => { const check = receipt.checks.find((entry) => entry.name === "node-test:plugin"); assert.equal(check?.status, "failed"); assert.match(check?.reason ?? "", /TAP acceptance rule/); + // The reason has to name what the rule rejected. This run's only defect is one declared skip, + // and without the counts the reader is sent looking for a broken test instead. + assert.match(check?.reason ?? "", /skipped 1/); + assert.match(check?.reason ?? "", /plugin ok/); +}); + +test("a failed check restates its own last lines and names the evidence file", () => { + const root = mkdtempSync(join(tmpdir(), "nmg-agent-verify-quiet-failure-")); + mkdirSync(join(root, "docs"), { recursive: true }); + writeFileSync(join(root, "docs", "owner.md"), "# Owner\n"); + writeFileSync( + join(root, "package.json"), + JSON.stringify({ + name: "fixture", + version: "1.0.0", + scripts: { + boom: `node -e "for (let i = 1; i <= 14; i++) console.log('check output line ' + i); console.error('the detail that matters'); process.exit(3)"`, + }, + }), + ); + writeFileSync( + join(root, "agent-context.yaml"), + "version: 1\nroutes:\n - id: fixture\n paths: [src/**]\n owners: [docs/owner.md]\n tests: []\n verify:\n blocking: [boom]\n advisory: []\n", + ); + + const script = fileURLToPath(new URL("../../tools/agent-verify.ts", import.meta.url)); + const result = spawnSync( + process.execPath, + ["--experimental-strip-types", script, "--root", root, "--scope", "src/file.ts"], + { encoding: "utf8", windowsHide: true }, + ); + assert.notEqual(result.status, 0, result.stderr || result.stdout); + assert.match(result.stdout, /npm run boom: failed/); + // The summary restates the failing command's own words, so a reader who lost the streamed + // output above it (scrolled away, or piped through a filter) still sees them in one place. + assert.match(result.stdout, /^ {2}\| the detail that matters$/m); + assert.match(result.stdout, /^ {2}\| check output line 14$/m); + // ...bounded: a failing check cannot flood the summary. + assert.doesNotMatch(result.stdout, /^ {2}\| check output line 1$/m); + assert.match(result.stdout, /last 10 lines; full output in the evidence file/); + assert.match(result.stdout, /Evidence: .*\.nmg[\\/]verification[\\/]latest\.json/); +}); + +test("narrow mode restates the failing check's last lines too", () => { + const root = narrowFixture({ failRouteTest: true }); + const script = fileURLToPath(new URL("../../tools/agent-verify.ts", import.meta.url)); + const result = spawnSync( + process.execPath, + ["--experimental-strip-types", script, "--root", root, "--changed"], + { encoding: "utf8", windowsHide: true }, + ); + assert.notEqual(result.status, 0, result.stderr || result.stdout); + // The failing test's own TAP summary is what the reader needs, under its own check. + assert.match(result.stdout, /^ {2}\| # fail 1$/m); + assert.match(result.stdout, /^ {2}\| # tests 1$/m); + assert.match(result.stdout, /Evidence: .*\.nmg[\\/]verification[\\/]latest\.json/); +}); + +test("a live mutation sweep makes the verifier refuse instead of reading the mutant", () => { + const root = mkdtempSync(join(tmpdir(), "nmg-agent-verify-sweep-")); + mkdirSync(join(root, "docs"), { recursive: true }); + mkdirSync(join(root, "src"), { recursive: true }); + writeFileSync(join(root, "docs", "owner.md"), "# Owner\n"); + writeFileSync(join(root, "src", "file.ts"), "export const value = 1;\n"); + writeFileSync(join(root, "package.json"), JSON.stringify({ name: "fixture", version: "1.0.0" })); + writeFileSync( + join(root, "agent-context.yaml"), + "version: 1\nroutes:\n - id: fixture\n paths: [src/**]\n owners: [docs/owner.md]\n tests: []\n verify:\n blocking: []\n advisory: []\n", + ); + const lockPath = join(root, ".temp", "mutation-lock.json"); + mkdirSync(join(root, ".temp"), { recursive: true }); + // The test's own pid is alive, so the lock is honoured as live; a lock whose owner is gone is ignored. + writeFileSync( + lockPath, + JSON.stringify({ + pid: process.pid, + startedAt: new Date().toISOString(), + target: "src/integration/ooo-execution.ts", + live: true, + }), + ); + + const script = fileURLToPath(new URL("../../tools/agent-verify.ts", import.meta.url)); + const run = () => + spawnSync( + process.execPath, + [ + "--experimental-strip-types", + script, + "--root", + root, + // A scope, so the plan does not fall back to changed-file discovery and need a Git worktree... + "--scope", + "src/file.ts", + // ...and no `--dry-run`: a dry run plans from the route config, so it is deliberately exempt from + // the refusal (what it reads cannot be a mutant). The refusal guards a run that would record a + // verdict, which is why this case exercises one. + "--json", + ], + { encoding: "utf8", windowsHide: true }, + ); + + const refused = run(); + assert.notEqual(refused.status, 0, refused.stdout); + assert.match(refused.stderr, /refusing to verify/); + assert.match(refused.stderr, /src\/integration\/ooo-execution\.ts/); + + rmSync(lockPath); + assert.equal(run().status, 0, "a quiet tree verifies again"); }); diff --git a/tests/tools/mutation-lock.test.ts b/tests/tools/mutation-lock.test.ts new file mode 100644 index 00000000..d368279e --- /dev/null +++ b/tests/tools/mutation-lock.test.ts @@ -0,0 +1,159 @@ +/** + * The lock a mutation sweep leaves behind, and who refuses to read the tree because of it + * (post-mortem 0003). + * + * The failure class escaped twice in one session: first as checks read *while* a sweep held a mutant, then + * as a sweep that was **killed** between its substitution and its restore - leaving a live mutant in the + * tree that the next sweep's own baseline run then failed on. A stale lock is therefore not harmless and + * is not taken over silently. + */ +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +import { + liveMutationLock, + mutationHazard, + mutationLockPath, + readMutationLock, + staleMutationLock, + type MutationLock, +} from "../../tools/mutation-lock.ts"; + +/** A pid that is certainly gone: a child that has already exited. */ +function deadPid(): number { + const child = spawnSync(process.execPath, ["-e", ""], { encoding: "utf8" }); + assert.ok(child.pid); + return child.pid; +} + +function withLock(lock: MutationLock, run: (root: string) => void): void { + const root = mkdtempSync(join(tmpdir(), "nmg-mutation-lock-")); + mkdirSync(join(root, ".temp"), { recursive: true }); + writeFileSync(mutationLockPath(root), `${JSON.stringify(lock, null, 2)}\n`); + try { + run(root); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +const lockOf = (pid: number, live: boolean): MutationLock => ({ + pid, + startedAt: new Date().toISOString(), + target: "evals/ooo-execution/plan-driver.ts", + live, +}); + +test("a running sweep is reported as live, and a killed one as stale", () => { + withLock(lockOf(process.pid, true), (root) => { + assert.ok(liveMutationLock(root), "this process owns the lock and is alive"); + assert.equal(staleMutationLock(root), null); + assert.match(mutationHazard(root), /holds the tree/); + }); + withLock(lockOf(deadPid(), true), (root) => { + assert.equal(liveMutationLock(root), null, "a dead owner is not a running sweep"); + assert.ok(staleMutationLock(root), "but its lock is not harmless"); + // The hazard names the file to inspect instead of inviting a silent takeover. + assert.match(mutationHazard(root), /died holding evals\/ooo-execution\/plan-driver\.ts/); + assert.match(mutationHazard(root), /git diff/); + }); + withLock(lockOf(deadPid(), false), (root) => { + // `live: false` means it was between mutants; the lock is still not a licence to read the tree. + assert.ok(staleMutationLock(root)); + }); +}); + +test("no lock file means no hazard", () => { + const root = mkdtempSync(join(tmpdir(), "nmg-mutation-lock-quiet-")); + try { + assert.equal(mutationHazard(root), ""); + assert.equal(staleMutationLock(root), null); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("a stale lock refuses a new sweep instead of being taken over", () => { + // The refusal is the guardrail: a killed sweep may have left its mutant in the target, and a new sweep + // that silently took the lock over would measure the plan against that mutant (which is what happened). + withLock(lockOf(deadPid(), true), (root) => { + const script = fileURLToPath(new URL("../../tools/mutation-teeth.ts", import.meta.url)); + const result = spawnSync( + process.execPath, + ["--experimental-strip-types", script, "--targets", "src/core/store/clock.ts"], + { encoding: "utf8", windowsHide: true, env: { ...process.env, MUTATION_LOCK_ROOT: root } }, + ); + assert.notEqual(result.status, 0, result.stdout); + assert.match(result.stderr, /refusing to start a mutation sweep/); + assert.match(result.stderr, /died holding/); + }); +}); + +test("a running sweep reports its mutant as live, not only its target", async () => { + // The lock's `live` field is what a reader uses to tell "between targets" from "a mutant is on disk", + // and it is written by the harness, not by this module - so a missing write leaves a field that lies. + // This runs a real (cheap) sweep and watches the lock while it works. + const root = mkdtempSync(join(tmpdir(), "nmg-mutation-lock-live-")); + const script = fileURLToPath(new URL("../../tools/mutation-teeth.ts", import.meta.url)); + const repo = fileURLToPath(new URL("../..", import.meta.url)); + const errPath = join(root, "child.err"); + const child = spawn( + process.execPath, + [ + "--experimental-strip-types", + script, + "--targets", + "src/core/store/clock.ts", + "--json", + join(root, "result.json"), + ], + { + cwd: repo, + env: { ...process.env, MUTATION_LOCK_ROOT: root }, + stdio: ["ignore", "ignore", openSync(errPath, "a")], + windowsHide: true, + }, + ); + try { + let live = false; + let target = ""; + const deadline = Date.now() + 240_000; + while (Date.now() < deadline && child.exitCode === null) { + const lock = readMutationLock(root); + if (lock?.live) { + live = true; + target = lock.target; + } + if (live) break; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + assert.equal( + live, + true, + `a substituted mutant is reported as live while the suite runs; child stderr:\n${readFileSync(errPath, "utf8")}`, + ); + assert.equal(target, "src/core/store/clock.ts"); + // Let it finish rather than killing it: a sweep killed between substitution and restore leaves its mutant + // in the tree under test, which is the hazard this whole file is about - and this test would be the one + // causing it. Waiting also buys the assertion that the run cleaned up after itself. + child.stdin?.end(); + await new Promise((resolve) => child.on("exit", () => resolve())); + const result = JSON.parse(readFileSync(join(root, "result.json"), "utf8")) as { + targets: { restoredByteIdentically: boolean; mutants: { caught: boolean }[] }[]; + }; + assert.equal(result.targets[0]?.restoredByteIdentically, true); + assert.equal( + result.targets[0]?.mutants.every((mutant) => mutant.caught), + true, + ); + assert.equal(mutationHazard(root), "", "the finished sweep leaves no hazard behind"); + } finally { + if (child.exitCode === null) child.kill(); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tests/tools/narrow-verify.test.ts b/tests/tools/narrow-verify.test.ts index 9a0d296c..4ed20966 100644 --- a/tests/tools/narrow-verify.test.ts +++ b/tests/tools/narrow-verify.test.ts @@ -95,6 +95,54 @@ test("a change spanning multiple routes escalates to full", () => { assert.match(plan.escalationReason!, /spans multiple routes/u); }); +test("a route that declares the shared checks not applicable narrows to its own tests", () => { + // The declaration is the route owner's, not an inference from the path: a narrow plan for a + // change this route solely owns carries no always-run shared checks, and the route's own tests + // are still required - `verify.sharedChecks: none` without them is refused at config load. + const declined: RouteLike = { + id: "repository-tooling", + paths: [".gitignore"], + tests: ["tests/tools/**"], + verify: { blocking: ["check"], advisory: [], sharedChecks: "none" }, + }; + const plan = planNarrowVerify([declined], [".gitignore"]); + assert.equal(plan.narrow, true); + assert.equal(plan.route!.id, "repository-tooling"); + assert.deepEqual(plan.shared, [], "the shared checks are not run for this surface"); + assert.deepEqual(plan.testGlobs, ["tests/tools/**"], "and the route's own tests still are"); +}); + +test("the declaration is not a way out of a shared/cross-cutting path", () => { + const declined: RouteLike = { + id: "repository-tooling", + paths: ["tools/**"], + tests: ["tests/tools/**"], + verify: { blocking: ["check"], advisory: [], sharedChecks: "none" }, + }; + const plan = planNarrowVerify([declined], ["tools/narrow-verify.ts"]); + assert.equal(plan.narrow, false, "a shared root still escalates to the declared blocking set"); + assert.match(plan.escalationReason!, /shared/u); +}); + +test('declaring the shared checks "always" is what every route does by default', () => { + const explicit: RouteLike = { + id: "documentation", + paths: ["docs/**"], + tests: ["tests/docs/**"], + verify: { blocking: ["docs:check"], advisory: [], sharedChecks: "always" }, + }; + const plan = planNarrowVerify([explicit], ["docs/README.md"]); + assert.deepEqual(plan.shared, [ + "check", + "docs:check", + "format:check", + "glossary:check", + "lint", + "package:check", + "rtm:check", + ]); +}); + test("no changed paths escalates to full", () => { const plan = planNarrowVerify(ROUTES, []); assert.equal(plan.narrow, false); diff --git a/tests/tools/repo-context.test.ts b/tests/tools/repo-context.test.ts index f39096f1..52713780 100644 --- a/tests/tools/repo-context.test.ts +++ b/tests/tools/repo-context.test.ts @@ -1,10 +1,11 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; +import { parse as parseYaml } from "yaml"; import { collectAgentContext, @@ -192,6 +193,48 @@ test("route schema rejects duplicate ids and commands with conflicting classific ); }); +test("verify.sharedChecks must be a known declaration, and declining needs its own tests", () => { + const root = fixture(); + const write = (verify: string[]) => + writeFileSync( + join(root, "agent-context.yaml"), + [ + "version: 1", + "routes:", + " - id: repository-tooling", + " paths: [.gitignore]", + " owners: []", + ` tests: ${verify[1]}`, + " verify:", + " blocking: [check]", + ` advisory: []`, + ` sharedChecks: ${verify[0]}`, + "", + ].join("\n"), + ); + + // A typo must not silently mean "always" (or silently mean "none"). + write(["sometimes", "[tests/tools/**]"]); + assert.throws( + () => validateAgentContext(root), + /verify\.sharedChecks must be "always" or "none", not "sometimes"/, + ); + + // A route that declines the always-run shared checks and declares no tests of its own would leave + // a narrow plan with nothing to execute; a verification tool must never report that as a pass. + write(["none", "[]"]); + assert.throws( + () => validateAgentContext(root), + /a route that declines the shared checks must declare its own tests/, + ); + + // Declining with its own tests is a valid declaration, and so is the explicit default. + write(["none", "[tests/tools/**]"]); + assert.doesNotThrow(() => validateAgentContext(root)); + write(["always", "[]"]); + assert.doesNotThrow(() => validateAgentContext(root)); +}); + test("markdown output remains a concise navigation surface", () => { const report = collectAgentContext(fixture(), ["src/store/rows.ts"]); const text = formatAgentContext(report); @@ -457,3 +500,44 @@ test("manual scope survives unavailable Git and reports the inspection failure", ["store"], ); }); + +/** The integration layer is split across three routes by owner document - the Agent Surface, the + * OoO/task execution orchestra, and the retrieval-index enrichment - so its files are claimed one + * by one: `matches()` reads the first `*` in a pattern as a directory prefix, so a mid-name pattern + * such as `src/integration/ooo-*.ts` selects nothing. A list rots silently - a new file would belong + * to no route and nothing would complain - so this keeps the declaration exactly as wide as the + * directory. The list of knowingly unrouted files is now empty: a file added here either gets a + * route or is named below on purpose. */ +const INTEGRATION_FILES_WITHOUT_A_ROUTE: string[] = []; + +test("the integration layer's routes claim exactly its files", () => { + const root = fileURLToPath(new URL("../../", import.meta.url)); + const config = parseYaml(readFileSync(join(root, "agent-context.yaml"), "utf8")) as { + routes: { id: string; paths: string[] }[]; + }; + const claimingRoutes = new Map(); + for (const route of config.routes) { + for (const path of route.paths) { + if (!path.startsWith("src/integration/")) continue; + claimingRoutes.set(path, [...(claimingRoutes.get(path) ?? []), route.id]); + } + } + const files = readdirSync(join(root, "src/integration")) + .filter((name) => name.endsWith(".ts")) + .map((name) => `src/integration/${name}`); + const claimedBySeveral = files.filter((file) => (claimingRoutes.get(file)?.length ?? 0) > 1); + assert.deepEqual( + claimedBySeveral, + [], + "a file claimed by two routes has two owner documents, which is one home too many", + ); + const declared = [ + ...files.filter((file) => claimingRoutes.has(file)), + ...INTEGRATION_FILES_WITHOUT_A_ROUTE.map((name) => `src/integration/${name}`), + ]; + assert.deepEqual( + declared.sort(), + files.sort(), + "every integration file is either claimed by one route or named as a known gap", + ); +}); diff --git a/tools/agent-verify.ts b/tools/agent-verify.ts index 8ce599b0..065f69f5 100644 --- a/tools/agent-verify.ts +++ b/tools/agent-verify.ts @@ -5,6 +5,7 @@ import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { collectAgentContext, type AgentContextReport } from "./repo-context.ts"; +import { mutationHazard } from "./mutation-lock.ts"; import { planNarrowVerify } from "./narrow-verify.ts"; import { digestCanonical } from "../src/rcp/canonical.ts"; import { compileContractFile } from "../src/rcp/contract.ts"; @@ -15,7 +16,6 @@ import { FileReceiptSink, LocalNpmVerifierProvider, NarrowVerifierProvider, - NARROW_SHARED_CHECKS, nodeTestCheckName, } from "../src/rcp/providers.ts"; import { reconcileOnce } from "../src/rcp/reconcile.ts"; @@ -162,10 +162,37 @@ function persistEvidence(path: string, evidence: unknown): void { renameSync(temporary, path); } +/** How much of a failing check's own output the summary restates. The runner already + * streams it while the check runs; this keeps the reason next to the verdict, bounded, + * so it survives a scrollback loss or a pipe filter. The full text stays in the + * evidence file the summary names. */ +const FAILURE_TAIL_LINES = 10; +const FAILURE_LINE_LIMIT = 300; + +function failureTail(output: string | undefined): string[] { + if (!output) return []; + const lines = output + .replace(/\r\n/g, "\n") + .split("\n") + .filter((line) => line.trim() !== ""); + const shown = lines + .slice(-FAILURE_TAIL_LINES) + .map((line) => + line.length > FAILURE_LINE_LIMIT + ? ` | ${line.slice(0, FAILURE_LINE_LIMIT)}…` + : ` | ${line}`, + ); + if (lines.length > FAILURE_TAIL_LINES) { + shown.push(` | …(last ${FAILURE_TAIL_LINES} lines; full output in the evidence file)`); + } + return shown; +} + function formatResult( report: AgentContextReport, result: VerificationRunResult, - rcp?: RcpEvidence, + rcp: RcpEvidence | undefined, + evidencePath: string, ): string { const lines = [ `Verification scopes: ${report.scopes.join(", ") || "none"}`, @@ -176,12 +203,14 @@ function formatResult( lines.push( `- [${item.classification}] npm run ${item.command}: ${item.status}${detail} <- ${item.routes.join(", ")}`, ); + if (item.status === "failed") lines.push(...failureTail(item.output)); } for (const warning of report.warnings) lines.push(`warning: ${warning}`); if (rcp) { lines.push(`RCP: ${rcp.contractId} ${rcp.status}`); if (rcp.receiptPath) lines.push(`RCP receipt: ${rcp.receiptPath}`); } + lines.push(`Evidence: ${evidencePath}`); return `${lines.join("\n")}\n`; } @@ -338,6 +367,17 @@ const invokedPath = process.argv[1] ? resolve(process.argv[1]) : ""; /** Run an arbitrary command (npm script or node --test globs) and shape the * result like the rest of the verification plan. */ +/** The receipt's `gate.reason`: why this change runs the narrow surface it runs. The one thing a + * reader cannot infer from `mode` is that the owning route declared the always-run shared checks + * not applicable to its own surface, so the reason says so instead of leaving it to be discovered + * by counting checks. Exported because the receipt this produces is the auditable half of the + * declaration. */ +export function narrowReason(routeId: string, sharedChecks: number): string { + return sharedChecks === 0 + ? `cleanly owned by route ${routeId}; the route declares the always-run shared checks not applicable` + : `cleanly owned by route ${routeId}`; +} + function runCommand( label: string, argv: string[], @@ -436,6 +476,20 @@ if (invokedPath === fileURLToPath(import.meta.url)) { process.exit(0); } const startedAt = new Date().toISOString(); + // A verification is a claim about a tree, and a tree with a live mutant in it is not the tree the + // change produced (post-mortem 0003). Refuse rather than report: a passing lane read in that window + // is evidence about code that never existed, and that is the reading nobody investigates. + // + // A `--dry-run` is exempt, and the reason is what it reads: the plan comes from the route config and + // the change list, not from the target file, so a dry run cannot report on a mutant - while refusing + // it would make "what would you run?" unanswerable exactly when a session needs it. A run that would + // record a verdict is the one that must not. + const sweeping = options.dryRun ? "" : mutationHazard(options.root); + if (sweeping) + throw new Error( + `refusing to verify: ${sweeping}; the reading would describe the mutant, not the change ` + + `(docs/postmortem/0003-checks-read-a-live-mutant.md)`, + ); const report = collectAgentContext(options.root, options.scopes, { changed: options.changed, }); @@ -460,8 +514,10 @@ if (invokedPath === fileURLToPath(import.meta.url)) { const narrowPlan = planNarrowVerify(report.routes, report.scopes); const route = narrowPlan.route; const wantNarrow = !options.full && (options.narrow || narrowPlan.narrow); + // One home for the check list: the plan decides, including whether the owning route declared + // the shared checks not applicable to its own surface. const narrowChecks = route - ? [...NARROW_SHARED_CHECKS, ...(route.tests.length ? [nodeTestCheckName(route.id)] : [])] + ? [...narrowPlan.shared, ...(route.tests.length ? [nodeTestCheckName(route.id)] : [])] : []; let execution: | Awaited> @@ -482,7 +538,7 @@ if (invokedPath === fileURLToPath(import.meta.url)) { json: options.json, routeId: route.id, checks: narrowChecks, - reason: `cleanly owned by route ${route.id}`, + reason: narrowReason(route.id, narrowPlan.shared.length), }, ); } else { @@ -524,7 +580,7 @@ if (invokedPath === fileURLToPath(import.meta.url)) { process.stdout.write( options.json ? `${JSON.stringify({ report, ...result, rcp, evidencePath: options.output }, null, 2)}\n` - : formatResult(report, result, rcp), + : formatResult(report, result, rcp, options.output), ); if (!result.ok) process.exitCode = 1; } catch (error) { diff --git a/tools/ci-uncovered-tests.ts b/tools/ci-uncovered-tests.ts index 689954f8..9e1f08a5 100644 --- a/tools/ci-uncovered-tests.ts +++ b/tools/ci-uncovered-tests.ts @@ -10,8 +10,10 @@ // // The acknowledged root is a deliberate decision, not an accident of naming: research harnesses // under `evals/` drive real worktrees, real child processes and (for live rounds) a paid provider, -// which is why they are opt-in rather than required checks. Anything *else* that falls out of CI -// has to be acknowledged here on purpose. +// which is why they are opt-in rather than required checks. Archived run directories are the +// second case: a finished run's candidate tree holds the `.test.ts` files it was judged on as +// evidence of that run, not as suites anyone maintains, so scanning them is a false positive. +// Anything *else* that falls out of CI has to be acknowledged here on purpose. import { execFileSync } from "node:child_process"; import { globSync, readFileSync } from "node:fs"; import { join } from "node:path"; @@ -20,7 +22,7 @@ import { join } from "node:path"; const CI_ENTRY_POINTS = ["verify:product-ci", "verify:research", "verify:chaos"]; /** Unreached suites that may stay out of the required checks, with the reason they may. */ -const ACKNOWLEDGED_ROOTS = ["evals/"]; +const ACKNOWLEDGED_ROOTS = ["evals/", "docs/experiments/execution/archive/"]; type Scripts = Record; diff --git a/tools/mutation-lock.ts b/tools/mutation-lock.ts new file mode 100644 index 00000000..655a9765 --- /dev/null +++ b/tools/mutation-lock.ts @@ -0,0 +1,107 @@ +/** + * The lock a mutation sweep holds while it holds a live mutant (post-mortem 0003). + * + * `mutation:teeth` substitutes a named wrong version into a target file, runs the suites that are meant + * to catch it, and restores the file byte-identically. Between those two writes the target file on disk + * **is** the mutant, so any check that reads the tree reports on the mutant rather than on the change - + * and a check that *passes* in that window is evidence about a tree that never existed. + * + * The rule used to live only inside the harness's process ("do not edit or stage a file a mutation run is + * rewriting"), which is why a reader could not see it: a check only *reads*, so the rule never fired. + * This file is what the tree says instead, and both the harness and the readers of the tree use it. + */ +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; + +export type MutationLock = { + pid: number; + startedAt: string; + /** The file the sweep is working on; empty while it is still selecting. */ + target: string; + /** True while a mutant is substituted into `target`: the tree is not readable for checks. */ + live: boolean; +}; + +/** Where the lock lives. Inside the gitignored scratch directory, so it is never staged. `MUTATION_LOCK_ROOT` + * lets a test point the lock somewhere other than the repository it is exercising. */ +export function mutationLockPath(root: string = lockRoot()): string { + return resolve(root, ".temp", "mutation-lock.json"); +} + +function lockRoot(): string { + return process.env.MUTATION_LOCK_ROOT ?? process.cwd(); +} + +export function readMutationLock(root: string = lockRoot()): MutationLock | null { + const path = mutationLockPath(root); + if (!existsSync(path)) return null; + try { + const parsed = JSON.parse(readFileSync(path, "utf8")) as MutationLock; + return typeof parsed?.pid === "number" ? parsed : null; + } catch { + // A half-written lock is not a reason to refuse: the sweep writes it before it substitutes anything. + return null; + } +} + +export function writeMutationLock(lock: MutationLock, root: string = lockRoot()): void { + const path = mutationLockPath(root); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${JSON.stringify(lock, null, 2)}\n`); +} + +/** Removes the lock only if this process owns it, so a stale-lock takeover cannot delete the live one. */ +export function clearMutationLock(pid: number, root: string = lockRoot()): void { + const lock = readMutationLock(root); + if (lock && lock.pid !== pid) return; + rmSync(mutationLockPath(root), { force: true }); +} + +function running(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +/** The lock **if a sweep is really still running**, i.e. its owner process is alive. */ +export function liveMutationLock(root: string = lockRoot()): MutationLock | null { + const lock = readMutationLock(root); + if (!lock) return null; + if (lock.pid === process.pid || running(lock.pid)) return lock; + return null; +} + +/** A lock whose owner is gone: a sweep that was killed. It is **not** harmless - it died between the + * substitution and the restore, so the target file may still be the mutant, and a check that reads the + * tree would report on that mutant (this is how the same failure class escaped a second time on + * 2026-09-18). Never take it over silently: the reader has to look at the file first. */ +export function staleMutationLock(root: string = lockRoot()): MutationLock | null { + const lock = readMutationLock(root); + if (!lock) return null; + if (lock.pid === process.pid || running(lock.pid)) return null; + return lock; +} + +export function describeMutationLock(lock: MutationLock): string { + return ( + `a mutation sweep holds the tree (pid ${lock.pid}, target ${lock.target || "selecting"}, ` + + `mutant live: ${lock.live}, since ${lock.startedAt})` + ); +} + +/** Why a reader must not treat the tree as readable, or an empty string when it may. */ +export function mutationHazard(root: string = lockRoot()): string { + const sweeping = liveMutationLock(root); + if (sweeping) return describeMutationLock(sweeping); + const killed = staleMutationLock(root); + if (killed) + return ( + `a previous mutation sweep died holding ${killed.target || "its target"} (pid ${killed.pid}, ` + + `since ${killed.startedAt}), so the file may still be its mutant - \`git diff\` it and remove ` + + `${mutationLockPath(root)} before trusting any check` + ); + return ""; +} diff --git a/tools/mutation-teeth.ts b/tools/mutation-teeth.ts index 353e821c..642cbf32 100644 --- a/tools/mutation-teeth.ts +++ b/tools/mutation-teeth.ts @@ -4,6 +4,11 @@ * A test can pass for the wrong reason: the rule it claims to protect may be dead code, or the * assertion may hold for a reason other than the one written down. Each mutant below replaces one * load-bearing line with a plausible-but-wrong version; the suite must then fail, and the *expected + * + * Two rules the config learned the hard way. `expect` is the NAME of the test that must fail - + * prose there makes a real catch read as "NOT caught". The `ast` locator is indentation-sensitive, + * so an anchor whose leading spaces no longer match the file is a stale anchor to be fixed, not a + * cosmetic difference; and a site that cannot be located is a failure, never a claimed check. * test* must be the one that fails. Three outcomes per target, not two: * * 1. the clean tree passes the target's suites; @@ -25,6 +30,16 @@ * never counted as caught. `--targets` therefore names what was actually run, so each branch's * evidence stays reproducible. * + * Where a mutant says where it applies: + * + * - `ast: { within: "" }` (or `ast: { call, argCount }`) locates the site through the syntax + * tree. Use this in any file that is still being edited. It survives reformatting, and it refuses + * when the code it guards has moved out of the member it belongs to - a move that a byte anchor + * would have followed silently. + * - a bare `from`/`to` pair matches bytes, then the same bytes with whitespace normalized. It is for + * settled files, where the anchor is cheap and the code is not moving. A re-taken anchor is + * printed, so a reflow never retires a tooth without saying so. + * * Usage: * npm run mutation:teeth -- [--targets=[,...]] [--json ] * Exit status is non-zero if the clean run fails, any mutant survives, any anchor is missing in a @@ -33,13 +48,32 @@ import { execFileSync } from "node:child_process"; import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { parseArgs } from "node:util"; +import ts from "typescript"; +import { + clearMutationLock, + mutationHazard, + writeMutationLock, + type MutationLock, +} from "./mutation-lock.ts"; import { writeJsonAtomic } from "./parts/fs.ts"; interface Mutant { /** What the wrong version does, in the words of the rule it breaks. */ readonly name: string; - readonly from: string; + /** The exact bytes to replace. Omitted when `ast` locates the site instead. */ + readonly from?: string; + /** A format-independent locator: find the site by syntax tree, not by text. Prettier reflows these + * files on every commit, and a text anchor silently stops applying the first time that happens. */ + readonly ast?: { + /** The method or function whose body is searched. The structurally scoped form: it survives + * reflow, and it refuses when the code it guards has moved out of the member it belongs to. */ + readonly within?: string; + /** The call or constructor to locate, by its callee name. */ + readonly call?: string; + /** How many arguments it takes, when the count is what tells the sites apart. */ + readonly argCount?: number; + }; readonly to: string; /** The test that must be the one to fail. */ readonly expect: string; @@ -102,6 +136,15 @@ const TARGETS: readonly Target[] = [ expect: "case 2: a lost obligation or a widened permission is refused, and an assumption cannot stand in for a dependency", }, + { + // The status query is the same rule read back: if it kept asking with a one-task budget it + // would report a ready set narrower than what the run declared, and the two answers would differ. + name: "the-status-query-ignores-the-declared-budget", + ast: { within: "deriveStatus" }, + from: " const ready = startableTasks(dispatchTasks(units, facts), slots);", + to: " const ready = startableTasks(dispatchTasks(units, facts), 1);", + expect: "a run that declares more slots reports the tasks it may start, not just the head", + }, ], }, { @@ -109,71 +152,573 @@ const TARGETS: readonly Target[] = [ suites: [ "tests/core/task-board-retention.test.ts", "tests/core/task-board-deliverable.test.ts", + "tests/core/store-transaction-port.test.ts", + "tests/core/store-readonly-open.test.ts", + "tests/core/store/task-runs.test.ts", + "tests/integration/ooo-managed-fence.test.ts", + "tests/integration/ooo-managed-write.test.ts", ], mutants: [ + { + // The handle, not a PRAGMA, is what makes this read-only: restoring writability must fail the test. + name: "the-read-only-factory-opens-a-writable-handle", + ast: { call: "DatabaseSync", argCount: 2 }, + to: "new DatabaseSync(databasePath)", + expect: "a read-only open neither creates, migrates nor writes", + }, + { + // The store owns the boundary: a write reached inside a transition without its port must be + // refused rather than become a second BEGIN. + name: "nested-write-transaction-is-allowed", + ast: { within: "writeTransaction" }, + from: ' if (this.openTransaction)\n throw new Error("a write transaction is already open: join it with the port it issued");', + to: ' if (this.openTransaction && false)\n throw new Error("a write transaction is already open: join it with the port it issued");', + expect: "a write entry reached inside a transition without a port is refused, not nested", + }, + { + // A failure the caller swallows still forbids the commit: nothing may be written up to the + // failure and then kept by a normal return value. + name: "swallowed-failure-still-commits", + ast: { within: "withPort" }, + from: " state.rollbackOnly = true;", + to: " void state.rollbackOnly;", + expect: "a failure the caller swallows still forbids the commit", + }, + { + // The mechanical invariant: a hand-rolled BEGIN anywhere in the store makes a second + // boundary possible, and behaviour tests would not notice a path that still works. + name: "a-method-opens-its-own-transaction", + ast: { within: "removeMemoryFromChain" }, + from: " return this.writeTransaction(() => {", + to: ' this.db.exec("BEGIN IMMEDIATE");\n return this.writeTransaction(() => {', + expect: "the store runs its transaction boundary in exactly one place", + }, + { + // The claim CAS is the whole of the fence: a reader who cannot take a live claim must + // lose it, and the condition that says so is the only thing between two readers and the + // same work. + name: "a-live-claim-can-be-taken-by-another-agent", + ast: { within: "claimTaskBoardEntry" }, + from: " AND (\n (claimed_by IS NULL OR claim_expires_at IS NULL OR claim_expires_at <= ?)\n OR claimed_by = ?\n )`,", + to: " AND (\n (claimed_by IS NULL OR claim_expires_at IS NULL OR claim_expires_at <= ?)\n OR ? IS NOT NULL\n )`,", + expect: "one reader of the same ready task is given the claim, the second is refused", + }, { name: "stale-claim-may-deliver-again", + ast: { within: "claimTaskBoardEntry" }, from: " if (!renewed) {", to: " if (false && !renewed) {", expect: "renewing your own live claim does not start a new attempt", }, { name: "deliverer-may-judge-its-own-work", + ast: { within: "judgeTaskBoardEntry" }, from: " if (existing.deliveredBy === input.agentId) {", to: " if (false && existing.deliveredBy === input.agentId) {", expect: "the deliverer cannot judge its own deliverable", }, { name: "prune-ignores-retention", + ast: { within: "pruneExpiredTaskBoardEntries" }, from: " `DELETE FROM task_board_entries WHERE task_id = ? AND expires_at <= ?\n AND id NOT IN (SELECT entry_id FROM task_board_retentions)`,", to: " `DELETE FROM task_board_entries WHERE task_id = ? AND expires_at <= ?`,", expect: "a retained entry, its delivery and its acknowledgement survive the prune", }, { name: "bounded-pin-never-expires", + ast: { within: "expireStaleRetentions" }, from: ' "DELETE FROM task_board_retentions WHERE retained_until IS NOT NULL AND retained_until <= ?",', to: ' "DELETE FROM task_board_retentions WHERE 0",', expect: "a bounded pin stops pinning when its bound passes", }, + { + // A run's plan is what every later decision is read against, so a second registration + // must not be able to replace it. + name: "a-second-plan-overwrites-the-frozen-one", + ast: { within: "insertTaskRunManifest" }, + from: " if (\n String(existing.plan_digest) !== input.planDigest ||\n String(existing.policy) !== input.policy\n )", + to: " if (\n false &&\n String(existing.plan_digest) !== input.planDigest &&\n String(existing.policy) !== input.policy\n )", + expect: "a run registers once, and a second plan for the same run is refused", + }, + { + // Frozen means frozen: the same task id with a different definition is a different plan, + // and replacing it in place would rewrite the input a decision was already read against. + name: "a-frozen-task-is-replaced-by-a-different-definition", + ast: { within: "insertTaskRunTask" }, + from: " if (!same)", + to: " if (!same && false)", + expect: "freezing a task twice is a no-op, and a different definition for it is refused", + }, + { + // The fact's own identity is what makes a retry after a lost response append once. + name: "a-retried-run-fact-is-appended-twice", + ast: { within: "insertTaskRunFact" }, + from: " if (known) return { sequence: Number(known.sequence), recorded: false };", + to: " if (known && false) return { sequence: Number(known.sequence), recorded: false };", + expect: "appending the same fact twice records it once and keeps the first sequence", + }, + { + // The fact write has to join the transition the caller opened, not open a second one; the + // board write and the run fact of one transition stand or fall together. + name: "the-run-fact-opens-its-own-transaction", + ast: { within: "appendTaskRunFact" }, + from: " return port\n ? this.withPort(port, () => this.insertTaskRunFact(input))\n : this.writeTransaction(() => this.insertTaskRunFact(input));", + to: " void port;\n return this.writeTransaction(() => this.insertTaskRunFact(input));", + expect: "a board write and a run fact land together, and neither lands alone", + }, + { + // One managed entry belongs to one run: answering with the first binding would hand a + // second run's facts to whoever asked. + name: "a-second-run-adopts-a-bound-entry", + ast: { within: "taskRunForEntry" }, + from: " if (runs.size > 1)", + to: " if (runs.size > 1 && false)", + expect: "an entry bound by two runs is refused rather than answered with one of them", + }, + { + // A managed entry's write belongs to its run's scope: the store is the only thing that can + // tell a coordinated write from a verb reached around it. + name: "a-managed-entry-ignores-the-coordinated-scope", + ast: { within: "requireManagedWriteScope" }, + from: " if (this.coordinatedRun === binding.runId) return;", + to: " if (true) return;", + expect: "a direct board verb cannot move an entry a run has adopted", + }, ], }, { target: "src/integration/ooo-board.ts", - suites: ["evals/ooo-execution/patch-cycle.test.ts"], + suites: [ + "evals/ooo-execution/patch-cycle.test.ts", + "evals/ooo-execution/board-slots.test.ts", + "tests/integration/ooo-run-namespace.test.ts", + "tests/integration/ooo-task-tables.test.ts", + "tests/integration/ooo-transition-atomicity.test.ts", + "tests/integration/ooo-acceptance-one-predicate.test.ts", + "tests/integration/ooo-round-query.test.ts", + "tests/integration/ooo-ordinary-failure.test.ts", + "tests/integration/ooo-read-paths-agree.test.ts", + "tests/integration/ooo-managed-fence.test.ts", + ], mutants: [ + { + // The borrowed view must not reopen the writer's path: this mutant makes the read path + // migrate and publish the store it was asked only to read. + name: "the-status-read-path-opens-the-rounds-store", + ast: { within: "openRoundQuery" }, + from: "const db = new DatabaseSync(databasePath, { readOnly: true });", + to: "const db = new BoardAdmission(databasePath) as unknown as DatabaseSync;", + expect: "the query port reads a round without migrating, publishing or exposing a write", + }, + { + // Located inside the reader: a rename of the call proves the check counts call sites. + name: "the-board-read-path-stops-calling-the-predicate", + ast: { within: "readAccepted" }, + from: "acceptedFact({", + to: "locallyAccepted({", + expect: "acceptance has one home, and both readers reach it", + }, + { + // A bypass that still type-checks: the suite must notice the second decision. + name: "the-board-decides-acceptance-on-its-own", + ast: { within: "readAccepted" }, + from: "!acceptedFact({", + to: '!(recorded?.verdict === "accepted" ? false : true) && !acceptedFact({', + expect: "acceptance has one home, and both readers reach it", + }, + { + // The claim is the write that would corrupt a neighbour run: the same task id exists in + // every run, so a claim that is not scoped by run claims somebody else's row too. + name: "claim-is-not-scoped-to-its-run", + ast: { within: "claim" }, + from: ' "UPDATE ooo_probe_facts SET attempt=?, owner=?, claim_time=? WHERE run_id=? AND id=?",', + to: ' "UPDATE ooo_probe_facts SET attempt=?, owner=?, claim_time=? WHERE ? IS NOT NULL AND id=?",', + expect: + "two runs in one store do not collide, do not see each other, and cancel separately", + }, + { + // The composed write must join the transition it is called in: a publication that opens its + // own boundary commits even when the transition around it fails. + name: "round-publication-opens-its-own-transaction", + ast: { within: "publish" }, + from: " },\n port,\n ).id;", + to: " },\n ).id;", + expect: "the round's own publication rolls back with the transition that made it", + }, + { + // The cache exists to be recomputable. A rebuild that returns without writing is the + // difference between "the sources decide" and "the schema says so". + name: "derived-rebuild-is-a-no-op", + ast: { within: "refreshDerived" }, + from: " this.putInputDigest(\n row.id,\n row.attempt >= 1 ? (frozen?.digest ?? this.inputDigest(row)) : null,\n );", + to: " void row.id;", + expect: "deleting the derived cache and rebuilding it yields the same view", + }, { name: "round-releases-dependents-on-delivered-bytes", - from: " verdict: recorded?.verdict ?? null,", + ast: { within: "readAccepted" }, + from: " verdict: recorded?.verdict ?? null,", to: ' verdict: "accepted",', expect: "an outside rejection withdraws the release of a dependent, and the round fails closed", }, { name: "verdict-lookup-not-bound-to-the-artifact", - from: " const recorded = verdictOf.get(channel, digest) as unknown as", - to: ' const recorded = verdictOf.get(channel, "%") as unknown as', + ast: { within: "readAccepted" }, + from: " const recorded = verdictOf.get(roundChannel(runId), digest) as unknown as", + to: ' const recorded = verdictOf.get(roundChannel(runId), "%") as unknown as', expect: "the board verdict is what accepts an artifact, not the round's own column", }, { - name: "selection-ignores-a-withdrawn-acceptance", - from: " const schedulable = rows.filter(\n (row) => !this.delivered(row) || Object.hasOwn(accepted, row.id),\n );", - to: " const schedulable = rows;", - expect: - "an outside rejection withdraws the release of a dependent, and the round fails closed", + name: "cancellation-does-not-stop-a-claim", + ast: { within: "claim" }, + from: ' if (!id || !agentId) throw new Error("task and agent required");\n if (this.cancelled() !== null) throw new Error("round cancelled");', + to: ' if (!id || !agentId) throw new Error("task and agent required");', + expect: "a cancellation names the lease it revokes, and the batch behind it is refused", }, { name: "round-does-not-pin-what-it-references", - from: ' this.retainTaskBoardEntry({\n taskId: channel,\n entryId,\n owner: RETENTION_OWNER,\n reason: `round ${this.runId ?? "initial"} handoff for ${row.id}`,\n now: new Date(this.now).toISOString(),\n });', + ast: { within: "publishReady" }, + from: ' this.retainTaskBoardEntry({\n taskId: this.channel,\n entryId,\n owner: RETENTION_OWNER,\n reason: `round ${this.runId ?? "initial"} handoff for ${row.id}`,\n now: new Date(this.now).toISOString(),\n });', to: " void entryId;", expect: "acceptance survives the entry's own TTL, because the round retains what it references", }, { name: "round-never-releases-its-pin", + ast: { within: "fenceRow" }, from: " // The artifact is being cleared, so the round no longer relies on this entry's\n // verdict: the pins go with the value they protected.\n this.releaseRowRetention(row);", to: " // The artifact is being cleared, so the round no longer relies on this entry's\n // verdict: the pins go with the value they protected.", expect: "cancelling a round releases the pins it held, so nothing it referenced leaks", }, + { + // The borrowed view is one implementation serving two paths. Letting the offline port + // answer from its own rule is exactly the divergence this target exists to catch. + name: "the-offline-reader-decides-acceptance-on-its-own", + ast: { within: "openRoundQuery" }, + from: " accepted: () => readAccepted(db, resolved),", + to: " accepted: () => ({}),", + expect: "the owner's view and the offline reader report the same facts", + }, + { + // Reopening withdraws what was built from the value that no longer exists. Clearing every + // accepted task instead takes back work the host already accepted. + name: "reopening-one-task-clears-every-acceptance", + ast: { within: "reopen" }, + from: " const affected = new Set([id]);", + to: " const affected = new Set(rows.map((row) => row.id));", + expect: "a later refusal does not withdraw the prefix that was already accepted", + }, + { + // The first decision is the one that took effect. A second cancellation that rewrites it + // is a state patch overwriting a terminal fact. + name: "a-second-cancellation-overwrites-the-first-decision", + ast: { within: "cancel" }, + from: " const already = this.cancelled();\n if (already !== null) return [];", + to: " const already = this.cancelled();\n if (false && already !== null) return [];", + expect: "a cancellation names the lease it revokes, and the batch behind it is refused", + }, + { + // The frozen plan is the run's input, and installing a second one over it is the second + // editable task truth the design forbids. The constructor refuses by policy digest. + name: "a-second-plan-silently-adopts-the-run", + from: " if (recorded && recorded.policy !== wanted)", + to: " if (false && recorded && recorded.policy !== wanted)", + expect: "the frozen plan has one owner, and a second, different plan is refused", + }, + { + // Verification is await-capable, so a ticket can be retired while it runs. Dropping the + // re-check at the commit is how a decision made before the wait is applied after it. + name: "the-commit-trusts-a-claim-the-board-retired", + ast: { within: "commitArtifact" }, + from: ' if (!this.live(row)) return "stale";', + to: ' if (false && !this.live(row)) return "stale";', + expect: "a claim the board retires inside the verification window cannot be committed", + }, + { + // The decision is a fact in the store. Keeping it only in the process that made it is + // what would let a restart resume a stopped round. + name: "cancelling-a-round-forgets-its-reason", + ast: { within: "cancel" }, + from: ' .prepare("UPDATE ooo_probe_runs SET cancel_reason=?, cancelled_at=? WHERE run_id=?")\n .run(reason.slice(0, 1_000), new Date(this.now).toISOString(), this.runId);', + to: ' .prepare("UPDATE ooo_probe_runs SET cancel_reason=NULL, cancelled_at=? WHERE run_id=?")\n .run(reason.slice(0, 1_000), new Date(this.now).toISOString(), this.runId);', + expect: "the terminal decision outlives the host that made it, and still refuses new work", + }, + { + // Selection and ranking must not be able to disagree with each other. `next()` is the head of + // `candidates()`, so a caller that starts one unit and a caller that starts several read the + // same order - and a mutant that moves the head by one breaks the round's dispatch. + name: "next-is-not-the-head-of-the-ordered-candidates", + ast: { within: "next" }, + from: " return this.candidates()[0] ?? null;", + to: " return this.candidates()[1] ?? null;", + expect: + "contract: a verified patch candidate is what dependents bind to, and only acceptance releases them", + }, + { + // The licence is the budget's part of the ordered set, not its head: with two declared slots a + // second task is claimable, and with one it is not. Narrowing it back to the head is exactly + // the rule the C arm could not cross. + name: "the-licence-is-the-head-whatever-the-budget", + ast: { within: "claimableRow" }, + from: ' if (!this.startable().includes(id)) throw new Error("task not selected by narrow dispatch");', + to: ' if (this.next() !== id) throw new Error("task not selected by narrow dispatch");', + expect: + "a declared budget holds two claims at once, and the store is why each handoff is directed", + }, + { + // A budget above one is unusable without a target, because the store queues a second + // un-directed actionable entry behind the first. Accepting it silently would report two slots + // and deliver one. + name: "a-second-slot-is-declared-without-a-target", + ast: { within: "admissionSlots" }, + from: " if (slots > 1 && !options.handoffTarget)", + to: " if (false && slots > 1 && !options.handoffTarget)", + expect: + "a declared budget holds two claims at once, and the store is why each handoff is directed", + }, + { + // Every startable task gets its handoff, not only the head: publishing one is what makes the + // other slots claims rather than a promise. + name: "only-the-heads-handoff-is-published", + ast: { within: "publishReady" }, + from: " if (!startable.includes(row.id)) continue;", + to: " if (row.id !== startable[0]) continue;", + expect: + "a declared budget holds two claims at once, and the store is why each handoff is directed", + }, + { + // A startable handoff is not retired just because it is not the head: retiring it would + // withdraw the second slot's offer right after publishing it. + name: "a-startable-handoff-is-retired-as-unselected", + ast: { within: "publishReady" }, + from: " if (startable.includes(row.id) || this.live(row)) continue;", + to: " if (row.id === startable[0] || this.live(row)) continue;", + expect: + "a declared budget holds two claims at once, and the store is why each handoff is directed", + }, + { + // The store's serialization is why a multi-slot run directs its handoffs. Publishing them + // un-directed leaves the second one queued as `pending`, and its claim is refused. + name: "a-multi-slot-handoff-is-published-un-directed", + ast: { within: "publishReady" }, + from: " this.slots > 1 ? this.handoffTarget!(row.id) : undefined,", + to: " undefined,", + expect: + "a declared budget holds two claims at once, and the store is why each handoff is directed", + }, + ], + }, + { + target: "src/integration/ooo-execution.ts", + suites: [ + "tests/integration/ooo-ordinary-failure.test.ts", + "tests/integration/ooo-publication-invariants.test.ts", + "tests/integration/ooo-advisers.test.ts", + "evals/ooo-execution/patch-cycle.test.ts", + "evals/ooo-execution/narrow-dispatch.test.ts", + "tests/integration/ooo-speculation.test.ts", + ], + mutants: [ + { + // A cancellation is a fact about the task, so it gates dispatch the way a rejection gates + // a dependent. This is the half acceptance already had and eligibility did not. + name: "a-cancelled-task-is-still-dispatched", + ast: { within: "selection" }, + from: " current(task) &&\n !task.cancelled &&", + to: " current(task) &&", + expect: "a cancelled task is not dispatched, and nothing reads one as a closed input", + }, + { + name: "the-dispatch-does-not-require-a-cancelled-input-to-be-closed", + ast: { within: "acceptedDependency" }, + from: " if (!task || !task.accepted || task.cancelled || !current(task) || visiting.has(id)) return false;", + to: " if (!task || !task.accepted || !current(task) || visiting.has(id)) return false;", + expect: "nextTask refuses a task marked cancelled, whatever else the caller set", + }, + { + name: "selection-ignores-a-withdrawn-acceptance", + ast: { within: "selection" }, + from: " const selectable = plan.filter((task) => task.accepted || !task.delivered);", + to: " const selectable = plan;", + expect: + "an outside rejection withdraws the release of a dependent, and the round fails closed", + }, + { + // The budget is what a claim spends, so a spent budget is an empty set. Nothing else may + // decide whether selection is open: this is the rule the C arm's slot count was once absent from. + name: "a-live-claim-does-not-block-selection", + ast: { within: "selection" }, + from: " if (room < 1 || pending.filter(waiting).length > 1) return none;", + to: " if (pending.filter(waiting).length > 1) return none;", + expect: "with no fusion point the plan falls back to its declared order", + }, + { + // A task someone is working is not on offer, whatever the budget. Without this, a run with + // slots to spare would hand the same task to a second worker. + name: "a-claimed-task-stays-on-offer", + ast: { within: "selection" }, + from: " tasks.filter((task) => !task.claimed && ready(task)).map((task) => task.id);", + to: " tasks.filter((task) => ready(task)).map((task) => task.id);", + expect: "a declared budget is spent by claims in flight, not by the next task's rank", + }, + { + // The cut is the whole point of declaring slots: without it the budget is a comment, and a + // run that asked for two would start the whole legal set. + name: "the-budget-is-not-cut-from-the-startable-set", + ast: { within: "startableTasks" }, + from: " return legal.slice(0, room);", + to: " return legal;", + expect: "a declared budget is spent by claims in flight, not by the next task's rank", + }, + { + // Zero or half a slot is not a smaller budget, and rounding it would hide the caller's typo. + name: "half-a-slot-is-a-smaller-budget", + ast: { within: "checkedSlots" }, + from: ' if (!Number.isSafeInteger(slots) || slots < 1) throw new Error("slots must be a positive integer");', + to: ' if (!Number.isSafeInteger(slots)) throw new Error("slots must be a positive integer");', + expect: "a claim in flight does not release a dependent, and half a slot is not a budget", + }, + { + // The head rule is a legality condition, not a preference: a head blocked by a stale input + // is not skipped in favour of a later ready task. This is the rule an ordering step is most + // likely to bypass by accident, so it has its own tooth. + name: "a-head-blocked-by-a-stale-input-is-skipped", + ast: { within: "selection" }, + from: " if (!current(first) || !waiting(first)) return none;", + to: " if (false) return none;", + expect: "the round's own answer is the shared rule's answer, not an ordering's", + }, + { + // `nextTask` returns the head of what it decided, so an ordering cannot disagree with the + // shared rule about what may be selected: dropping the head moves both. + name: "next-task-is-not-the-head-of-the-legal-set", + ast: { within: "nextTask" }, + from: " return selectableTasks(plan, slots)[0] ?? null;", + to: " return selectableTasks(plan, slots)[1] ?? null;", + expect: "the round's own answer is the shared rule's answer, not an ordering's", + }, + { + // The design's first experiment allows exactly one pending fact. This lets a candidate guess + // two at once, which is the boundary the design says to prove before widening. + name: "speculation-guesses-several-facts-at-once", + ast: { within: "isBoundedSpeculation" }, + from: " candidate.assumptions.length === 1 &&", + to: " candidate.assumptions.length >= 1 &&", + expect: "the first experiment allows one pending fact, and a second is refused by name", + }, + { + // A missing reading is not permission to publish: the design's "不确定就等待" is the whole + // reason the outcome has three states instead of two. + name: "a-guess-with-no-evidence-publishes", + ast: { within: "speculationOutcome" }, + from: + ' outcome: "wait",\n sessionReusable: true,\n' + + " reason: `no evidence for ${assumption.predicateId}`,", + to: ' outcome: "publish",\n sessionReusable: true,\n reason: "assumed",', + expect: "no evidence waits: an unknown fact never becomes a silent publish", + }, + { + name: "an-unattested-reading-counts-as-evidence", + ast: { within: "speculationOutcome" }, + from: " if (!fact.authoritative)", + to: " if (false)", + expect: "a reading nobody attested is not evidence", + }, + { + name: "evidence-about-another-version-is-the-same-fact", + ast: { within: "speculationOutcome" }, + from: " if (fact.version !== assumption.version)", + to: " if (false)", + expect: "evidence about another version is not evidence about this fact", + }, + { + // The invalidation rule: a contradicted guess closes its branch session, so the real path + // cannot take an answer from a model that has already been told the guess. + name: "a-contradicted-guess-keeps-its-session", + ast: { within: "speculationOutcome" }, + from: ' outcome: "discard",\n sessionReusable: false,', + to: ' outcome: "discard",\n sessionReusable: true,', + expect: "a guess the evidence contradicts is discarded, and its branch session is closed", + }, + ], + }, + { + target: "src/integration/task-semantics-interleavings.ts", + suites: ["tests/integration/ooo-publication-invariants.test.ts"], + mutants: [ + { + // Every declared budget publishes its own set, and the ones a one-slot run cannot offer are + // the whole point of declaring more. Deriving every prefix at one slot hides them. + name: "every-budget-is-walked-at-one-slot", + ast: { within: "budgetViews" }, + from: " for (const budget of budgets) {", + to: " for (const budget of budgets.slice(0, 1)) {", + expect: "a declared budget publishes more than one slot can, and never a claimed task", + }, + { + // A task someone is working is not offered to a second worker, at any budget. + name: "the-budget-offers-a-claimed-task", + from: " if (input.claimed.includes(unit))", + to: " if (false && input.claimed.includes(unit))", + expect: + "the budget properties fire on a hand-built view, so deleting them cannot pass quietly", + }, + { + // A bigger budget adds candidates; it may not drop one a smaller budget offered. + name: "a-bigger-budget-may-drop-a-candidate", + from: " if (!input.ready.includes(unit))", + to: " if (false && !input.ready.includes(unit))", + expect: + "the budget properties fire on a hand-built view, so deleting them cannot pass quietly", + }, + { + // Every condition in the checker is deleted once, and the case that names it has to fail: + // a condition no case can reach is a comment, not a check. + name: "the-completion-does-not-bind-the-verdict-to-the-bytes", + from: " else if (verdict.digest !== artifact)", + to: " else if (false && verdict.digest !== artifact)", + expect: "the checker reports a completion whose verdict judged other bytes", + }, + { + // Whether an input is current, and whether its bytes are the bytes its verdict judged, is the + // one acceptance predicate's answer; the checker asks it instead of comparing by hand. + name: "the-input-is-not-required-to-be-accepted", + ast: { within: "checkInputs" }, + from: " if (dependencyUnit && !isAccepted(dependencyUnit, context.facts))", + to: " if (false && dependencyUnit && !isAccepted(dependencyUnit, context.facts))", + expect: "the checker reports a completion resting on an input that drifted", + }, + { + name: "the-input-may-be-cancelled", + ast: { within: "checkInputs" }, + from: " if (cancelled(context.facts, dependency))", + to: " if (false && cancelled(context.facts, dependency))", + expect: "the checker reports a completion resting on a cancelled input", + }, + { + name: "the-completion-ignores-a-cancelled-unit", + ast: { within: "checkCompletion" }, + from: " if (cancelled(context.facts, unit.id))", + to: " if (false)", + expect: "the checker reports a completion of a cancelled unit", + }, + { + name: "the-dispatch-does-not-require-a-closed-input", + ast: { within: "checkDispatch" }, + from: ' checkInputs(context, unit, "dispatch");', + to: " void checkInputs;", + expect: "the checker reports a dispatch whose input is not accepted", + }, + { + // The enumeration is the other half of the claim: a merge that stops at the first order + // checks one interleaving and reports it as all of them. + name: "the-merge-enumerates-one-order", + ast: { within: "interleavings" }, + from: " return out;", + to: " return out.slice(0, 1);", + expect: "the merge enumerates every legal order, not one of them", + }, ], }, { @@ -193,9 +738,11 @@ const TARGETS: readonly Target[] = [ expect: "a dependency that is not accepted makes fusion pay a rollback and a retry", }, { + // The closure takes the unit-level predicate, so the marker names the two conditions it + // joins: a unit's own acceptance and every dependency being in the set. name: "acceptance-closure-dropped", - from: " own(unit.id) && unit.inputs.dependencies.every((dependency) => accepted.has(dependency));", - to: " own(unit.id);", + from: " own(unit) && unit.inputs.dependencies.every((dependency) => accepted.has(dependency));", + to: " own(unit);", expect: "a dependency that is not accepted makes fusion pay a rollback and a retry", }, ], @@ -224,7 +771,598 @@ const TARGETS: readonly Target[] = [ name: "judge-may-judge-its-own-delivery", from: " if (entry.deliveredBy === agentId) {", to: " if (false && entry.deliveredBy === agentId) {", - expect: "the board drivers run the protocol end to end on a scratch store", + expect: + "the board drivers run the protocol end to end through the daemon that serves the store", + }, + { + // The whole boundary is that a driver reaches the board through the daemon. A convenience + // import of the store is how that boundary rots, and the structural check is what catches it + // rather than a later round discovering a second writer. + name: "a-driver-falls-back-to-opening-the-store", + from: "const state = roundDaemon(resolve(values.daemon));", + to: 'const state = roundDaemon(resolve(values.daemon));\nconst store = (await import("../../src/core/store/base.ts")).NmgStoreBase;', + expect: "a driver refuses without a daemon, and no driver opens a database of its own", + }, + ], + }, + { + // The adapter is the only thing between a driver and the file it must not open: without its + // refusal, a missing daemon reads as a store the driver is free to open itself. + target: "evals/ooo-execution/round-client.ts", + suites: ["tests/integration/ooo-evidence-drivers.test.ts"], + mutants: [ + { + name: "the-round-client-does-not-require-a-daemon", + from: ' if (!state || state.transport !== "http" || !state.host || !state.port || !state.token) {', + to: ' if (false && (!state || state.transport !== "http" || !state.host || !state.port || !state.token)) {', + expect: "a driver refuses without a daemon, and no driver opens a database of its own", + }, + { + // Whether a call to an endpoint the caller itself serves can be answered depends on the + // caller not blocking - the assumption that turned this failure into a 305-second wait. + name: "the-round-client-calls-the-endpoint-it-serves", + from: " if (state.pid === process.pid) {", + to: " if (false && state.pid === process.pid) {", + expect: "a client refuses to call the endpoint its own process serves", + }, + { + // Without a bound, a blocked host is indistinguishable from a slow one, and the caller waits + // out the transport's own timeout instead of being told what to look at. + name: "the-round-client-has-no-limit-on-how-long-it-waits", + from: " return await httpCall(state, method, params, { timeoutMs });", + to: " return await httpCall(state, method, params, {});", + expect: "a call to a host that never answers gives up in seconds and names the reason", + }, + ], + }, + { + // The arms' driver: it decides only *how many* of the legal set to start at once, so each mutant + // removes one of its jobs - the batch width, the single dispatch of a unit, the failure report, + // the parent check, and the way each unit's work reaches that composition - and the case that + // fails names the job. + target: "evals/ooo-execution/plan-driver.ts", + suites: ["evals/ooo-execution/plan-driver.test.ts", "evals/ooo-execution/families.test.ts"], + mutants: [ + { + name: "the-driver-ignores-the-slot-count", + from: " const batch = legal.slice(0, fusionDeclared ? 1 : spec.slots);", + to: " const batch = legal.slice(0, 1);", + expect: "a declared slot count is reached, and the claims overlap in time", + }, + { + // The batch is the unit of overlap, and the overlap that matters is a unit's *check* beside + // another unit's work: awaiting each unit in turn keeps a batch's claims from ever running + // beside each other, which is the property the C arm buys. + name: "the-driver-awaits-each-unit-instead-of-the-batch", + from: " const held = await Promise.all(\n batch.map((id) => (fusionDeclared ? runChain(id, chains) : dispatch(id))),\n );", + to: " const held: boolean[] = [];\n for (const id of batch) held.push(await (fusionDeclared ? runChain(id, chains) : dispatch(id)));", + expect: "a unit's check is outstanding while an independent unit's worker runs", + }, + { + // The budget is declared to the admission layer, not only reported by the driver: a driver + // that asks the layer for one slot while promising the spec's count cannot overlap claims. + name: "the-driver-declares-one-slot-whatever-the-spec-says", + from: " slots: spec.slots,", + to: " slots: 1,", + expect: "a declared slot count is reached, and the claims overlap in time", + }, + { + name: "a-unit-is-dispatched-twice-in-one-batch", + from: " const batch = legal.slice(0, fusionDeclared ? 1 : spec.slots);", + to: " const batch = [...legal, ...legal].slice(0, spec.slots);", + expect: "one slot runs the units in plan order, each to acceptance", + }, + { + // The bound is what keeps a fused run from swallowing the plan. Without it one session would + // run every legal successor in turn. + name: "fusion-ignores-the-declared-bound", + from: " if (session.units.length >= bound) return undefined;", + to: " if (false) return undefined;", + expect: "a fused chain stops at the declared bound and does not swallow the plan", + }, + { + // The evidence of fusion is the session the worker reports, not the session the driver asked + // for: a worker that quietly starts its own session must not be reported as fused. + name: "fusion-counts-a-session-the-worker-did-not-use", + from: " if (unit?.sessionId !== session.id) {", + to: " if (false && unit?.sessionId !== session.id) {", + expect: "a worker that starts its own session is not reported as fusion", + }, + { + name: "a-failed-worker-is-reported-as-a-run-that-finished", + from: " if (result.failure !== undefined || result.artifact === undefined)", + to: " if (false && (result.failure !== undefined || result.artifact === undefined))", + expect: "a failed worker is recorded as incomplete rather than silently skipped", + }, + { + name: "the-parent-check-ignores-its-own-verdict", + from: " return {\n verdict: verified.verdict,", + to: ' return {\n verdict: "accept",', + expect: + "the parent check is the composed acceptance, and a failing check is reported as such", + }, + { + // A submitted patch carries the unit's whole frozen view, so composing by overwriting the + // candidate with each accepted submission puts the *last* unit's untouched copies of its + // siblings over the work they did. Only the files a unit changed are its work. + name: "the-parent-takes-the-last-units-whole-view", + from: " if (spec.baseline[path] !== content) files[path] = content;", + to: " files[path] = content;", + expect: "report: both plans accept the instrument's answers, and the same composed ones", + }, + { + name: "a-unit-ignores-the-checks-it-declares", + from: " const checks = unit.checks ? checkList(unit.checks) : fallback;", + to: " const checks = fallback;", + expect: "report: both plans accept the instrument's answers, and the same composed ones", + }, + { + name: "a-unit-nothing-checks-is-still-a-unit", + from: " if (!checks)\n throw new Error(\n `${id}: no checks", + to: " if (!checks && false)\n throw new Error(\n `${id}: no checks", + expect: "a unit nothing checks is refused rather than accepted on nothing", + }, + ], + }, + { + // A lease is the store's answer to "who serves this?" - a lease held by a process that is gone is + // a store nothing can serve, which is the same class of failure as a blocked host: the writer is + // absent and every client's answer depends on it coming back. + target: "src/cli/http-server.ts", + suites: ["tests/integration/ooo-evidence-drivers.test.ts"], + mutants: [ + { + name: "the-serving-process-never-releases-its-lease", + from: " lease.release();", + to: " // lease.release();", + expect: "a host releases its lease when it stops, so the next host can take the store", + }, + ], + }, + { + // The advice seam: an optional HA/MGR source may rank inside the legal set. Each mutant removes + // one of the refusals, so the case that fails names the obligation that stopped being enforced. + // The soft-premise rule has no line of its own to mutate: a suggestion type with no field for a + // dependency and no read of `assumptions` is what enforces it, and the out-of-set refusal is the + // layer a mutant can reach - which is why the first mutant's `expect` is the case that asserts + // both an out-of-set task and a soft premise claiming the dependency is satisfied. + target: "src/integration/task-advisers.ts", + suites: ["tests/integration/ooo-advisers.test.ts"], + mutants: [ + { + name: "a-suggestion-outside-the-legal-set-is-scored", + from: " if (!legalSet.has(suggestion.taskId)) {", + to: " if (false && !legalSet.has(suggestion.taskId)) {", + expect: "a suggestion outside the legal set is refused, however high it scores", + }, + { + name: "the-ordering-adds-a-task-to-the-set", + from: " return [...legal].sort((left, right) => {", + to: " return [...legal, ...best.keys()].sort((left, right) => {", + expect: "a suggestion outside the legal set is refused, however high it scores", + }, + { + name: "an-unmodelled-action-is-scored", + from: ' if (suggestion.action !== "next") {', + to: " if (false) {", + expect: "an unmodelled action is refused rather than scored", + }, + { + name: "a-disabled-source-is-asked-anyway", + from: " if (source.enabled === false) {", + to: " if (false && source.enabled === false) {", + expect: "a disabled or failing source falls back to the rule policy, and says why", + }, + { + name: "a-failing-source-takes-the-decision-with-it", + from: " } catch (error) {", + to: " } catch (error) {\n throw error;", + expect: "a disabled or failing source falls back to the rule policy, and says why", + }, + { + name: "a-score-from-another-scope-is-reused", + from: " if (\n provenance.sessionId !== projection.sessionId ||\n provenance.branchId !== projection.branchId\n ) {", + to: " if (false) {", + expect: "a score from another session or branch is not reused", + }, + { + name: "a-version-mismatch-still-counts-as-the-same-reading", + from: " if (provenance.parametersVersion !== projection.parametersVersion)\n missing.push(`parametersVersion=${provenance.parametersVersion}`);", + to: " if (false) missing.push(`parametersVersion=${provenance.parametersVersion}`);", + expect: "a changed parameter or projection version makes an old score a new one", + }, + { + name: "a-score-with-no-recorded-history-counts-as-a-reproduction", + from: ' if (!provenance.observationOrder?.length) missing.push("observationOrder");\n else if (!sameOrder(provenance.observationOrder, projection.observationOrder))\n missing.push(`observationOrder=${provenance.observationOrder.join(",")}`);', + to: "", + expect: + "a score that cannot name its own history is re-scored, never reported as a reproduction", + }, + { + name: "a-ranking-survives-into-the-claim", + from: " return legalNow.includes(adopted.taskId)", + to: " return true", + expect: "an adopted ranking is re-checked where the write happens", + }, + ], + }, + { + target: "src/integration/task-coordinator.ts", + suites: [ + "tests/integration/ooo-managed-write.test.ts", + "tests/integration/ooo-managed-adopt.test.ts", + "tests/cli/task-run-surface.test.ts", + ], + mutants: [ + { + // Binding is what makes an entry managed, so the refusal has to read the stored fact rather + // than whatever the caller believes about the entry. + name: "an-adopted-entry-takes-the-direct-path", + from: " if (!binding) return request.apply();", + to: " if (binding) return request.apply();", + expect: + "the routing rule sends a managed entry to its run and leaves an unmanaged one alone", + }, + { + // A run cannot adopt an entry for work it never froze: otherwise the binding names a task + // no decision was ever read against. + name: "a-binding-ignores-whether-the-task-was-frozen", + from: " if (!isFrozen(store, request.runId, request.taskId))", + to: " if (false && !isFrozen(store, request.runId, request.taskId))", + expect: "a binding refuses what the store does not hold", + }, + { + // The binding names an entry the board really holds, on the channel the caller names. + name: "a-binding-does-not-check-the-entry-exists", + from: " if (!store.getTaskBoardEntryById(request.boardTaskId, request.entryId))", + to: " if (false && !store.getTaskBoardEntryById(request.boardTaskId, request.entryId))", + expect: "a binding refuses what the store does not hold", + }, + { + // One entry carries one task: without this a second run would fence an entry it does not + // own, and the fence would refuse the first run's own writes. + name: "one-entry-is-bound-to-two-tasks", + from: " if (bound && (bound.runId !== request.runId || bound.taskId !== request.taskId))", + to: " if (false && bound && (bound.runId !== request.runId || bound.taskId !== request.taskId))", + expect: "a binding refuses what the store does not hold", + }, + { + // A second entry for the same task and attempt is a disagreement. The stored fact is keyed + // by task and attempt, so accepting it would keep the first binding and report the second. + name: "a-second-entry-rebinds-the-task", + from: " if (existing && existing.entryId !== request.entryId)", + to: " if (false && existing && existing.entryId !== request.entryId)", + expect: "a binding is idempotent for its task and attempt, and refuses a second entry", + }, + { + // The transition is the run's record of what happened to its entry; without it the board + // moved and the run has nothing to read. + name: "a-coordinated-write-skips-its-run-fact", + from: " const fact = store.appendTaskRunFact(\n {\n runId: request.runId,\n kind: managedTransitionKind(request.verb),\n taskId: binding.taskId,\n attempt: binding.attempt,\n entryId: request.entryId,\n payload: JSON.stringify({ actorId: request.actorId, status }),\n },\n port,\n );", + to: " const fact = { sequence: 0, recorded: true };", + expect: "a coordinated write lands the board transition and the run's fact together", + }, + { + // A cancelled run is the end of its managed entries' lifecycle, and the fence is the only + // thing that says so. + name: "a-cancelled-run-still-accepts-writes", + from: " if (cancelled)\n return `run ${runId} was cancelled at sequence ${cancelled.sequence}; its managed entries take no further lifecycle writes`;", + to: " if (cancelled && false)\n return `run ${runId} was cancelled at sequence ${cancelled.sequence}; its managed entries take no further lifecycle writes`;", + expect: "a cancelled run takes no further lifecycle writes on what it adopted", + }, + { + // The binding is re-read where the write happens, not where the caller decided to make it. + name: "a-coordinated-write-skips-the-binding-recheck", + from: " if (binding.runId !== request.runId)", + to: " if (false && binding.runId !== request.runId)", + expect: "a coordinated write refuses an entry that is not this run's", + }, + { + // The run surface's transitions: a plan the run cannot satisfy is refused while it is still + // a proposal rather than frozen into a task that can never be ready. + name: "the-plan-may-freeze-a-dangling-dependency", + from: " if (!known.has(dependency))", + to: " if (false && !known.has(dependency))", + expect: "a freeze cannot dangle, repeat a task, or lean on itself", + }, + { + // A task that waits for itself is a task that is never ready, and the freeze is the last + // point at which that is still only a proposal. + name: "a-task-may-depend-on-itself", + from: " if (dependency === task.taskId)", + to: " if (false && dependency === task.taskId)", + expect: "a freeze cannot dangle, repeat a task, or lean on itself", + }, + { + // Freezing is one transition: a batch where the store refuses one task must not leave the + // earlier ones frozen, or a plan exists that no caller ever proposed. + name: "the-plan-freezes-one-task-per-transaction", + from: " return store.coordinateRunWrite(request.runId, (port) => {\n // The array order is the plan order: the position comes from here, not from the request.\n request.tasks.forEach((task, position) =>\n store.freezeTaskRunTask({ ...task, runId: request.runId, position }, port),\n );\n return { runId: request.runId, frozen: request.tasks.length };\n });", + to: " request.tasks.forEach((task, position) =>\n store.freezeTaskRunTask({ ...task, runId: request.runId, position }),\n );\n return { runId: request.runId, frozen: request.tasks.length };", + expect: "a refused freeze leaves the plan exactly as it was", + }, + { + // The plan order is the array order: the position comes from that loop, so freezing every + // task at zero would leave the stored plan's order to the task ids. + name: "every-task-is-frozen-at-position-zero", + from: " request.tasks.forEach((task, position) =>\n store.freezeTaskRunTask({ ...task, runId: request.runId, position }, port),\n );", + to: " request.tasks.forEach((task) =>\n store.freezeTaskRunTask({ ...task, runId: request.runId, position: 0 }, port),\n );", + expect: "a run registers, freezes a plan, adopts entries, and reads it all back", + }, + { + // A cancelled run is closed: its plan is not extended behind the cancellation that every + // other rule in this file already honours. + name: "a-cancelled-run-takes-a-new-plan", + from: " const refusal = managedWriteRefusal(store, request.runId);\n if (refusal) throw new Error(refusal);", + to: " const refusal: string | null = null;\n if (refusal) throw new Error(refusal);", + ast: { within: "freezeRunPlan" }, + expect: "a cancelled run takes no further plan", + }, + { + // The binding records which channel carries the entry, which is what lets a status reader + // resolve it without searching every channel. + name: "a-binding-does-not-record-its-channel", + from: " payload: JSON.stringify({ boardTaskId: request.boardTaskId }),", + to: " payload: null,", + expect: "a run registers, freezes a plan, adopts entries, and reads it all back", + }, + { + // Creating the entry and adopting it are one transition. Two calls would leave an unmanaged + // entry behind when the binding is refused - the hole the run fence exists to close. + name: "the-entry-is-created-before-its-binding-is-checked", + from: " return store.writeTransaction((port) => {\n const entry = store.putTaskBoardEntry(request.entry, port);", + to: " return store.writeTransaction(() => {\n const entry = store.putTaskBoardEntry(request.entry);", + expect: + "adoption is part of the transition that creates the entry, so a refusal leaves no entry", + }, + { + // A run-level cancellation is the run's fact, not a task's: the schema's empty task id is + // what keeps it from colliding with a task that has no name. + name: "a-run-cancellation-names-a-task", + from: ' taskId: request.taskId ?? "",', + to: ' taskId: request.taskId ?? "-",', + expect: "cancelling a run is recorded once, stops its managed writes, and is readable", + }, + { + // Cancelling a task the plan never froze would name nothing while reading as a fact about + // the run. + name: "a-cancellation-ignores-whether-the-task-was-frozen", + from: " if (request.taskId !== undefined && !isFrozen(store, request.runId, request.taskId))", + to: " if (false && request.taskId !== undefined && !isFrozen(store, request.runId, request.taskId))", + expect: "cancelling one task names it, and a task the plan never froze cannot be cancelled", + }, + { + // There is nothing to cancel in a run this store cannot name, and the refusal says so + // rather than leaving it to the transaction's own message. + name: "an-unknown-run-can-be-cancelled", + from: " if (!store.taskRunManifest(request.runId))", + to: " if (false && !store.taskRunManifest(request.runId))", + expect: + "status is a read: an unknown run has no manifest and is not registered by being asked", + }, + { + // A status read registers and appends nothing: a view that repaired what it could not find + // would make its own answer true. + name: "status-registers-the-run-it-cannot-find", + from: " manifest: store.taskRunManifest(runId),", + to: ' manifest: (store.registerTaskRun({ runId, planDigest: "", policy: "", revision: "", retention: "" }), store.taskRunManifest(runId)),', + expect: + "status is a read: an unknown run has no manifest and is not registered by being asked", + }, + ], + }, + { + target: "src/cli/service.ts", + suites: ["tests/integration/ooo-managed-write.test.ts", "tests/cli/task-run-surface.test.ts"], + mutants: [ + { + // The routing is what keeps the daemon's verbs out of the store's refusal: a managed entry + // reached directly from a handler cannot be moved at all. The rule itself lives in the + // coordinator (one home for it), so this mutant pins that the daemon's claim still goes + // through it rather than at the store. + name: "the-daemon-verb-skips-the-coordinated-path", + from: ' entry: coordinatedEntryWrite(store, {\n verb: "claim",\n entryId: p.entryId,\n actorId: p.agentId,\n apply: () => store.claimTaskBoardEntry(p),\n }),', + to: " entry: store.claimTaskBoardEntry(p),", + expect: "a daemon board verb routes a managed entry through the run's transition", + }, + { + // The wire drops an adoption request: the entry is created, the caller is told the put + // succeeded, and no run manages it - the silent divergence the epoch rule exists for. + name: "the-wire-drops-an-adoption-request", + from: " adopt: optionalAdoption(params.adopt),", + to: " adopt: undefined,", + expect: + "adoption is part of the transition that creates the entry, so a refusal leaves no entry", + }, + ], + }, + { + // The shared-floor decision: a route may declare the always-run shared checks not applicable to + // its own surface, and that declaration must be honoured for exactly that route and nothing else. + target: "tools/narrow-verify.ts", + suites: ["tests/tools/narrow-verify.test.ts"], + mutants: [ + { + name: "the-declined-shared-checks-still-run", + from: ' const declined = route.verify.sharedChecks === "none";', + to: " const declined = false;", + expect: "a route that declares the shared checks not applicable narrows to its own tests", + }, + { + // An absent declaration means "always". Reading anything that is not the explicit + // "always" as a decline would silently drop the floor for every route that never asked. + name: "an-undeclared-route-is-read-as-declining", + from: 'route.verify.sharedChecks === "none"', + to: 'route.verify.sharedChecks !== "always"', + expect: "a change cleanly owned by one leaf route narrows to its own tests", + }, + ], + }, + { + // The declaration is refused at config load when it would leave a plan with nothing to execute, + // and an unknown value must not silently mean either answer. + target: "tools/repo-context.ts", + suites: ["tests/tools/repo-context.test.ts"], + mutants: [ + { + name: "a-declined-floor-may-have-no-tests", + from: 'if (sharedChecks === "none" && !route.tests.length) {', + to: 'if (false && sharedChecks === "none" && !route.tests.length) {', + expect: + "verify.sharedChecks must be a known declaration, and declining needs its own tests", + }, + { + name: "an-unknown-shared-checks-value-is-accepted", + from: 'if (sharedChecks !== undefined && sharedChecks !== "always" && sharedChecks !== "none") {', + to: 'if (false && sharedChecks !== undefined && sharedChecks !== "always" && sharedChecks !== "none") {', + expect: + "verify.sharedChecks must be a known declaration, and declining needs its own tests", + }, + ], + }, + { + // One home for the check list: the plan decides it, including a route's decline. A caller that + // rebuilt the floor from the constant would execute checks the plan said not to. + target: "tools/agent-verify.ts", + suites: ["tests/tools/agent-verify.test.ts"], + mutants: [ + { + name: "the-caller-rebuilds-the-shared-floor", + from: "? [...narrowPlan.shared, ...(route.tests.length ? [nodeTestCheckName(route.id)] : [])]", + to: '? ["check", "docs:check", "format:check", "glossary:check", "lint", "package:check", "rtm:check", ...(route.tests.length ? [nodeTestCheckName(route.id)] : [])]', + expect: "a route that declines the shared checks plans only its own tests", + }, + ], + }, + { + // Fusion legality gets its own entry for the same file: the harness reads only the first failures + // of a suite run, so a second suite in the existing target pushed that target's own named failures + // out of the window and four of its mutants read as uncaught. One mutant per condition the design + // puts on reusing a session, so a condition that stops being enforced fails a test by name rather + // than quietly widening what a host may run in one session. + target: "src/integration/ooo-execution.ts", + suites: ["tests/integration/ooo-fusion.test.ts"], + mutants: [ + { + name: "fusion-shares-a-session-across-different-capabilities", + ast: { within: "compatibleDeclarations" }, + from: " first.capability === next.capability &&", + to: " first.capability === first.capability &&", + expect: "fusion refuses a unit that needs a different execution capability", + }, + { + name: "fusion-shares-a-session-across-different-authorities", + ast: { within: "compatibleDeclarations" }, + from: " first.authority === next.authority &&", + to: " first.authority === first.authority &&", + expect: "fusion refuses a unit acting under a different authority", + }, + { + name: "fusion-widens-what-a-unit-may-read", + ast: { within: "compatibleDeclarations" }, + from: " subset(next.visible, first.visible)", + to: " subset(first.visible, next.visible)", + expect: "fusion refuses a successor whose visibility the session would widen", + }, + { + name: "fusion-continues-from-an-unverified-answer", + ast: { within: "sharedSessionLegal" }, + from: " if (!first.accepted) return false;", + to: " if (false && !first.accepted) return false;", + expect: "fusion refuses to continue from a unit whose verdict is not accepted", + }, + { + name: "fusion-starts-a-successor-whose-dependency-is-not-accepted", + ast: { within: "sharedSessionLegal" }, + from: " if (next.dependencies.some((id) => !acceptedDependency(byId, id))) return false;", + to: " if (false && next.dependencies.some((id) => !acceptedDependency(byId, id))) return false;", + expect: "fusion refuses a successor whose dependency is delivered but not accepted", + }, + { + name: "fusion-carries-a-cancelled-unit-into-its-next-unit", + ast: { within: "neitherCancelled" }, + from: " return !first.cancelled && !next.cancelled;", + to: " return true;", + expect: "fusion refuses a cancelled unit, before or after", + }, + { + name: "fusion-crosses-a-host-yield-boundary", + ast: { within: "sharedSessionLegal" }, + from: " if (!!first.externalEvent && !first.externalReady) return false;", + to: " if (false && !!first.externalEvent && !first.externalReady) return false;", + expect: "fusion ends the session at a declared external wait that is not ready", + }, + { + name: "fusion-reuses-history-across-a-pending-branch", + ast: { within: "acrossAPendingBranch" }, + from: " return pending.includes(before) || pending.includes(after);", + to: " return false;", + expect: "fusion never reuses the history across a fact whose branch is still pending", + }, + ], + }, + { + // Fusion's accounting: the two lines must stay two, the shared startup is booked once per session, + // and no verdict comes out of the assumed term. Each `from` is one of those rules. + target: "evals/ooo-execution/cost-model.ts", + suites: ["evals/ooo-execution/cost-model.test.ts"], + mutants: [ + { + name: "fusion-books-the-shared-startup-per-unit", + from: " sharedStartupMs: sessions * params.sessionStartMs,", + to: " sharedStartupMs: shape.units * params.sessionStartMs,", + expect: "fusion books the shared startup once per session, not once per unit", + }, + { + name: "fusion-counts-one-session-per-unit", + from: " const sessions = Math.ceil(shape.units / per);", + to: " const sessions = shape.units;", + expect: "fusion books the shared startup once per session, not once per unit", + }, + { + name: "fusion-removes-boundaries-that-are-not-there", + from: " boundarySavedMs: (shape.units - sessions) * params.contextMsPerUnit,", + to: " boundarySavedMs: shape.units * params.contextMsPerUnit,", + expect: "a fusion bound of one unit removes no boundary and still pays the startup", + }, + { + name: "fusion-reads-a-gain-out-of-an-assumed-term", + from: ' if (!params.sessionStartMeasured) return "unmeasured";', + to: ' if (false) return "unmeasured";', + expect: "an assumed session startup never reads as a gain", + }, + ], + }, + { + // The current-value window's clock grace. Each mutant is one of the ways the window can stop doing + // its job: dropping the grace on a boundary, making it zero, and writing a unit SQLite does not + // know (which makes the whole expression NULL and excludes every row instead of failing loudly). + target: "src/core/store/clock.ts", + suites: ["tests/core/store/current-value-window.test.ts"], + mutants: [ + { + name: "the-window-does-not-grace-valid-from", + from: ' `((${alias}.valid_from IS NULL OR ${alias}.valid_from <= ${clockNow("later")})` +', + to: ' `((${alias}.valid_from IS NULL OR ${alias}.valid_from <= ${clockNow("earlier")})` +', + expect: "a value stamped a moment in the future is current, not missing", + }, + { + name: "the-window-does-not-grace-expiry", + from: ' return `(${alias}.expires_at IS NULL OR ${alias}.expires_at > ${clockNow("earlier")})`;', + to: ' return `(${alias}.expires_at IS NULL OR ${alias}.expires_at > ${clockNow("later")})`;', + expect: "a value that expired a moment ago is still current", + }, + { + name: "the-grace-is-zero", + from: "export const CLOCK_GRACE_MS = 50;", + to: "export const CLOCK_GRACE_MS = 0;", + expect: "a value stamped a moment in the future is current, not missing", + }, + { + name: "the-grace-uses-a-unit-sqlite-does-not-know", + from: " return `strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '${modifier}${seconds} seconds')`;", + to: " return `strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '${modifier}${CLOCK_GRACE_MS} milliseconds')`;", + expect: "a just-written memory is never read as not active", }, ], }, @@ -247,6 +1385,101 @@ interface Outcome { readonly mutants: readonly MutantOutcome[]; } +/** Whitespace-normalized text search: exact bytes first, then reflowed form. + * + * The commit hook runs prettier, so a reflowed anchor must not retire a tooth. More than one match + * is still refused, because replacing the first would leave the rule intact somewhere else. */ +function matchText( + haystack: string, + anchor: string, +): { start: number; end: number; retaken: boolean } | { reason: string } { + const occurrences = haystack.split(anchor).length - 1; + if (occurrences === 1) { + const start = haystack.indexOf(anchor); + return { start, end: start + anchor.length, retaken: false }; + } + if (occurrences > 1) + return { reason: `marker occurs ${occurrences} times, refusing to claim a check` }; + // Built without a regex literal: one containing `${` confuses Node's type-stripping parser. + const special = ".*+?^$()[]{}|\\"; + const escaped = anchor + .trim() + .split(/\s+/u) + .map((part) => [...part].map((ch) => (special.includes(ch) ? "\\" + ch : ch)).join("")) + .join("\\s+"); + const matches = [...haystack.matchAll(new RegExp(escaped, "gu"))]; + if (matches.length !== 1) + return { + reason: `marker not found (${matches.length} matches once whitespace is normalized), refusing to claim a check`, + }; + const match = matches[0]!; + return { start: match.index, end: match.index + match[0].length, retaken: true }; +} + +/** Where a mutant applies: by syntax tree when it says so, by bytes otherwise. + * + * A site that cannot be located is a failure, not an "not applicable": that verdict is reserved for + * a target file that is not on this branch at all. Files that are still being edited should carry an + * `ast` locator, because a text anchor in them retires itself the first time the formatter runs. */ +function locate( + text: string, + mutant: Mutant, +): { start: number; end: number; retaken: boolean } | { reason: string } { + if (mutant.ast) { + const source = ts.createSourceFile("mutant.ts", text, ts.ScriptTarget.Latest, true); + if (mutant.ast.within !== undefined) { + const members: ts.Node[] = []; + const visit = (node: ts.Node): void => { + const named = + (ts.isMethodDeclaration(node) || ts.isFunctionDeclaration(node)) && + node.name?.getText(source) === mutant.ast!.within; + if (named) members.push(node); + ts.forEachChild(node, visit); + }; + visit(source); + if (members.length !== 1) + return { + reason: `ast scope ${mutant.ast.within} matched ${members.length} members, refusing to claim a check`, + }; + const member = members[0]!; + if (mutant.from === undefined) + return { reason: "an ast scope needs a from anchor to find inside it" }; + const inner = matchText(member.getText(source), mutant.from); + if ("reason" in inner) return { reason: `inside ${mutant.ast.within}: ${inner.reason}` }; + const offset = member.getStart(source); + return { start: offset + inner.start, end: offset + inner.end, retaken: inner.retaken }; + } + const found: ts.Node[] = []; + const walk = (node: ts.Node): void => { + if (ts.isCallExpression(node) || ts.isNewExpression(node)) { + const args = node.arguments?.length ?? 0; + if ( + node.expression.getText(source) === mutant.ast!.call && + (mutant.ast!.argCount === undefined || args === mutant.ast!.argCount) + ) + found.push(node); + } + ts.forEachChild(node, walk); + }; + walk(source); + if (found.length !== 1) + return { + reason: `ast locator ${mutant.ast.call} matched ${found.length} sites, refusing to claim a check`, + }; + return { start: found[0]!.getStart(source), end: found[0]!.getEnd(), retaken: false }; + } + if (mutant.from === undefined) + return { reason: "mutant has neither an ast locator nor a from anchor" }; + const found = matchText(text, mutant.from); + if ("reason" in found) return found; + if (found.retaken) + process.stdout.write( + ` re-taken anchor: ${mutant.name} (formatting reflowed it; ${String(found.end - found.start)} bytes) +`, + ); + return found; +} + /** The failure lines a suite run reported. Used for both verdicts: a surviving mutant * and a clean run that failed. The clean run is the one that proves nothing, so * naming its failing case is what turns "the harness proves nothing" into a fix. */ @@ -258,11 +1491,18 @@ function observedFailures(out: string): string[] { } function runSuites(suites: readonly string[]): { ok: boolean; out: string } { + // `NODE_TEST_CONTEXT` is what Node's test runner sets for the file it is running; if a sweep is + // started from inside a `node --test` process (a test that drives the harness), inheriting it makes the + // nested runner exit 0 without running a single test - and a suite that proves nothing is reported as + // "the suite passed", so every mutant of that target reads as *not caught* (post-mortem 0003's class, + // one layer deeper). The harness strips it so the suites it runs are really run. + const env = { ...process.env }; + delete env.NODE_TEST_CONTEXT; try { const out = execFileSync( process.execPath, ["--experimental-strip-types", "--test", "--test-concurrency=1", ...suites], - { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }, + { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env }, ); return { ok: true, out }; } catch (error) { @@ -288,7 +1528,35 @@ const strict = requested.length > 0; const outcomes: Outcome[] = []; const problems: string[] = []; const skipped: string[] = []; + +/** Announce the sweep in the tree for as long as it runs (post-mortem 0003): the target file on disk is + * a live mutant between the substitution and the restore, so a check that reads the tree in that window + * reports on the mutant. A second sweep in one worktree is the same hazard with a different reader, and + * a sweep that was *killed* leaves the hazard behind - so any lock at all refuses a start (the reader + * inspects the target and clears it rather than a new sweep taking it over silently). Released on exit. */ +const sweep: MutationLock = { + pid: process.pid, + startedAt: new Date().toISOString(), + target: "", + live: false, +}; +const hazard = mutationHazard(); +if (hazard) + throw new Error( + `refusing to start a mutation sweep: ${hazard} - the tree is not readable for checks while a mutant ` + + `may be live`, + ); +writeMutationLock(sweep); +for (const signal of ["SIGINT", "SIGTERM"] as const) + process.on(signal, () => { + clearMutationLock(process.pid); + process.exit(130); + }); +process.on("exit", () => clearMutationLock(process.pid)); + for (const { target, suites, mutants } of selected) { + sweep.target = target; + writeMutationLock(sweep); const present = suites.filter((suite) => existsSync(suite)); const absent = suites.filter((suite) => !existsSync(suite)); if (absent.length > 0) { @@ -322,22 +1590,23 @@ for (const { target, suites, mutants } of selected) { const mutantOutcomes: MutantOutcome[] = []; for (const mutant of mutants) { const text = original.toString("utf8"); - // The marker must occur exactly once: zero occurrences means the code moved, and more than - // one means replacing the first would leave the rule intact somewhere else. - const hits = text.split(mutant.from).length - 1; - if (hits !== 1) { - const reason = `${target} / mutant ${mutant.name}: marker occurs ${hits} times, refusing to claim a check`; - if (strict) problems.push(` ${reason}`); - else skipped.push(reason); + const site = locate(text, mutant); + if ("reason" in site) { + // A site that cannot be located is a failure even in the default list: the file is on this + // branch, so its code moved or was reflowed past recognition, and the tooth did not run. + const reason = `${target} / mutant ${mutant.name}: ${site.reason}`; + problems.push(` ${reason}`); mutantOutcomes.push({ name: mutant.name, applicable: false, caught: false, - note: `marker occurs ${hits} times`, + note: site.reason, }); continue; } - writeFileSync(target, text.replace(mutant.from, mutant.to)); + sweep.live = true; + writeMutationLock(sweep); + writeFileSync(target, text.slice(0, site.start) + mutant.to + text.slice(site.end)); const result = runSuites(present); const caught = !result.ok && result.out.includes(mutant.expect); mutantOutcomes.push( @@ -355,6 +1624,8 @@ for (const { target, suites, mutants } of selected) { } } writeFileSync(target, original); + sweep.live = false; + writeMutationLock(sweep); const restored = Buffer.compare(original, readFileSync(target)) === 0; if (!restored) problems.push(`${target}: restore is not byte-identical`); outcomes.push({ diff --git a/tools/narrow-verify.ts b/tools/narrow-verify.ts index e4705b46..726d4ec5 100644 --- a/tools/narrow-verify.ts +++ b/tools/narrow-verify.ts @@ -3,11 +3,13 @@ // Principle: verification is a composition of lightweight primitives over one // vocabulary. A change that is cleanly owned by one route's bounded domain runs // that route's OWN tests (node --test ) plus the always-run shared -// checks; a change touching any shared / cross-cutting file falls back to -// the declared whole blocking set. When in doubt, run full — that is the whole -// safety rule. No new executor: the tools the scripts wrap already accept the -// narrow globs (route.tests), so this only adds route granularity + a coverage -// rule, not machinery. +// checks - unless that route declares the shared checks not applicable to its own +// surface (`verify.sharedChecks: "none"`), which is a declaration its owner makes +// and which never covers a shared/cross-cutting path; a change touching any shared / +// cross-cutting file falls back to the declared whole blocking set. When in doubt, run +// full - that is the whole safety rule. No new executor: the tools the scripts wrap +// already accept the narrow globs (route.tests), so this only adds route granularity + +// a coverage rule, not machinery. import { NARROW_SHARED_CHECKS } from "../src/rcp/providers.ts"; @@ -15,7 +17,7 @@ export interface RouteLike { id: string; paths: string[]; tests: string[]; - verify: { blocking: string[]; advisory: string[] }; + verify: { blocking: string[]; advisory: string[]; sharedChecks?: "always" | "none" }; } export interface NarrowVerifyPlan { @@ -24,7 +26,8 @@ export interface NarrowVerifyPlan { narrow: boolean; /** The single owning route (only when narrow). */ route?: RouteLike; - /** Always-run shared checks for any change. */ + /** Always-run shared checks for any change, unless the owning route declares them not + * applicable to its own surface (`verify.sharedChecks: "none"`). */ shared: string[]; /** node --test globs from the owning route's own tests (may be empty). */ testGlobs: string[]; @@ -120,10 +123,14 @@ export function planNarrowVerify( }; } const route = owners[0]!.route; + // The floor is the always-run shared checks. A route whose owner has declared it not applicable + // to that route's own surface drops it for a change the route solely owns; the route's own tests + // still run, and `verify.sharedChecks: "none"` is refused at config load unless they exist. + const declined = route.verify.sharedChecks === "none"; return { narrow: true, route, - shared, + shared: declined ? [] : shared, testGlobs: route.tests, }; } diff --git a/tools/repo-context.ts b/tools/repo-context.ts index 1efe372b..88c4cd07 100644 --- a/tools/repo-context.ts +++ b/tools/repo-context.ts @@ -5,10 +5,17 @@ import { fileURLToPath } from "node:url"; import { parse as parseYaml } from "yaml"; import { digestRepositoryPaths, observeGitWorktree } from "../src/rcp/repository.ts"; +import { mutationHazard } from "./mutation-lock.ts"; export interface VerificationConfig { blocking: string[]; advisory: string[]; + /** Whether the always-run shared checks (the narrow plan's floor) apply when this route solely + * owns a change. `"none"` is a declaration the route's owner makes about its own surface, not + * an inference: it is only honoured on the narrow path, it never covers a shared/cross-cutting + * path, and it is refused when the route declares no tests - the plan would then execute + * nothing, which is the one outcome a verification tool must never report as a pass. */ + sharedChecks?: "always" | "none"; } export interface RouteConfig { @@ -160,23 +167,38 @@ function readConfig(root: string): AgentContextConfig { throw new Error(`${route.id}: ${field} must be a string array`); } } - if ( - !route.verify || - !isStringArray(route.verify.blocking) || - !isStringArray(route.verify.advisory) - ) { - throw new Error(`${route.id}: verify must declare blocking and advisory script arrays`); - } - const overlap = route.verify.blocking.find((command) => - route.verify.advisory.includes(command), - ); - if (overlap) { - throw new Error(`${route.id}: npm script ${overlap} cannot be both blocking and advisory`); - } + validateRouteVerify(route); } return parsed as AgentContextConfig; } +/** One route's verification declaration. Kept out of `readConfig` because that function is already + * above the complexity gate's limit: a new rule must not ratchet it. */ +function validateRouteVerify(route: RouteConfig): void { + if ( + !route.verify || + !isStringArray(route.verify.blocking) || + !isStringArray(route.verify.advisory) + ) { + throw new Error(`${route.id}: verify must declare blocking and advisory script arrays`); + } + const overlap = route.verify.blocking.find((command) => route.verify.advisory.includes(command)); + if (overlap) { + throw new Error(`${route.id}: npm script ${overlap} cannot be both blocking and advisory`); + } + const sharedChecks = route.verify.sharedChecks; + if (sharedChecks !== undefined && sharedChecks !== "always" && sharedChecks !== "none") { + throw new Error( + `${route.id}: verify.sharedChecks must be "always" or "none", not ${JSON.stringify(sharedChecks)}`, + ); + } + if (sharedChecks === "none" && !route.tests.length) { + throw new Error( + `${route.id}: a route that declines the shared checks must declare its own tests, or a narrow run would execute nothing`, + ); + } +} + function isStringArray(value: unknown): value is string[] { return ( Array.isArray(value) && @@ -555,6 +577,18 @@ export function validateAgentContext(root: string): string[] { ]; } +/** The reconciliation section says so when a sweep holds the tree (post-mortem 0003): every check ordered + * from here would read the mutant. Returned as lines rather than pushed in place, because + * `formatAgentContext` is already above the complexity limit and this must not add a branch to it. + * + * The root is the report's own, never `process.cwd()`: the warning is about the tree the report + * describes, and reading the caller's directory made the same report differ depending on where it was + * rendered from - which a suite comparing exact output caught. */ +function mutationHazardLines(root: string): string[] { + const hazard = mutationHazard(root); + return hazard ? [`- Warning: ${hazard} - checks would read the mutant`] : []; +} + export function formatAgentContext(report: AgentContextReport): string { const lines = [ `# Repository context: ${report.project}@${report.version}`, @@ -575,6 +609,8 @@ export function formatAgentContext(report: AgentContextReport): string { lines.push(`- TODO: ${report.canonical.todo}`); lines.push("", "## Reconciliation"); lines.push(`- Status: ${report.reconciliation.status}`); + // The first command a session runs is the place to say that the tree is mid-mutation (post-mortem 0003). + lines.push(...mutationHazardLines(report.root)); lines.push(`- Desired revision: ${report.state.desiredRevision.slice(0, 12)}`); lines.push(`- Observed revision: ${report.state.observedRevision.slice(0, 12)}`); for (const condition of report.reconciliation.conditions) {