diff --git a/api-surface.json b/api-surface.json index 01741c80..48667ac0 100644 --- a/api-surface.json +++ b/api-surface.json @@ -1585,7 +1585,7 @@ "runBenchmark": "value a00b82e95062", "runCancelRequestFile": "value 229aecaab891", "runCancellationFile": "value 229aecaab891", - "runFinalizer": "value 34d3218fb8ef", + "runFinalizer": "value fc5202156e7a", "runGraph": "value 7d72ff33c325", "runInWorkspace": "value 624b01e4ba38", "runIsolatedCheck": "value ebb5d56a353b", diff --git a/docs/api/runtime.md b/docs/api/runtime.md index cbfc58b9..d9731cce 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -34436,7 +34436,11 @@ readonly `T`[] Run a finalizer over a settled-worker ledger under the delivered-only invariant: filter the ledger to structurally delivered children, materialize their outputs, and hand the finalizer a blob reader that throws on any ref outside that set. This is the one call site both driver arms -(the in-process tool-loop and the MCP-mounted harness) finalize through. +(the in-process tool-loop and the MCP-mounted harness) finalize through. When a parent declares +a deliverable, its candidate must also pass that check: children may have narrower assignments. +The default selects the highest-scoring child that passes the parent's check; custom finalizers +assemble their candidate before the check. A rejection leaves the parent incomplete; a thrown +oracle surfaces a validation error. Neither changes child validity. #### Parameters @@ -34462,6 +34466,10 @@ readonly [`FinalizerSettled`](#finalizersettled)[] `Readonly`\<\{ `resources?`: `Readonly`\<`Record`\<`string`, \{ `unit`: `string`; `limit`: `number`; `remaining`: `number`; `reserved`: `number`; `committed`: `number`; `known`: `boolean`; \}\>\>; `tokensLeft`: `number`; `tokensKnown`: `boolean`; `cacheBreakdownKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> +###### deliverable? + +[`DeliverableSpec`](#deliverablespec)\<`unknown`\> + #### Returns `Promise`\<`unknown`\> diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 3fc009a2..ef98b093 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -127,6 +127,14 @@ A general "loop" primitive is the single most common modelling error in this rep **The trap** is a single grammar (`defineLoop`, a `runXxxLoop`) spanning all of the above: there can't be one, because some are code and one is a model deciding. No new loop primitive lands without a tiny executable proof, **over real agents**, of the exact substrate join it claims to simplify. +A child can satisfy its own assignment while its parent's objective remains incomplete. +When the parent declares a `deliverable`, its finalizer's candidate must pass that check before the parent completes. +The default selects the highest-scoring child that also passes the parent's check. +Custom finalizers assemble their candidate before the check. +Child validity and delivery counts remain unchanged. +An external director can then use `repromptOnUnmet` to continue from a rejected candidate within its existing resource limits. +A thrown parent check reports a validation error through the existing driver failure record. + | I want to… | Use (import) | Do NOT build | |---|---|---| | Run one product chat turn with streamed events, ordered persistence hooks, and stable execution/turn identity | `handleChatTurn(...)` + `deriveExecutionId(...)`: `/durable`; pass the derived id as both `executionId` and `turnId` on initial dispatch | importing the broad package entry from an edge worker, treating `executionId` alone as dispatch idempotency, or rebuilding framing and persistence ordering in the product | diff --git a/src/runtime/supervise/coordination-driver.ts b/src/runtime/supervise/coordination-driver.ts index 1d20cd06..605fdecc 100644 --- a/src/runtime/supervise/coordination-driver.ts +++ b/src/runtime/supervise/coordination-driver.ts @@ -1289,6 +1289,7 @@ export function driverAgent(opts: DriverAgentOptions): Agent { blobs: opts.blobs, tree: runTree(scope), budget: scope.budget, + ...(opts.deliverable ? { deliverable: opts.deliverable } : {}), }) }, } diff --git a/src/runtime/supervise/finalizer.ts b/src/runtime/supervise/finalizer.ts index 4fc072b1..4040ea5e 100644 --- a/src/runtime/supervise/finalizer.ts +++ b/src/runtime/supervise/finalizer.ts @@ -19,6 +19,7 @@ */ import { ValidationError } from '../../errors' +import type { DeliverableSpec } from './completion-gate' import type { ResultBlobStore, Scope, TreeView } from './types' /** One settled worker as the finalizer sees it — the ledger row (structural fields only). */ @@ -114,7 +115,11 @@ export const collectDelivered: SupervisorFinalizer = (ctx) => { * Run a finalizer over a settled-worker ledger under the delivered-only invariant: filter the * ledger to structurally delivered children, materialize their outputs, and hand the finalizer a * blob reader that throws on any ref outside that set. This is the one call site both driver arms - * (the in-process tool-loop and the MCP-mounted harness) finalize through. + * (the in-process tool-loop and the MCP-mounted harness) finalize through. When a parent declares + * a deliverable, its candidate must also pass that check: children may have narrower assignments. + * The default selects the highest-scoring child that passes the parent's check; custom finalizers + * assemble their candidate before the check. A rejection leaves the parent incomplete; a thrown + * oracle surfaces a validation error. Neither changes child validity. */ export async function runFinalizer( finalizer: SupervisorFinalizer, @@ -123,6 +128,7 @@ export async function runFinalizer( readonly blobs: ResultBlobStore readonly tree: TreeView readonly budget: Scope['budget'] + readonly deliverable?: DeliverableSpec }, ): Promise { const deliveredRows = args.settled.filter((w) => w.status === 'done' && w.valid === true) @@ -146,11 +152,31 @@ export async function runFinalizer( return args.blobs.get(outRef) }, } - return finalizer({ + const accepted = async (candidate: unknown): Promise => { + if (candidate === undefined) return false + if (args.deliverable === undefined) return true + try { + return (await args.deliverable.check(candidate)) === true + } catch (error) { + throw new ValidationError( + `finalizer: parent completion check threw: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + // A high-scoring partial assignment cannot displace a complete portfolio member. Aggregating + // finalizers still need every child-valid component, so their inputs must not be filtered here. + if (finalizer === bestDelivered && args.deliverable !== undefined) { + for (const output of [...delivered].sort((a, b) => (b.score ?? 0) - (a.score ?? 0))) { + if (await accepted(output.out)) return output.out + } + return undefined + } + const candidate = await finalizer({ delivered, allSettled: args.settled, tree: args.tree, blobs: guardedBlobs, budget: args.budget, }) + return (await accepted(candidate)) ? candidate : undefined } diff --git a/src/runtime/supervise/supervisor-agent.ts b/src/runtime/supervise/supervisor-agent.ts index 31387316..5b234bd7 100644 --- a/src/runtime/supervise/supervisor-agent.ts +++ b/src/runtime/supervise/supervisor-agent.ts @@ -982,9 +982,18 @@ function buildSupervisorAgent( const baseTokensLeft = scope.budget.tokensLeft const contractDeclared = deps.deliverable !== undefined const maxReprompts = deps.repromptOnUnmet ?? 0 + let candidate: unknown + const finalize = () => + runFinalizer(deps.finalizer ?? bestDelivered, { + settled: mcp.settled(), + blobs: deps.blobs, + tree: runTree(scope), + budget: scope.budget, + ...(deps.deliverable ? { deliverable: deps.deliverable } : {}), + }) const readProgress = (): DriverProgressMark => { const settled = mcp.settled() - // The same delivered-only rule the finalizer applies: settled `done` AND check-passed. + // Child delivery is progress, but its check may cover only the child's assignment. const deliveredCount = settled.filter( (w) => w.status === 'done' && w.valid === true, ).length @@ -996,13 +1005,14 @@ function buildSupervisorAgent( deliveredCount, contract: !contractDeclared ? 'none' - : submitted || deliveredCount > 0 + : submitted || candidate !== undefined ? 'met' : 'unmet', } } await runDriverWithRetry({ drive: async (attempt, reentry) => { + candidate = undefined if (deps.controlDir !== undefined && deps.abortRun !== undefined) { applyRunCancellation(deps.controlDir, deps.abortRun, () => new Date().toISOString()) } @@ -1029,6 +1039,12 @@ function buildSupervisorAgent( // an accepted submission the backend error propagates into the retry decision. if (!mcp.submittedResult() && !mcp.isStopped()) throw error } + // Decide this parent's completion before the retry loop reads progress. Cache the + // checked candidate so neither the finalizer nor its oracle runs twice on return. + if (contractDeclared && !mcp.submittedResult()) { + await mcp.drainResolved() + candidate = await finalize() + } }, progress: readProgress, budget: () => scope.budget, @@ -1059,9 +1075,8 @@ function buildSupervisorAgent( : {}), ...(deps.onDriverAttempt ? { onAttempt: deps.onDriverAttempt } : {}), }) - // Drain settled-but-unpulled children first — a gate-verified delivery the harness never - // awaited must still reach the finalize ledger. - await mcp.drainResolved() + // Without a parent oracle, preserve the single finalization after the driver finishes. + if (!contractDeclared) await mcp.drainResolved() // Direct work is eligible only through `submit_result`, after the injected independent // check passes. Raw harness prose remains ineligible. const submitted = mcp.submittedResult() @@ -1071,12 +1086,7 @@ function buildSupervisorAgent( } // The deliverable comes from the finalizer seam over DELIVERED children only — never the // harness's own output (Foreman 0/18). Default keep-best. - return await runFinalizer(deps.finalizer ?? bestDelivered, { - settled: mcp.settled(), - blobs: deps.blobs, - tree: runTree(scope), - budget: scope.budget, - }) + return contractDeclared ? candidate : await finalize() } finally { await mcp.close() } diff --git a/tests/kernel/supervise-convenience.test.ts b/tests/kernel/supervise-convenience.test.ts index ad65dd36..d3b41c8a 100644 --- a/tests/kernel/supervise-convenience.test.ts +++ b/tests/kernel/supervise-convenience.test.ts @@ -251,6 +251,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ it('uses a child-specific completion check when a managed child submits its own result', async () => { let childCheckCalls = 0 + const journal = new InMemorySpawnJournal() const child = testAgentProfile('specialist', { harness: 'opencode', tools: runtimeToolDeclarations('submit_result'), @@ -288,6 +289,8 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ perWorker: { maxIterations: 4, maxTokens: 10_000 }, makeLeafAgent: () => deliveringLeaf('unused', {}), driveHarness, + journal, + runId: 'parent-contract', // The run-wide check deliberately rejects the child's output. A profile-managed child // must instead receive the check selected for its exact authorized assignment. deliverable: { check: () => false }, @@ -307,9 +310,22 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ }, ) - expect(result.kind).toBe('winner') - if (result.kind === 'winner') expect(result.out).toEqual({ answer: 42 }) + // The child's own check is what judged the child: consulted exactly once, and it passed. expect(childCheckCalls).toBe(1) + expect(await journal.loadTree('parent-contract')).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'settled', + id: 'parent-contract:s0', + status: 'done', + verdict: expect.objectContaining({ valid: true }), + }), + ]), + ) + // A child that satisfies its narrower assignment does not complete the parent. The run-wide + // check rejects the child's output, no continuation is configured, so the parent's contract + // stays unmet and the partial component is not promoted to the winner. + expect(result.kind).toBe('no-winner') }) it('runDir makes the run durable and resumable; unset stays in-memory', async () => { diff --git a/tests/kernel/supervisor-agent.test.ts b/tests/kernel/supervisor-agent.test.ts index 4447e587..2b512a77 100644 --- a/tests/kernel/supervisor-agent.test.ts +++ b/tests/kernel/supervisor-agent.test.ts @@ -1515,6 +1515,161 @@ describe('supervisorAgent — coordination bind + prompt hoisting on the harness expect(tasks).toHaveLength(1) }) + it('EXTERNAL arm: a valid partial child leaves the parent contract unmet and re-enters its director', async () => { + const blobs = new InMemoryResultBlobStore() + const journal = new InMemorySpawnJournal() + const checked: unknown[] = [] + const progress: unknown[] = [] + let drives = 0 + let finalizations = 0 + const partial = { component: 'database', ready: true } + const complete = { answer: 42 } + const root = supervisorAgent( + testAgentProfile('sup', { + harness: 'pi', + tools: runtimeToolDeclarations('spawn_worker', 'await_event', 'submit_result'), + }), + { + blobs, + makeWorkerAgent: () => deliveringLeaf('component', partial), + perWorker, + driveHarness: async ({ coordinationMcpUrl }) => { + drives += 1 + if (drives === 1) { + await jsonRpc(coordinationMcpUrl, 'tools/call', { + name: 'spawn_worker', + arguments: { profile: testAgentProfile('component'), task: 'build the database' }, + }) + await jsonRpc(coordinationMcpUrl, 'tools/call', { + name: 'await_event', + arguments: {}, + }) + return + } + await jsonRpc(coordinationMcpUrl, 'tools/call', { + name: 'submit_result', + arguments: { result: complete }, + }) + }, + finalizer: ({ delivered }) => { + finalizations += 1 + expect(delivered).toHaveLength(1) + return delivered[0]?.out + }, + deliverable: { + check: (out) => { + checked.push(out) + return JSON.stringify(out) === JSON.stringify(complete) + }, + }, + repromptOnUnmet: 1, + onUnmetContract: (context) => { + progress.push(context.progress) + return { steer: 'Integrate the database into the complete product.' } + }, + }, + ) + + const result = await runSupervisor(root, blobs, journal) + expect(result.kind).toBe('winner') + if (result.kind === 'winner') expect(result.out).toEqual(complete) + expect(drives).toBe(2) + expect(finalizations).toBe(1) + expect(checked).toEqual([partial, complete]) + expect(progress).toEqual([expect.objectContaining({ deliveredCount: 1, contract: 'unmet' })]) + expect(await journal.loadTree('sup')).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'settled', + id: 'sup:s0', + status: 'done', + verdict: expect.objectContaining({ valid: true }), + }), + ]), + ) + }) + + it.each([ + { name: 'partial', out: { component: 'database' }, valid: false }, + { name: 'complete', out: { answer: 42 }, valid: true }, + { name: 'throwing check', out: { throw: true }, valid: false }, + { name: 'aggregate', out: { component: 'database' }, aggregate: true, valid: true }, + ])('both arms check the parent contract for a $name child candidate', async (scenario) => { + for (const harness of ['pi', 'cli-base'] as const) { + const blobs = new InMemoryResultBlobStore() + const journal = new InMemorySpawnJournal() + let checks = 0 + let finalizations = 0 + const root = supervisorAgent( + testAgentProfile('sup', { + harness, + tools: runtimeToolDeclarations('spawn_worker', 'await_event'), + }), + { + blobs, + makeWorkerAgent: () => deliveringLeaf('component', scenario.out), + perWorker, + ...(harness === 'pi' + ? { + driveHarness: async ({ coordinationMcpUrl }: Parameters[0]) => { + await jsonRpc(coordinationMcpUrl, 'tools/call', { + name: 'spawn_worker', + arguments: { profile: testAgentProfile('component'), task: 'build component' }, + }) + await jsonRpc(coordinationMcpUrl, 'tools/call', { + name: 'await_event', + arguments: {}, + }) + }, + } + : { + brain: scriptedBrain([ + { + toolCalls: [ + { + name: 'spawn_worker', + arguments: { + profile: testAgentProfile('component'), + task: 'build component', + }, + }, + ], + }, + { toolCalls: [{ name: 'await_event', arguments: {} }] }, + { content: 'done' }, + ]), + }), + finalizer: ({ delivered }) => { + finalizations += 1 + return scenario.aggregate ? { answer: 42 } : delivered[0]?.out + }, + deliverable: { + check: (out) => { + checks += 1 + if (scenario.name === 'throwing check') throw new Error('oracle unavailable') + return JSON.stringify(out) === JSON.stringify({ answer: 42 }) + }, + }, + }, + ) + + const result = await runSupervisor(root, blobs, journal) + expect(result.kind, `${harness}: ${scenario.name}`).toBe( + scenario.valid ? 'winner' : 'no-winner', + ) + if (scenario.name === 'throwing check' && result.kind === 'no-winner') { + expect(result.reason).toBe('driver-failed') + if (result.reason === 'driver-failed') { + expect(result.error.message).toContain( + 'parent completion check threw: oracle unavailable', + ) + } + } + expect(checks).toBe(1) + expect(finalizations).toBe(1) + } + }) + it('EXTERNAL arm: a run the coordination server STOPPED is never re-prompted', async () => { // The driver called `stop`. That was a decision, and Runtime refuses the re-prompt before the // product hook is consulted, so no hook can talk the run past its own stop. diff --git a/tests/runtime/supervisor-finalizer.test.ts b/tests/runtime/supervisor-finalizer.test.ts index 98f4ce26..a605c668 100644 --- a/tests/runtime/supervisor-finalizer.test.ts +++ b/tests/runtime/supervisor-finalizer.test.ts @@ -138,6 +138,42 @@ describe('SupervisorFinalizer — bestDelivered is the unchanged default', () => expect(await runWith(bestDelivered, fx)).toBeUndefined() expect(await runWith(collectDelivered, fx)).toBeUndefined() }) + + it.each([true, false])( + 'checks the parent objective in score order with a complete sibling present: %s', + async (completeSibling) => { + const fx = await ledgerFixture([ + { id: 'component', status: 'done', valid: true, score: 0.9, payload: 'database ready' }, + { + id: 'product', + status: 'done', + valid: true, + score: 0.5, + payload: completeSibling ? 'product verified' : 'api ready', + }, + { id: 'invalid', status: 'done', valid: false, score: 1, payload: 'product verified' }, + ]) + const checked: unknown[] = [] + const result = await runFinalizer(bestDelivered, { + settled: fx.rows, + blobs: fx.blobs, + tree: emptyTree, + budget: poolReadout, + deliverable: { + check: (candidate) => { + checked.push(candidate) + return candidate === 'product verified' + }, + }, + }) + expect(result).toBe(completeSibling ? 'product verified' : undefined) + expect(checked).toEqual([ + 'database ready', + completeSibling ? 'product verified' : 'api ready', + ]) + expect(fx.rows.map((row) => row.valid)).toEqual([true, true, false]) + }, + ) }) describe('SupervisorFinalizer — collectDelivered is the second implementation', () => {