From 60978f128d6117c17d30f7ea31aa790f0b373591 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 15 Sep 2026 23:05:03 -0700 Subject: [PATCH 1/2] fix(supervise): separate completion from failure retries --- api-surface.json | 8 +- docs/agent-managed-compute/reliability.md | 5 +- docs/api/durable.md | 9 +- docs/api/runtime.md | 43 +- docs/api/testing.md | 27 +- docs/canonical-api.md | 6 +- src/runtime/supervise/coordination-driver.ts | 39 +- src/runtime/supervise/driver-retry.test.ts | 134 ++++-- src/runtime/supervise/driver-retry.ts | 40 +- src/runtime/supervise/supervise.ts | 31 +- src/runtime/supervise/supervisor-agent.ts | 24 +- .../supervise-full-profile-bridge.test.ts | 23 + .../supervise-retained-owner-recovery.test.ts | 403 +++++++++++------- tests/kernel/supervisor-agent.test.ts | 27 +- 14 files changed, 530 insertions(+), 289 deletions(-) diff --git a/api-surface.json b/api-surface.json index 8e9f205c..307fb023 100644 --- a/api-surface.json +++ b/api-surface.json @@ -890,10 +890,10 @@ "DriverContractState": "type a287e992fd8a", "DriverProgressMark": "type cb7863a2e835", "DriverReentry": "type 856da6132418", - "DriverRepromptPolicy": "type 0761759617c8", + "DriverRepromptPolicy": "type 0f79f31dc99e", "DriverRepromptRefusal": "type 2b7a1302459a", "DriverRetryPolicy": "type b5607c21ee53", - "DriverUnmetContractContext": "type 9e6627e11f90", + "DriverUnmetContractContext": "type 2c4d72703ac4", "DriverUnmetContractDecision": "type f62a992629d1", "EVIDENCE_MAX_CHARS": "value 60366a39f55c", "EdgeDeliveryOutcome": "type ede8c2ce5f1a", @@ -1253,7 +1253,7 @@ "StructuralRolloutPolicy": "type d372912ef050", "StructuralRolloutResult": "type 9248b72cae04", "SuperviseDispatchOptions": "type 55e71c9c1e6b", - "SuperviseOptions": "type 7280b2a63be3", + "SuperviseOptions": "type 1047255658db", "SuperviseOptionsForDispatch": "type 7d89526c1040", "SuperviseRegistry": "type 4fd60c297f74", "SuperviseRegistryTable": "type cc1468cd50c1", @@ -1261,7 +1261,7 @@ "SuperviseSurfaceResult": "type 2c378dbc3193", "SupervisedResult": "type 924803bb65bd", "Supervisor": "type 7d9aff9cd744", - "SupervisorAgentDeps": "type b2683b00f9e6", + "SupervisorAgentDeps": "type 763549b3d5d6", "SupervisorCleanupReceipt": "type d862eb60266d", "SupervisorFinalizer": "type f8628e65536f", "SupervisorNodeContext": "type b8e545bbb355", diff --git a/docs/agent-managed-compute/reliability.md b/docs/agent-managed-compute/reliability.md index 38b5647c..deb36839 100644 --- a/docs/agent-managed-compute/reliability.md +++ b/docs/agent-managed-compute/reliability.md @@ -35,7 +35,10 @@ Recovery validates the actual retained bytes before accepting another append. The index is not a second durable record or a cross-process ownership fence. Observer records detach their inputs before queued I/O, so later hook mutations cannot rewrite the evidence being saved. Completed director invocations reset the consecutive transport-failure counter even when the pursuit remains incomplete. -The explicit total-attempt, deadline, cancellation, and resource bounds still apply. +The failure-attempt, deadline, cancellation, and resource bounds still apply. +Successful incomplete invocations do not consume `driverRetry.maxAttempts`; only failed invocations consume that allowance. +Use `repromptOnUnmet: 'until-complete'` with a completion check and finite positive budget deadline to omit the continuation count cap. +Numeric continuation caps retain their meaning, and zero still disables continuation. The file run lock protects one local coordinator. It does not fence provider mutations from a partitioned coordinator on another machine. diff --git a/docs/api/durable.md b/docs/api/durable.md index 64a24350..201a1e6c 100644 --- a/docs/api/durable.md +++ b/docs/api/durable.md @@ -1749,7 +1749,7 @@ Per-re-entry record for every retried worker spawn — what makes a saturated ex ##### repromptOnUnmet? -> `readonly` `optional` **repromptOnUnmet?**: `number` +> `readonly` `optional` **repromptOnUnmet?**: `number` \| `"until-complete"` How many times an EXTERNAL-harness driver that RETURNED with `deliverable` still unmet is re-entered on the SAME live session with the unmet items. @@ -1760,12 +1760,15 @@ runs): 376 of 376 winning runs ended on the driver's own completion, and the com could only LABEL an undelivered result `valid:false`, never send the driver back for it. A re-prompt is the retry path, not a second loop: same scope, same coordination server, same -live children, and the same budget, deadline, abort, and `driverRetry.maxAttempts` bounds. A +live children, and the same budget, deadline, and abort bounds. Successful continuations do +not consume `driverRetry.maxAttempts`, which counts failed invocations only. A run the coordination server already stopped is never re-prompted — that stop was a decision. Requires `deliverable`, and applies to every external manager with a completion check. A recursive manager receives the check selected for its exact assignment. Refused for a router-brained manager, which runs its turn loop in process. Omit/`0` = never. +Use `'until-complete'` with a finite positive budget deadline to remove the continuation cap. +Completion, explicit stop, cancellation, resource limits, and failure limits still stop work. ###### Inherited from @@ -1776,7 +1779,7 @@ router-brained manager, which runs its turn loop in process. Omit/`0` = never. > `readonly` `optional` **onUnmetContract?**: [`OnUnmetContract`](runtime.md#onunmetcontract) Compose the re-entry instruction for an unmet contract, or return `'stop'` to end the run. - Requires `repromptOnUnmet >= 1`. Omit = Runtime's own instruction, which names what the run + Requires positive `repromptOnUnmet` or `'until-complete'`. Omit = Runtime's instruction, which names what the run owes and reports how many workers passed the check. ###### Inherited from diff --git a/docs/api/runtime.md b/docs/api/runtime.md index ad2cb801..6c34ade6 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -13842,10 +13842,9 @@ Consecutive failures that changed NOTHING (no metered spend, no settlement, no s > `readonly` `optional` **maxAttempts?**: `number` -Absolute ceiling on attempts, regardless of progress. Default 8. The barren counter alone - cannot bound a driver that crashes every turn AFTER metering a little: each attempt looks like - progress, so without this backstop such a run would retry until it had eaten the entire - envelope. A caller who wants budget-only bounding sets this high deliberately. +Ceiling on failed invocations across this driver run, regardless of progress. Default 8, + minimum 1. Successful continuations do not consume this allowance or reset it. + This bounds repeated crashes that each make enough progress to reset the barren streak. ##### initialBackoffMs? @@ -14019,7 +14018,7 @@ How many re-prompts this run has already issued. ##### maxReprompts -> `readonly` **maxReprompts**: `number` +> `readonly` **maxReprompts**: `number` \| `"until-complete"` ##### progress @@ -14048,10 +14047,11 @@ How a completed-but-undelivered drive is re-entered. Absent = the historical beh ##### maxReprompts -> `readonly` **maxReprompts**: `number` +> `readonly` **maxReprompts**: `number` \| `"until-complete"` -How many times one run may re-enter its live session with the unmet items. `0` = never. Each - re-prompt also consumes an attempt, so `maxAttempts` bounds it too. +How many times one run may re-enter its live session with the unmet items. `0` = never. + `'until-complete'` removes the count cap and requires a finite positive scope deadline. + Budget, cancellation, explicit stop, and failure retry limits still apply. ##### onUnmetContract? @@ -14891,7 +14891,7 @@ Per-re-entry record for every retried worker spawn — what makes a saturated ex ##### repromptOnUnmet? -> `readonly` `optional` **repromptOnUnmet?**: `number` +> `readonly` `optional` **repromptOnUnmet?**: `number` \| `"until-complete"` How many times an EXTERNAL-harness driver that RETURNED with `deliverable` still unmet is re-entered on the SAME live session with the unmet items. @@ -14902,12 +14902,15 @@ runs): 376 of 376 winning runs ended on the driver's own completion, and the com could only LABEL an undelivered result `valid:false`, never send the driver back for it. A re-prompt is the retry path, not a second loop: same scope, same coordination server, same -live children, and the same budget, deadline, abort, and `driverRetry.maxAttempts` bounds. A +live children, and the same budget, deadline, and abort bounds. Successful continuations do +not consume `driverRetry.maxAttempts`, which counts failed invocations only. A run the coordination server already stopped is never re-prompted — that stop was a decision. Requires `deliverable`, and applies to every external manager with a completion check. A recursive manager receives the check selected for its exact assignment. Refused for a router-brained manager, which runs its turn loop in process. Omit/`0` = never. +Use `'until-complete'` with a finite positive budget deadline to remove the continuation cap. +Completion, explicit stop, cancellation, resource limits, and failure limits still stop work. ###### Inherited from @@ -14918,7 +14921,7 @@ router-brained manager, which runs its turn loop in process. Omit/`0` = never. > `readonly` `optional` **onUnmetContract?**: [`OnUnmetContract`](#onunmetcontract) Compose the re-entry instruction for an unmet contract, or return `'stop'` to end the run. - Requires `repromptOnUnmet >= 1`. Omit = Runtime's own instruction, which names what the run + Requires positive `repromptOnUnmet` or `'until-complete'`. Omit = Runtime's instruction, which names what the run owes and reports how many workers passed the check. ###### Inherited from @@ -19054,7 +19057,7 @@ Per-re-entry record for every retried worker spawn — what makes a saturated ex ##### repromptOnUnmet? -> `readonly` `optional` **repromptOnUnmet?**: `number` +> `readonly` `optional` **repromptOnUnmet?**: `number` \| `"until-complete"` How many times an EXTERNAL-harness driver that RETURNED with `deliverable` still unmet is re-entered on the SAME live session with the unmet items. @@ -19065,19 +19068,22 @@ runs): 376 of 376 winning runs ended on the driver's own completion, and the com could only LABEL an undelivered result `valid:false`, never send the driver back for it. A re-prompt is the retry path, not a second loop: same scope, same coordination server, same -live children, and the same budget, deadline, abort, and `driverRetry.maxAttempts` bounds. A +live children, and the same budget, deadline, and abort bounds. Successful continuations do +not consume `driverRetry.maxAttempts`, which counts failed invocations only. A run the coordination server already stopped is never re-prompted — that stop was a decision. Requires `deliverable`, and applies to every external manager with a completion check. A recursive manager receives the check selected for its exact assignment. Refused for a router-brained manager, which runs its turn loop in process. Omit/`0` = never. +Use `'until-complete'` with a finite positive budget deadline to remove the continuation cap. +Completion, explicit stop, cancellation, resource limits, and failure limits still stop work. ##### onUnmetContract? > `readonly` `optional` **onUnmetContract?**: [`OnUnmetContract`](#onunmetcontract) Compose the re-entry instruction for an unmet contract, or return `'stop'` to end the run. - Requires `repromptOnUnmet >= 1`. Omit = Runtime's own instruction, which names what the run + Requires positive `repromptOnUnmet` or `'until-complete'`. Omit = Runtime's instruction, which names what the run owes and reports how many workers passed the check. ##### childSettleGraceMs? @@ -20153,22 +20159,23 @@ Per-attempt record for the external driver — how an operator sees "failed afte ##### repromptOnUnmet? -> `readonly` `optional` **repromptOnUnmet?**: `number` +> `readonly` `optional` **repromptOnUnmet?**: `number` \| `"until-complete"` How many times an EXTERNAL driver that RETURNED with `deliverable` still unmet is re-entered on the SAME live session with the unmet items. The harness owns its own turn loop, so it can end while the run has delivered nothing — 376 of 376 winning discovery-lab runs (2026-09-01) ended on the driver's own completion, and the completion gate could only label that result, never change it. A re-prompt reuses the retry path: same scope, same coordination server, same - live children, same budget/deadline/abort/attempt bounds. Requires `deliverable`; refused for - a router-brained supervisor, which runs its loop in process. Omit/`0` = never re-prompt. + live children, same budget/deadline/abort bounds. Successful turns do not consume failure + retries. Use `'until-complete'` with a finite positive scope deadline to omit the count cap. + Requires `deliverable`; refused for a router-brained supervisor. Omit/`0` = never re-prompt. ##### onUnmetContract? > `readonly` `optional` **onUnmetContract?**: [`OnUnmetContract`](#onunmetcontract) Compose the re-entry instruction for an unmet contract, or return `'stop'` to end the run. - Requires `repromptOnUnmet >= 1`. Omit = Runtime's own instruction. + Requires positive `repromptOnUnmet` or `'until-complete'`. Omit = Runtime's own instruction. ##### nodeContext? diff --git a/docs/api/testing.md b/docs/api/testing.md index 9f32e7f2..9881bf37 100644 --- a/docs/api/testing.md +++ b/docs/api/testing.md @@ -865,7 +865,7 @@ Per-re-entry record for every retried worker spawn — what makes a saturated ex ##### repromptOnUnmet? -> `readonly` `optional` **repromptOnUnmet?**: `number` +> `readonly` `optional` **repromptOnUnmet?**: `number` \| `"until-complete"` How many times an EXTERNAL-harness driver that RETURNED with `deliverable` still unmet is re-entered on the SAME live session with the unmet items. @@ -876,12 +876,15 @@ runs): 376 of 376 winning runs ended on the driver's own completion, and the com could only LABEL an undelivered result `valid:false`, never send the driver back for it. A re-prompt is the retry path, not a second loop: same scope, same coordination server, same -live children, and the same budget, deadline, abort, and `driverRetry.maxAttempts` bounds. A +live children, and the same budget, deadline, and abort bounds. Successful continuations do +not consume `driverRetry.maxAttempts`, which counts failed invocations only. A run the coordination server already stopped is never re-prompted — that stop was a decision. Requires `deliverable`, and applies to every external manager with a completion check. A recursive manager receives the check selected for its exact assignment. Refused for a router-brained manager, which runs its turn loop in process. Omit/`0` = never. +Use `'until-complete'` with a finite positive budget deadline to remove the continuation cap. +Completion, explicit stop, cancellation, resource limits, and failure limits still stop work. ###### Inherited from @@ -892,7 +895,7 @@ router-brained manager, which runs its turn loop in process. Omit/`0` = never. > `readonly` `optional` **onUnmetContract?**: [`OnUnmetContract`](runtime.md#onunmetcontract) Compose the re-entry instruction for an unmet contract, or return `'stop'` to end the run. - Requires `repromptOnUnmet >= 1`. Omit = Runtime's own instruction, which names what the run + Requires positive `repromptOnUnmet` or `'until-complete'`. Omit = Runtime's instruction, which names what the run owes and reports how many workers passed the check. ###### Inherited from @@ -1707,7 +1710,7 @@ Per-re-entry record for every retried worker spawn — what makes a saturated ex ##### repromptOnUnmet? -> `readonly` `optional` **repromptOnUnmet?**: `number` +> `readonly` `optional` **repromptOnUnmet?**: `number` \| `"until-complete"` How many times an EXTERNAL-harness driver that RETURNED with `deliverable` still unmet is re-entered on the SAME live session with the unmet items. @@ -1718,12 +1721,15 @@ runs): 376 of 376 winning runs ended on the driver's own completion, and the com could only LABEL an undelivered result `valid:false`, never send the driver back for it. A re-prompt is the retry path, not a second loop: same scope, same coordination server, same -live children, and the same budget, deadline, abort, and `driverRetry.maxAttempts` bounds. A +live children, and the same budget, deadline, and abort bounds. Successful continuations do +not consume `driverRetry.maxAttempts`, which counts failed invocations only. A run the coordination server already stopped is never re-prompted — that stop was a decision. Requires `deliverable`, and applies to every external manager with a completion check. A recursive manager receives the check selected for its exact assignment. Refused for a router-brained manager, which runs its turn loop in process. Omit/`0` = never. +Use `'until-complete'` with a finite positive budget deadline to remove the continuation cap. +Completion, explicit stop, cancellation, resource limits, and failure limits still stop work. ###### Inherited from @@ -1734,7 +1740,7 @@ router-brained manager, which runs its turn loop in process. Omit/`0` = never. > `readonly` `optional` **onUnmetContract?**: [`OnUnmetContract`](runtime.md#onunmetcontract) Compose the re-entry instruction for an unmet contract, or return `'stop'` to end the run. - Requires `repromptOnUnmet >= 1`. Omit = Runtime's own instruction, which names what the run + Requires positive `repromptOnUnmet` or `'until-complete'`. Omit = Runtime's instruction, which names what the run owes and reports how many workers passed the check. ###### Inherited from @@ -2382,15 +2388,16 @@ Per-attempt record for the external driver — how an operator sees "failed afte ##### repromptOnUnmet? -> `readonly` `optional` **repromptOnUnmet?**: `number` +> `readonly` `optional` **repromptOnUnmet?**: `number` \| `"until-complete"` How many times an EXTERNAL driver that RETURNED with `deliverable` still unmet is re-entered on the SAME live session with the unmet items. The harness owns its own turn loop, so it can end while the run has delivered nothing — 376 of 376 winning discovery-lab runs (2026-09-01) ended on the driver's own completion, and the completion gate could only label that result, never change it. A re-prompt reuses the retry path: same scope, same coordination server, same - live children, same budget/deadline/abort/attempt bounds. Requires `deliverable`; refused for - a router-brained supervisor, which runs its loop in process. Omit/`0` = never re-prompt. + live children, same budget/deadline/abort bounds. Successful turns do not consume failure + retries. Use `'until-complete'` with a finite positive scope deadline to omit the count cap. + Requires `deliverable`; refused for a router-brained supervisor. Omit/`0` = never re-prompt. ###### Inherited from @@ -2401,7 +2408,7 @@ How many times an EXTERNAL driver that RETURNED with `deliverable` still unmet i > `readonly` `optional` **onUnmetContract?**: [`OnUnmetContract`](runtime.md#onunmetcontract) Compose the re-entry instruction for an unmet contract, or return `'stop'` to end the run. - Requires `repromptOnUnmet >= 1`. Omit = Runtime's own instruction. + Requires positive `repromptOnUnmet` or `'until-complete'`. Omit = Runtime's own instruction. ###### Inherited from diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 2ecb99b6..84751c88 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -133,6 +133,10 @@ The default selects the highest-scoring child that also passes the parent's chec Custom finalizers assemble their candidate before the check. Child validity and delivery counts remain unchanged. An external director can then use `repromptOnUnmet` to continue from a rejected candidate within its existing resource limits. +Set `repromptOnUnmet: 'until-complete'` and a finite positive budget deadline to continue without a turn-count cap. +Successful continuations do not consume `driverRetry.maxAttempts`; that limit counts failed invocations across the driver run. +Numeric `repromptOnUnmet` values still cap continuations, and zero disables them. +Cancellation, explicit stop, resource limits, and terminal failures remain authoritative. A thrown parent check reports a validation error through the existing driver failure record. | I want to… | Use (import) | Do NOT build | @@ -140,7 +144,7 @@ A thrown parent check reports a validation error through the existing driver fai | Run one product chat turn with streamed events, ordered persistence hooks, and stable execution/turn identity | `handleChatTurn(...)` + `deriveExecutionId(...)`: `/durable`; pass the derived id as both `executionId` and `turnId` on initial dispatch | importing the broad package entry from an edge worker, treating `executionId` alone as dispatch idempotency, or rebuilding framing and persistence ordering in the product | | Run a profile-authored supervisor toward a goal | `supervise(profile, task, { budget, backend? })`: `/kernel` — the profile supplies the standing prompt and capability grants; Runtime adds no default supervisor policy | hand-wiring `createSupervisor().run` + `blobs`/`perWorker`/`journal`/`executors`; reaching for lower-level calls before you need a specific counterparty | | Score a supervised sandbox worker by an executable check **against its live box** | `supervise(..., { backend: { backend: 'sandbox', sandboxClient, validator } })`: `/kernel` — the leaf forwards it to the composed `runAgentRounds`, which calls `validate(output, ctx)` with `ctx.box` still alive, and the verdict lands on the worker's settle | a post-settle hook (the box is destroyed by then), a second scoring loop beside `depthStrategy`, or pairing `validator` with `steering` (refused: a steerable session composes no loop to score) | -| Send an external-harness driver back when it FINISHED without delivering | `supervise(..., { deliverable, repromptOnUnmet: 1, onUnmetContract? })`: `/kernel` — a harness owns its own turn loop, so it ends when IT decides it is finished, which can be while the run has produced nothing (measured on discovery-lab, 2026-09-01, n = 1,422 settled runs: 376 of 376 winning runs ended on the driver's own completion, and the completion gate could only label the undelivered result). A re-prompt re-enters the SAME session, coordination server, and live children with the unmet items, under the same budget, deadline, abort, and attempt bounds a retry crosses; a run the coordination server already stopped is never re-prompted | a second `supervise()` call over the same task, a caller-side while-loop around the run, or reading a completed driver as a delivered run | +| Continue an external-harness director until its completion check passes | `supervise(..., { deliverable, repromptOnUnmet: 'until-complete', budget, onUnmetContract? })`: `/kernel` — requires a finite positive budget deadline; reuses the same session, coordination server, and live children. Successful turns do not consume failure retries. Numeric values cap continuations; explicit stop, cancellation, budget, deadline, and failure limits remain enforced. | a second `supervise()` call, a caller-side continuation loop, an arbitrary large count, or treating a completed turn as a completed task | | Run a static root, workers, and analysts as reviewable `AgentProfile` nodes with versioned edge directives | `runGraph(graph, options)`: `/kernel` | a second graph executor, prompt-only roles, or pretending a static graph can discover new nodes while running | | Drive a graph's ROOT with caller-owned orchestration (a deterministic conversation driver; a persona loop that makes its own LLM calls) | `runGraph(graph, { brain })`: `/kernel` — `brain: ToolLoopChat` is caller data for a router-brained root; node pinning, directive delivery, the edge ledger, and the journal twin stay the same shipped path, and the root profile keeps prompt control (`systemPrompt`/`instructions` still apply). Model selection, provider-identity validation, and usage reporting move to the caller with the brain | routing a production run through the `/testing` entry, a bespoke driver loop beside the graph, or pairing `brain` with `driverBackend` / an external-harness root (both refused: two answers to who makes the root's calls) | | **Supervise agents to solve a graded `AgenticSurface` task** (workers `runAgentic` the surface, settle on its own check, driver self-improves from the failing tests) | `superviseSurface(profile, task, { surface, worker })`: `/kernel` | a worker-seam + a "self-improving supervisor" wrapper around `supervise()`; passing a custom `makeWorkerAgent` that runs `runAgentic` | diff --git a/src/runtime/supervise/coordination-driver.ts b/src/runtime/supervise/coordination-driver.ts index 605fdecc..d5d8723b 100644 --- a/src/runtime/supervise/coordination-driver.ts +++ b/src/runtime/supervise/coordination-driver.ts @@ -58,6 +58,7 @@ import { import { chargedTokens, promptCacheTokenClasses, unmeteredSpend } from '../util' import type { DeliverableSpec } from './completion-gate' import type { PriorCoordination } from './coordination-log' +import { errMessage } from './error-message' import type { BusRecord } from './event-bus' import { bestDelivered, @@ -1059,19 +1060,27 @@ export function driverAgent(opts: DriverAgentOptions): Agent { // Calling the brain is the provider-attempt boundary. A rejection after that boundary // carries no trusted served identity, so preserve an empty attempt for root settlement. opts.onProviderModel?.(undefined) - await meterRuntimeOwnedProviderAttempt( - scope, - withBudgetResources(unmeteredSpend(0), scope.budget), - providerAttemptEvidence(undefined), - { - driver: opts.name, - inferenceFailed: true, - call, - callId: callContext.callId, - correlationId: callContext.correlationId, - ...detail, - }, - ) + try { + await meterRuntimeOwnedProviderAttempt( + scope, + withBudgetResources(unmeteredSpend(0), scope.budget), + providerAttemptEvidence(undefined), + { + driver: opts.name, + inferenceFailed: true, + call, + callId: callContext.callId, + correlationId: callContext.correlationId, + ...detail, + }, + ) + } catch (meteringError) { + // Preserve the provider diagnosis without turning an accounting refusal into a retry. + throw new RuntimeRunStateError( + `${errMessage(error)}; accounting failed: ${errMessage(meteringError)}`, + { cause: error }, + ) + } throw error } let evidenceError: ValidationError | undefined @@ -1504,7 +1513,3 @@ function safeJson(v: unknown): string { return String(v) } } - -function errMessage(e: unknown): string { - return e instanceof Error ? e.message : String(e) -} diff --git a/src/runtime/supervise/driver-retry.test.ts b/src/runtime/supervise/driver-retry.test.ts index f775edf5..ddea652a 100644 --- a/src/runtime/supervise/driver-retry.test.ts +++ b/src/runtime/supervise/driver-retry.test.ts @@ -553,7 +553,7 @@ describe('runDriverWithRetry — a completed drive whose contract is unmet', () expect(records[0]?.repromptRefusedBy).toBe('aborted') }) - it('bounds re-prompts by the absolute attempt ceiling as well as by their own cap', async () => { + it('does not charge successful continuations against the failure attempt allowance', async () => { const script = completingDrive(99) const records: DriverAttemptRecord[] = [] await runDriverWithRetry({ @@ -567,10 +567,90 @@ describe('runDriverWithRetry — a completed drive whose contract is unmet', () sleep: instantSleep, }) - expect(script.reentries).toHaveLength(2) - expect(records[1]?.repromptRefusedBy).toBe('max-attempts') + expect(script.reentries).toHaveLength(10) + expect(records[9]?.repromptRefusedBy).toBe('reprompts-exhausted') }) + it('continues until completion beyond the default retry ceiling', async () => { + const script = completingDrive(20) + const records: DriverAttemptRecord[] = [] + await runDriverWithRetry({ + drive: script.drive, + progress: () => mark({ contract: script.delivered() ? 'met' : 'unmet' }), + budget: () => budget({ deadlineMs: 1000 }), + now: () => 0, + signal: new AbortController().signal, + reprompt: { + maxReprompts: 'until-complete', + onUnmetContract: (context) => { + expect(context.maxReprompts).toBe('until-complete') + return { steer: 'Continue the same work.' } + }, + }, + onAttempt: (record) => void records.push(record), + }) + expect(script.reentries).toHaveLength(21) + expect(records.map((record) => record.attempt)).toEqual( + Array.from({ length: 21 }, (_, index) => index + 1), + ) + expect(records.at(-1)).toMatchObject({ contract: 'met', stop: 'completed' }) + }) + + it.each([0, -1, Number.POSITIVE_INFINITY, Number.NaN])( + 'refuses completion-driven continuation without a finite positive deadline (%s)', + async (deadlineMs) => { + const script = completingDrive(1) + await expect( + runDriverWithRetry({ + drive: script.drive, + progress: () => mark(), + budget: () => budget({ deadlineMs }), + signal: new AbortController().signal, + reprompt: { maxReprompts: 'until-complete' }, + }), + ).rejects.toThrow(/finite positive deadline/u) + expect(script.reentries).toHaveLength(0) + }, + ) + + it.each(['deadline', 'budget', 'unknown-cost', 'aborted', 'caller-stop'] as const)( + 'completion-driven continuation still honors %s', + async (stop) => { + const script = completingDrive(99) + const controller = new AbortController() + const records: DriverAttemptRecord[] = [] + await runDriverWithRetry({ + drive: async (attempt, reentry) => { + await script.drive(attempt, reentry) + if (attempt === 12 && stop === 'aborted') controller.abort('cancelled') + }, + progress: () => mark(), + budget: () => + budget({ + deadlineMs: 1000, + ...(script.reentries.length >= 12 && stop === 'budget' ? { tokensLeft: 0 } : {}), + ...(script.reentries.length >= 12 && stop === 'unknown-cost' + ? { usdCapped: true, usdKnown: false } + : {}), + }), + now: () => (script.reentries.length >= 12 && stop === 'deadline' ? 1000 : 0), + signal: controller.signal, + reprompt: { + maxReprompts: 'until-complete', + onUnmetContract: () => + script.reentries.length >= 12 && stop === 'caller-stop' + ? 'stop' + : { steer: 'Continue the same work.' }, + }, + onAttempt: (record) => void records.push(record), + }) + expect(script.reentries).toHaveLength(12) + expect(records.at(-1)?.repromptRefusedBy).toBe( + stop === 'budget' || stop === 'unknown-cost' ? 'budget-exhausted' : stop, + ) + }, + ) + it('lets the caller end the run instead of re-prompting', async () => { const script = completingDrive(99) const records: DriverAttemptRecord[] = [] @@ -805,25 +885,31 @@ describe('long-run retry streaks', () => { expect(calls).toBe(4) }) - it('does not weaken an explicit total attempt ceiling', async () => { - let calls = 0 - const records: DriverAttemptRecord[] = [] - await runDriverWithRetry({ - drive: async () => { - calls++ - if (calls % 2 === 1) throw new Error('temporary transport interruption') - }, - progress: () => mark(), - budget: () => budget(), - signal: new AbortController().signal, - policy: { maxAttempts: 4, maxConsecutiveFailures: 3 }, - reprompt: { maxReprompts: 10 }, - onAttempt: (record) => { - records.push(record) - }, - sleep: instantSleep, - }) - expect(calls).toBe(4) - expect(records.at(-1)?.repromptRefusedBy).toBe('max-attempts') - }) + it.each([10, 'until-complete'] as const)( + 'bounds cumulative failures across successful turns with %s continuation', + async (maxReprompts) => { + let calls = 0 + const records: DriverAttemptRecord[] = [] + await expect( + runDriverWithRetry({ + drive: async () => { + calls++ + if (calls % 2 === 1) throw new Error('temporary transport interruption') + }, + progress: () => mark(), + budget: () => budget({ deadlineMs: 1000 }), + now: () => 0, + signal: new AbortController().signal, + policy: { maxAttempts: 4, maxConsecutiveFailures: 3 }, + reprompt: { maxReprompts }, + onAttempt: (record) => { + records.push(record) + }, + sleep: instantSleep, + }), + ).rejects.toMatchObject({ stop: 'max-attempts' }) + expect(calls).toBe(7) + expect(records.filter((record) => record.error)).toHaveLength(4) + }, + ) }) diff --git a/src/runtime/supervise/driver-retry.ts b/src/runtime/supervise/driver-retry.ts index 45f59427..69ad4ce6 100644 --- a/src/runtime/supervise/driver-retry.ts +++ b/src/runtime/supervise/driver-retry.ts @@ -42,7 +42,8 @@ * ended on this loop's own `stop: 'completed'`, with the completion gate left to label the result * rather than to change it. A completed drive whose contract is unmet is now a first-class moment: * `reprompt.maxReprompts` re-enters the SAME live session with the unmet items, and every re-entry - * crosses the same budget, deadline, abort, and attempt bounds a retry crosses. + * crosses the same budget, deadline, and abort bounds. Successful continuations do not consume + * the failure retry allowance. */ import { @@ -72,10 +73,9 @@ export interface DriverRetryPolicy { /** Consecutive failures that changed NOTHING (no metered spend, no settlement, no submission) * before the run gives up. Default 3. A failure that made progress resets the count. */ readonly maxConsecutiveFailures?: number - /** Absolute ceiling on attempts, regardless of progress. Default 8. The barren counter alone - * cannot bound a driver that crashes every turn AFTER metering a little: each attempt looks like - * progress, so without this backstop such a run would retry until it had eaten the entire - * envelope. A caller who wants budget-only bounding sets this high deliberately. */ + /** Ceiling on failed invocations across this driver run, regardless of progress. Default 8, + * minimum 1. Successful continuations do not consume this allowance or reset it. + * This bounds repeated crashes that each make enough progress to reset the barren streak. */ readonly maxAttempts?: number /** Backoff before the first retry, doubling per consecutive failure. Default 2000ms. */ readonly initialBackoffMs?: number @@ -163,7 +163,7 @@ export interface DriverUnmetContractContext { readonly attempt: number /** How many re-prompts this run has already issued. */ readonly reprompts: number - readonly maxReprompts: number + readonly maxReprompts: DriverRepromptPolicy['maxReprompts'] /** The mark read AFTER the completed drive. */ readonly progress: DriverProgressMark readonly budget: DriverBudgetReadout @@ -182,9 +182,10 @@ export type OnUnmetContract = ( /** How a completed-but-undelivered drive is re-entered. Absent = the historical behavior, where * such a drive ends the run and only the completion gate's label records what happened. */ export interface DriverRepromptPolicy { - /** How many times one run may re-enter its live session with the unmet items. `0` = never. Each - * re-prompt also consumes an attempt, so `maxAttempts` bounds it too. */ - readonly maxReprompts: number + /** How many times one run may re-enter its live session with the unmet items. `0` = never. + * `'until-complete'` removes the count cap and requires a finite positive scope deadline. + * Budget, cancellation, explicit stop, and failure retry limits still apply. */ + readonly maxReprompts: number | 'until-complete' /** Compose the instruction, or return `'stop'`. Omit = {@link defaultUnmetContractSteer}. */ readonly onUnmetContract?: OnUnmetContract /** What the run owes, surfaced in the default instruction. */ @@ -377,10 +378,19 @@ export async function runDriverWithRetry(run: DriverRetryRun): Promise { const initialBackoff = Math.max(0, policy.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS) const maxBackoff = Math.max(initialBackoff, policy.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS) - const maxReprompts = Math.max(0, run.reprompt?.maxReprompts ?? 0) + const maxReprompts = run.reprompt?.maxReprompts ?? 0 + if (maxReprompts === 'until-complete') { + const deadline = run.budget().deadlineMs + if (!Number.isFinite(deadline) || deadline <= 0) { + throw new ValidationError( + 'runDriverWithRetry: until-complete requires a finite positive deadline', + ) + } + } const attempts: DriverAttemptRecord[] = [] let consecutiveBarren = 0 + let failures = 0 let reprompts = 0 let reentry: DriverReentry | undefined @@ -398,11 +408,12 @@ export async function runDriverWithRetry(run: DriverRetryRun): Promise { attempt: number, after: DriverProgressMark, ): Promise<{ steer: string } | { refusedBy: DriverRepromptRefusal }> => { - if (reprompts >= maxReprompts) return { refusedBy: 'reprompts-exhausted' } + if (maxReprompts !== 'until-complete' && reprompts >= maxReprompts) { + return { refusedBy: 'reprompts-exhausted' } + } if (run.signal.aborted) return { refusedBy: 'aborted' } const byBudget = budgetStop(run.budget(), now()) if (byBudget === 'deadline' || byBudget === 'budget-exhausted') return { refusedBy: byBudget } - if (attempt >= maxAttempts) return { refusedBy: 'max-attempts' } const context: DriverUnmetContractContext = { attempt, reprompts, @@ -459,6 +470,7 @@ export async function runDriverWithRetry(run: DriverRetryRun): Promise { try { await run.drive(attempt, reentry) } catch (error) { + failures += 1 const durationMs = now() - startedAt const classification = classifyDriverFailure(error, run.signal) const progressed = madeProgress(before, run.progress()) @@ -468,7 +480,7 @@ export async function runDriverWithRetry(run: DriverRetryRun): Promise { if (run.signal.aborted) return 'aborted' const byBudget = budgetStop(run.budget(), now()) if (byBudget) return byBudget - if (attempt >= maxAttempts) return 'max-attempts' + if (failures >= maxAttempts) return 'max-attempts' // Progress resets the barren counter: a driver that is doing real work between crashes // has earned another attempt, and the budget remains the bound on how many. if (progressed) return undefined @@ -521,7 +533,7 @@ export async function runDriverWithRetry(run: DriverRetryRun): Promise { const progressed = madeProgress(before, after) const contract = contractOf(after) const contractField = contract === 'none' ? {} : { contract } - if (contract === 'unmet' && maxReprompts > 0) { + if (contract === 'unmet' && (maxReprompts === 'until-complete' || maxReprompts > 0)) { const decision = await decideReprompt(attempt, after) if ('steer' in decision) { reprompts += 1 diff --git a/src/runtime/supervise/supervise.ts b/src/runtime/supervise/supervise.ts index 93051ab2..706610ab 100644 --- a/src/runtime/supervise/supervise.ts +++ b/src/runtime/supervise/supervise.ts @@ -40,7 +40,7 @@ import { unsupportedProfileDimensions, worktreeCliProfileMaterialization, } from '../../agent/profile-materialization' -import { ConfigError, ValidationError } from '../../errors' +import { ConfigError, RuntimeRunStateError, ValidationError } from '../../errors' import type { AnalystRegistry, AnalyzeOnSettleRoute, @@ -67,7 +67,13 @@ import { type DeliverableSpec, gateOnDeliverable } from './completion-gate' import { isLoopbackHost } from './coordination-mcp' import { DEFAULT_SUCCESSFUL_SHUTDOWN_MS, teardownExecutor } from './deadline' import { driverChild, driverExecutorFactory, isDriverSpec } from './driver-executor' -import type { DriverAttemptRecord, DriverRetryPolicy, OnUnmetContract } from './driver-retry' +import type { + DriverAttemptRecord, + DriverRepromptPolicy, + DriverRetryPolicy, + OnUnmetContract, +} from './driver-retry' +import { errMessage } from './error-message' import type { BusRecord } from './event-bus' import type { SupervisorFinalizer } from './finalizer' import { @@ -1364,12 +1370,12 @@ function driveHarnessFromBackend( ) } } catch (error) { - // The budget pool intentionally throws after durably recording unknown capped usage and - // closing that capacity. Only replace the original failure if the marker did not land. - const budget = scope.budget - if (budget.tokensKnown !== false || (budget.usdCapped && budget.usdKnown !== false)) { - failure = error - } + // The pool can reject before updating its readout. Preserve the provider diagnosis, + // but keep failed accounting terminal even when the original failure was transient. + failure = new RuntimeRunStateError( + `${errMessage(failure)}; accounting failed: ${errMessage(error)}`, + { cause: failure }, + ) } } try { @@ -1643,16 +1649,19 @@ export interface SuperviseOptions { * could only LABEL an undelivered result `valid:false`, never send the driver back for it. * * A re-prompt is the retry path, not a second loop: same scope, same coordination server, same - * live children, and the same budget, deadline, abort, and `driverRetry.maxAttempts` bounds. A + * live children, and the same budget, deadline, and abort bounds. Successful continuations do + * not consume `driverRetry.maxAttempts`, which counts failed invocations only. A * run the coordination server already stopped is never re-prompted — that stop was a decision. * * Requires `deliverable`, and applies to every external manager with a completion check. A * recursive manager receives the check selected for its exact assignment. Refused for a * router-brained manager, which runs its turn loop in process. Omit/`0` = never. + * Use `'until-complete'` with a finite positive budget deadline to remove the continuation cap. + * Completion, explicit stop, cancellation, resource limits, and failure limits still stop work. */ - readonly repromptOnUnmet?: number + readonly repromptOnUnmet?: DriverRepromptPolicy['maxReprompts'] /** Compose the re-entry instruction for an unmet contract, or return `'stop'` to end the run. - * Requires `repromptOnUnmet >= 1`. Omit = Runtime's own instruction, which names what the run + * Requires positive `repromptOnUnmet` or `'until-complete'`. Omit = Runtime's instruction, which names what the run * owes and reports how many workers passed the check. */ readonly onUnmetContract?: OnUnmetContract /** diff --git a/src/runtime/supervise/supervisor-agent.ts b/src/runtime/supervise/supervisor-agent.ts index 5b234bd7..2d5b707f 100644 --- a/src/runtime/supervise/supervisor-agent.ts +++ b/src/runtime/supervise/supervisor-agent.ts @@ -48,6 +48,7 @@ import { import { type DriverAttemptRecord, type DriverProgressMark, + type DriverRepromptPolicy, type DriverRetryPolicy, defaultUnmetContractSteer, type OnUnmetContract, @@ -477,11 +478,12 @@ export interface SupervisorAgentDeps { * end while the run has delivered nothing — 376 of 376 winning discovery-lab runs (2026-09-01) * ended on the driver's own completion, and the completion gate could only label that result, * never change it. A re-prompt reuses the retry path: same scope, same coordination server, same - * live children, same budget/deadline/abort/attempt bounds. Requires `deliverable`; refused for - * a router-brained supervisor, which runs its loop in process. Omit/`0` = never re-prompt. */ - readonly repromptOnUnmet?: number + * live children, same budget/deadline/abort bounds. Successful turns do not consume failure + * retries. Use `'until-complete'` with a finite positive scope deadline to omit the count cap. + * Requires `deliverable`; refused for a router-brained supervisor. Omit/`0` = never re-prompt. */ + readonly repromptOnUnmet?: DriverRepromptPolicy['maxReprompts'] /** Compose the re-entry instruction for an unmet contract, or return `'stop'` to end the run. - * Requires `repromptOnUnmet >= 1`. Omit = Runtime's own instruction. */ + * Requires positive `repromptOnUnmet` or `'until-complete'`. Omit = Runtime's own instruction. */ readonly onUnmetContract?: OnUnmetContract /** Trusted identity for this manager. Required with node-scoped tools or observation. */ readonly nodeContext?: SupervisorNodeContextSeed @@ -742,21 +744,23 @@ function buildSupervisorAgent( if ( deps.repromptOnUnmet !== undefined && + deps.repromptOnUnmet !== 'until-complete' && (!Number.isInteger(deps.repromptOnUnmet) || deps.repromptOnUnmet < 0) ) { throw new ValidationError( - 'supervisorAgent: repromptOnUnmet must be a non-negative integer (0 = never re-prompt)', + "supervisorAgent: repromptOnUnmet must be a non-negative integer or 'until-complete' (0 = never re-prompt)", ) } - if (deps.onUnmetContract !== undefined && (deps.repromptOnUnmet ?? 0) <= 0) { + const repromptEnabled = + deps.repromptOnUnmet === 'until-complete' || (deps.repromptOnUnmet ?? 0) > 0 + if (deps.onUnmetContract !== undefined && !repromptEnabled) { throw new ValidationError( - 'supervisorAgent: onUnmetContract needs repromptOnUnmet >= 1 — without a cap the hook is ' + - 'never consulted', + "supervisorAgent: onUnmetContract needs repromptOnUnmet >= 1 or 'until-complete'", ) } - if ((deps.repromptOnUnmet ?? 0) > 0 && deps.deliverable === undefined) { + if (repromptEnabled && deps.deliverable === undefined) { throw new ValidationError( 'supervisorAgent: repromptOnUnmet needs a `deliverable` completion check — with no check ' + 'there is no contract that can be unmet, and every run would re-prompt', @@ -1050,7 +1054,7 @@ function buildSupervisorAgent( budget: () => scope.budget, signal: scope.signal, ...(deps.driverRetry ? { policy: deps.driverRetry } : {}), - ...(maxReprompts > 0 + ...(repromptEnabled ? { reprompt: { maxReprompts, diff --git a/tests/kernel/supervise-full-profile-bridge.test.ts b/tests/kernel/supervise-full-profile-bridge.test.ts index 52affced..2a33799c 100644 --- a/tests/kernel/supervise-full-profile-bridge.test.ts +++ b/tests/kernel/supervise-full-profile-bridge.test.ts @@ -2473,6 +2473,29 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( expect(requests[1]?.session_id).toBe(requests[0]?.session_id) }) + it('preserves a provider failure when unknown cost closes a dollar-capped budget', async () => { + const providerError = new Error('provider admission failed') + let calls = 0 + + const result = await supervise(routerTestProfile('pi-leader', 'Lead the pursuit.'), 'Choose.', { + backend: { backend: 'bridge', bridgeUrl: 'http://127.0.0.1:1', bridgeBearer: 'test-token' }, + budget: { maxIterations: 4, maxTokens: 10_000, maxUsd: 1 }, + brain: async () => { + calls += 1 + throw providerError + }, + }) + + expect(result.kind).toBe('no-winner') + if (result.kind !== 'no-winner') return + expect(result.reason).toBe('driver-failed') + if (result.reason === 'driver-failed') { + expect(result.error.message).toContain('provider admission failed') + } + expect(calls).toBe(1) + expect(result.spentTotal.usdKnown).toBe(false) + }) + it('refuses a manager with unknown cost under a dollar-capped budget', async () => { server = createBridgeServer(async (req, res) => { const body = await readJson(req) diff --git a/tests/kernel/supervise-retained-owner-recovery.test.ts b/tests/kernel/supervise-retained-owner-recovery.test.ts index 6c8b1a3a..967b1ef2 100644 --- a/tests/kernel/supervise-retained-owner-recovery.test.ts +++ b/tests/kernel/supervise-retained-owner-recovery.test.ts @@ -7,6 +7,7 @@ import type { AgentTurnInput, } from '@tangle-network/agent-interface/environment-provider' import { afterEach, describe, expect, it } from 'vitest' +import type { DriverAttemptRecord } from '../../src/runtime/supervise/driver-retry' import { createFileRunContext } from '../../src/runtime/supervise/run-context' import { supervise } from '../../src/runtime/supervise/supervise' import type { SpawnEvent, SpawnJournal } from '../../src/runtime/supervise/types' @@ -24,171 +25,180 @@ afterEach(async () => { }) describe('retained external supervisor recovery', () => { - it('reconstructs a reprompt interrupted after environment admission without a third dispatch', async () => { - const directory = await mkdtemp(join(tmpdir(), 'retained-owner-reprompt-crash-')) - directories.push(directory) - const proxy = await coordinationProxy() - proxies.push(proxy) - const stateFile = join(directory, 'provider.json') - const runDirectory = join(directory, 'run') - const context = createFileRunContext(runDirectory) - let creates = 0 - let dispatches = 0 - let destroys = 0 - let failed = false - let port = 0 - let token = '' - const turns: AgentTurnInput[] = [] - const environmentIds: string[] = [] - const provider: AgentEnvironmentProvider = { - ...durableRetainedProvider(stateFile), - capabilities: async () => ({ - ...(await durableRetainedProvider(stateFile).capabilities()), - create: { runtimeAttachments: { mcp: true } }, - }), - create: async (input) => { - creates++ - token ||= input.env?.AGENT_RUNTIME_COORDINATION_TOKEN ?? '' - const environment = await durableRetainedProvider(stateFile).create(input) - return wrap(environment) - }, - get: async (id) => { - const environment = await durableRetainedProvider(stateFile).get!(id) - return environment ? wrap(environment) : null - }, - } - const wrap = (environment: AgentEnvironment): AgentEnvironment => ({ - ...environment, - session: (id, sessionOptions) => { - const session = environment.session!(id, sessionOptions) - return { - ...session, - result: async () => ({ - ...(await session.result()), - usage: - sessionOptions?.controlRef?.executionId === turns[1]?.executionId - ? { inputTokens: 5, outputTokens: 5 } - : { inputTokens: 0, outputTokens: 0 }, - }), - } - }, - dispatch: async (turn) => { - dispatches++ - turns.push(turn) - environmentIds.push(environment.id) - const result = await environment.dispatch(turn) - if (dispatches === 2) { - const response = await fetch(`http://127.0.0.1:${port}/manager`, { - method: 'POST', - headers: { Authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify({ - jsonrpc: '2.0', - id: 'resumed-final', - method: 'tools/call', - params: { name: 'submit_result', arguments: { result: { answer: 'resumed' } } }, + it.each([1, 'until-complete'] as const)( + 'recovers %s continuation after interrupted admission without a third dispatch', + async (repromptOnUnmet) => { + const directory = await mkdtemp(join(tmpdir(), 'retained-owner-reprompt-crash-')) + directories.push(directory) + const proxy = await coordinationProxy() + proxies.push(proxy) + const stateFile = join(directory, 'provider.json') + const runDirectory = join(directory, 'run') + const context = createFileRunContext(runDirectory) + let creates = 0 + let dispatches = 0 + let destroys = 0 + let failed = false + let port = 0 + let token = '' + const turns: AgentTurnInput[] = [] + const environmentIds: string[] = [] + const provider: AgentEnvironmentProvider = { + ...durableRetainedProvider(stateFile), + capabilities: async () => ({ + ...(await durableRetainedProvider(stateFile).capabilities()), + create: { runtimeAttachments: { mcp: true } }, + }), + create: async (input) => { + creates++ + token ||= input.env?.AGENT_RUNTIME_COORDINATION_TOKEN ?? '' + const environment = await durableRetainedProvider(stateFile).create(input) + return wrap(environment) + }, + get: async (id) => { + const environment = await durableRetainedProvider(stateFile).get!(id) + return environment ? wrap(environment) : null + }, + } + const wrap = (environment: AgentEnvironment): AgentEnvironment => ({ + ...environment, + session: (id, sessionOptions) => { + const session = environment.session!(id, sessionOptions) + return { + ...session, + result: async () => ({ + ...(await session.result()), + usage: + sessionOptions?.controlRef?.executionId === turns[1]?.executionId + ? { inputTokens: 5, outputTokens: 5 } + : { inputTokens: 0, outputTokens: 0 }, }), - }) - if (!response.ok) throw new Error(`submit_result returned ${response.status}`) - } - return result - }, - destroy: async () => { - destroys++ - await environment.destroy?.() - }, - }) - const profile = testAgentProfile('root', { - harness: 'codex', - tools: runtimeToolDeclarations('submit_result'), - }) - const options = { - runDir: runDirectory, - runId: 'reprompt-crash-root', - backend: { backend: 'provider' as const, provider }, - driverBackend: { backend: 'provider' as const, provider }, - budget: { maxIterations: 8, maxTokens: 100 }, - driverRetry: { enabled: false }, - repromptOnUnmet: 1, - deliverable: { - describe: 'resumed answer', - check: (v: unknown) => (v as { answer?: string }).answer === 'resumed', - }, - coordination: { - authentication: { - signingKeys: { activeKeyId: 'test', keys: { test: 'test-secret-'.repeat(4) } }, + } }, - publicUrl: (address: { port: number }) => { - port = address.port - proxy.forwardTo(port) - return `${proxy.url}/manager` + dispatch: async (turn) => { + dispatches++ + turns.push(turn) + environmentIds.push(environment.id) + const result = await environment.dispatch(turn) + if (dispatches === 2) { + const response = await fetch(`http://127.0.0.1:${port}/manager`, { + method: 'POST', + headers: { Authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 'resumed-final', + method: 'tools/call', + params: { name: 'submit_result', arguments: { result: { answer: 'resumed' } } }, + }), + }) + if (!response.ok) throw new Error(`submit_result returned ${response.status}`) + } + return result }, - }, - journal: { - beginTree: context.journal.beginTree.bind(context.journal), - loadTree: context.journal.loadTree.bind(context.journal), - appendEvent: async (root: string, event: SpawnEvent) => { - await context.journal.appendEvent(root, event) - if ( - !failed && - event.kind === 'execution-admitted' && - event.admission.phase === 'environment' - ) { - const prior = (await context.journal.loadTree(root)) ?? [] + destroy: async () => { + destroys++ + await environment.destroy?.() + }, + }) + const profile = testAgentProfile('root', { + harness: 'codex', + tools: runtimeToolDeclarations('submit_result'), + }) + const options = { + runDir: runDirectory, + runId: 'reprompt-crash-root', + backend: { backend: 'provider' as const, provider }, + driverBackend: { backend: 'provider' as const, provider }, + budget: { maxIterations: 8, maxTokens: 100, deadlineMs: 60_000 }, + driverRetry: { enabled: false }, + repromptOnUnmet, + deliverable: { + describe: 'resumed answer', + check: (v: unknown) => (v as { answer?: string }).answer === 'resumed', + }, + coordination: { + authentication: { + signingKeys: { activeKeyId: 'test', keys: { test: 'test-secret-'.repeat(4) } }, + }, + publicUrl: (address: { port: number }) => { + port = address.port + proxy.forwardTo(port) + return `${proxy.url}/manager` + }, + }, + journal: { + beginTree: context.journal.beginTree.bind(context.journal), + loadTree: context.journal.loadTree.bind(context.journal), + appendEvent: async (root: string, event: SpawnEvent) => { + await context.journal.appendEvent(root, event) if ( - prior.filter( - (e) => e.kind === 'execution-admitted' && e.admission.phase === 'environment', - ).length === 2 + !failed && + event.kind === 'execution-admitted' && + event.admission.phase === 'environment' ) { - failed = true - throw new Error('test crash after second environment admission') + const prior = (await context.journal.loadTree(root)) ?? [] + if ( + prior.filter( + (e) => e.kind === 'execution-admitted' && e.admission.phase === 'environment', + ).length === 2 + ) { + failed = true + throw new Error('test crash after second environment admission') + } } - } + }, }, - }, - blobs: context.blobs, - } - const interrupted = await supervise(profile, 'answer', options) - expect(interrupted).toMatchObject({ kind: 'no-winner', reason: 'driver-failed' }) - const resumeAbort = new AbortController() - const resumeTimer = setTimeout(() => resumeAbort.abort(new Error('resume timeout')), 5_000) - const result = await supervise(profile, 'answer', { - ...options, - retainedAtSettlement: 'release', - signal: resumeAbort.signal, - }) - clearTimeout(resumeTimer) - expect(result).toMatchObject({ kind: 'winner', out: { answer: 'resumed' } }) - expect(creates).toBe(1) - expect(dispatches).toBe(2) - expect(new Set(turns.map((turn) => turn.sessionId)).size).toBe(1) - expect(new Set(turns.map((turn) => turn.executionId)).size).toBe(2) - expect(new Set(turns.map((turn) => turn.turnId)).size).toBe(2) - expect(destroys).toBe(1) - const events = (await context.journal.loadTree('reprompt-crash-root')) ?? [] - const environments = events.filter( - (event) => event.kind === 'execution-admitted' && event.admission.phase === 'environment', - ) - expect(environments).toHaveLength(2) - const resumed = environments[1] - expect(resumed?.kind === 'execution-admitted' ? resumed.admission : undefined).toMatchObject({ - environmentId: environmentIds[1], - sessionId: turns[1]?.sessionId, - executionId: turns[1]?.executionId, - }) - expect( - events.reduce( - (total, event) => - event.kind === 'metered' - ? total + event.spend.tokens.input + event.spend.tokens.output - : total, - 0, - ), - ).toBe(10) - }) + blobs: context.blobs, + } + const interrupted = await supervise(profile, 'answer', options) + expect(interrupted).toMatchObject({ kind: 'no-winner', reason: 'driver-failed' }) + const resumeAbort = new AbortController() + const resumeTimer = setTimeout(() => resumeAbort.abort(new Error('resume timeout')), 5_000) + const result = await supervise(profile, 'answer', { + ...options, + retainedAtSettlement: 'release', + signal: resumeAbort.signal, + }) + clearTimeout(resumeTimer) + expect(result).toMatchObject({ kind: 'winner', out: { answer: 'resumed' } }) + expect(creates).toBe(1) + expect(dispatches).toBe(2) + expect(new Set(turns.map((turn) => turn.sessionId)).size).toBe(1) + expect(new Set(turns.map((turn) => turn.executionId)).size).toBe(2) + expect(new Set(turns.map((turn) => turn.turnId)).size).toBe(2) + expect(destroys).toBe(1) + const events = (await context.journal.loadTree('reprompt-crash-root')) ?? [] + const environments = events.filter( + (event) => event.kind === 'execution-admitted' && event.admission.phase === 'environment', + ) + expect(environments).toHaveLength(2) + const resumed = environments[1] + expect(resumed?.kind === 'execution-admitted' ? resumed.admission : undefined).toMatchObject({ + environmentId: environmentIds[1], + sessionId: turns[1]?.sessionId, + executionId: turns[1]?.executionId, + }) + expect( + events.reduce( + (total, event) => + event.kind === 'metered' + ? total + event.spend.tokens.input + event.spend.tokens.output + : total, + 0, + ), + ).toBe(10) + }, + ) - it.each(['release', 'keep', 'release-failed'] as const)( - 're-prompts one retained provider conversation with %s cleanup', - async (cleanup) => { + it.each([ + { cleanup: 'release', continuations: 2 }, + { cleanup: 'keep', continuations: 2 }, + { cleanup: 'release-failed', continuations: 2 }, + { cleanup: 'release', continuations: 'until-complete' }, + ] as const)( + 're-prompts one retained provider conversation with $cleanup cleanup and $continuations continuation', + async ({ cleanup, continuations }) => { + const expectedTurns = continuations === 'until-complete' ? 12 : 3 const directory = await mkdtemp(join(tmpdir(), 'retained-owner-reprompt-')) directories.push(directory) const proxy = await coordinationProxy() @@ -218,7 +228,7 @@ describe('retained external supervisor recovery', () => { }) const dispatched = await environment.dispatch(turn) reprompt++ - if (reprompt === 3) { + if (reprompt === expectedTurns) { const response = await fetch(`http://127.0.0.1:${coordinationPort}/manager`, { method: 'POST', headers: { @@ -280,9 +290,9 @@ describe('retained external supervisor recovery', () => { runId: 'reprompt-root', backend: { backend: 'provider', provider }, driverBackend: { backend: 'provider', provider }, - budget: { maxIterations: 8, maxTokens: 100 }, + budget: { maxIterations: 100, maxTokens: 1000, deadlineMs: 60_000 }, driverRetry: { enabled: false }, - repromptOnUnmet: 2, + repromptOnUnmet: continuations, retainedAtSettlement: cleanup === 'keep' ? 'keep' : 'release', deliverable: { describe: 'an answer from the final reprompt', @@ -304,17 +314,19 @@ describe('retained external supervisor recovery', () => { expect(creates).toBe(1) expect(result.kind).toBe('winner') expect(result).toMatchObject({ out: { answer: 'final reprompt' } }) - expect(turns).toHaveLength(3) + expect(turns).toHaveLength(expectedTurns) expect(new Set(turns.map((turn) => turn.environmentId)).size).toBe(1) expect(turns.every((turn) => turn.sessionId !== undefined)).toBe(true) expect(new Set(turns.map((turn) => turn.sessionId)).size).toBe(1) expect(turns.every((turn) => turn.executionId !== undefined)).toBe(true) - expect(new Set(turns.map((turn) => turn.executionId)).size).toBe(3) + expect(new Set(turns.map((turn) => turn.executionId)).size).toBe(expectedTurns) expect(turns.every((turn) => turn.turnId !== undefined)).toBe(true) - expect(new Set(turns.map((turn) => turn.turnId)).size).toBe(3) + expect(new Set(turns.map((turn) => turn.turnId)).size).toBe(expectedTurns) expect(destroys).toBe(cleanup === 'keep' ? 0 : 1) const events = (await context.journal.loadTree('reprompt-root')) ?? [] - expect(events.filter((event) => event.kind === 'execution-result')).toHaveLength(3) + expect(events.filter((event) => event.kind === 'execution-result')).toHaveLength( + expectedTurns, + ) expect(events.filter((event) => event.kind === 'environment-teardown')).toMatchObject( cleanup === 'keep' ? [] @@ -437,6 +449,69 @@ describe('retained external supervisor recovery', () => { expect(bindings).toEqual(['known', 'known']) expect(events.filter((event) => event.kind === 'materialized')).toHaveLength(1) }) + + it.each([false, true])( + 'preserves a provider create failure (frozen: %s) without retrying unknown cost', + async (frozen) => { + const directory = await mkdtemp(join(tmpdir(), 'retained-owner-create-failure-')) + directories.push(directory) + const proxy = await coordinationProxy() + proxies.push(proxy) + const runDirectory = join(directory, 'run') + const context = createFileRunContext(runDirectory) + const base = durableRetainedProvider(join(directory, 'provider.json')) + const providerError = new Error('egress enabled but no model API key resolved') + if (frozen) Object.freeze(providerError) + const attempts: DriverAttemptRecord[] = [] + let creates = 0 + const provider: AgentEnvironmentProvider = { + ...base, + capabilities: async () => ({ + ...(await base.capabilities()), + create: { runtimeAttachments: { mcp: true } }, + }), + create: async () => { + creates += 1 + throw providerError + }, + } + const result = await supervise( + testAgentProfile('root', { harness: 'codex' }), + 'Answer the task.', + { + runDir: runDirectory, + runId: 'create-failure-root', + journal: context.journal, + blobs: context.blobs, + backend: { backend: 'provider', provider }, + driverBackend: { backend: 'provider', provider }, + budget: { maxIterations: 4, maxTokens: 100, maxUsd: 1 }, + driverRetry: { maxAttempts: 3, initialBackoffMs: 0, maxBackoffMs: 0 }, + onDriverAttempt: (record) => void attempts.push(record), + coordination: { + authentication: { + signingKeys: { activeKeyId: 'test', keys: { test: 'test-secret-'.repeat(4) } }, + }, + publicUrl: (address) => { + proxy.forwardTo(address.port) + return `${proxy.url}/manager` + }, + }, + }, + ) + expect(result).toMatchObject({ kind: 'no-winner', reason: 'driver-failed' }) + if (result.kind === 'no-winner' && result.reason === 'driver-failed') { + expect(result.error.message).toContain('egress enabled but no model API key resolved') + } + expect(creates).toBe(1) + expect(attempts).toHaveLength(1) + expect(attempts[0]).toMatchObject({ classification: 'terminal', stop: 'terminal-error' }) + const events = (await context.journal.loadTree('create-failure-root')) ?? [] + expect(events.filter((event) => event.kind === 'execution-admitted')).toHaveLength(1) + expect(events.some((event) => event.kind === 'metered' && !event.spend.usdKnown)).toBe(true) + expect(result.kind === 'no-winner' ? result.spentTotal.usdKnown : true).toBe(false) + }, + ) }) function tokenTotal(events: SpawnEvent[]): number { diff --git a/tests/kernel/supervisor-agent.test.ts b/tests/kernel/supervisor-agent.test.ts index 2b512a77..2eb2da46 100644 --- a/tests/kernel/supervisor-agent.test.ts +++ b/tests/kernel/supervisor-agent.test.ts @@ -1707,18 +1707,21 @@ describe('supervisorAgent — coordination bind + prompt hoisting on the harness expect(hookCalls).toBe(0) }) - it('refuses repromptOnUnmet with no completion check to be unmet', () => { - const blobs = new InMemoryResultBlobStore() - expect(() => - supervisorAgent(testAgentProfile('sup', { harness: 'opencode' }), { - blobs, - makeWorkerAgent: () => deliveringLeaf('w', {}), - perWorker, - driveHarness: async () => {}, - repromptOnUnmet: 2, - }), - ).toThrow(/needs a `deliverable` completion check/u) - }) + it.each([2, 'until-complete'] as const)( + 'refuses %s with no completion check to be unmet', + (repromptOnUnmet) => { + const blobs = new InMemoryResultBlobStore() + expect(() => + supervisorAgent(testAgentProfile('sup', { harness: 'opencode' }), { + blobs, + makeWorkerAgent: () => deliveringLeaf('w', {}), + perWorker, + driveHarness: async () => {}, + repromptOnUnmet, + }), + ).toThrow(/needs a `deliverable` completion check/u) + }, + ) it('refuses onUnmetContract without a re-prompt cap, instead of never consulting it', () => { const blobs = new InMemoryResultBlobStore() From 78f2fdc9e4ba666cbcdb5f587d880766b5f3bad7 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 15 Sep 2026 23:08:39 -0700 Subject: [PATCH 2/2] docs(supervise): describe completion-driven continuation --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 909e42ce..56d48268 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## 0.232.0 +External directors can use `repromptOnUnmet: 'until-complete'` with a completion check and finite positive budget deadline. +Successful continuations reuse the retained conversation without consuming `driverRetry.maxAttempts`. +That limit now counts failed invocations; numeric continuation caps retain their separate meaning. +Completion, explicit stop, cancellation, deadlines, resources, and failure limits still stop the driver. +Provider failures remain visible when recording unknown cost also fails, and that accounting refusal remains terminal. + +Local regressions exercise 12 turns in one retained environment, interrupted continuation recovery, and refused further dispatch under unknown cost. +These checks do not establish live multi-hour fleet reliability. +Related system issue: SYS-011 — outcome: mitigates — proof: retained-owner recovery and driver-retry regression tests. + A child that DROPS now keeps its transcript, the receipt that says so has moved to where an unavailable trace cannot hide it, and the thing itself is finally called what it is.