diff --git a/CHANGELOG.md b/CHANGELOG.md index bb57ec29..c9a4fc64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,49 @@ # Changelog +## 0.235.0 + +**A retained child's settlement says WHY it is pending, as a value (#1204).** One reason string — +`retained provider execution requires reconciliation before replacement` — covered two situations +that call for opposite operator responses: an execution whose status genuinely cannot be +determined, where refusing to replace it is correct and the operator must reconcile before +retrying or pay twice for one turn; and a provider that broke its contract, where nothing needs +reconciling and the right response is to fix or report the provider. Six exhibits in three days +wore the first name for the second fault, and the runtime itself made it worse: it minted one +`RetainedRunProviderContractError` class both when the provider answered wrongly and when a read +of the provider merely failed. + +- `RetainedExecutionPendingError` now carries `pendingCause: RetainedPendingCause` — + `'unobservable'` (the safety refusal), `'provider-contract'`, `'request-rejected'`, + `'transport'`, `'nested-recovery'` — and its message names which one. The cause is classified + from the typed error's STRUCTURE (class name, `code`, HTTP `status`, a Zod issue list, the + members of an `AggregateError`), never from message text: the provider is not a dependency of + this package and its messages are not a contract. +- Classification is **phase-aware**. A 4xx, a not-found, or a client deadline hit AFTER admission + means the provider cannot resolve what it admitted — exhibit 4's situation — and stays + `'unobservable'`; `'request-rejected'` is named only at admission. A schema failure after + admission is the runtime refusing the provider's answer, `'provider-contract'`. +- A `RetainedRunProviderContractError` whose code says a READ failed + (`RETAINED_RESULT_READ_FAILED`, `RETAINED_CONTROL_REF_READ_FAILED`, and the new + `RETAINED_EVENT_STREAM_READ_FAILED`, split off `RETAINED_EVENT_STREAM_INVALID` so the + runtime's own stream checks keep their verdict) is a wrapper: what it wraps decides. The live + stream's event-binding check now mints `RETAINED_EVENT_BINDING_INVALID`, so an event bound to + another run classifies as the contract violation it is. Every `driverExecutor` throw that is a + nested-recovery failure says so instead of being inferred from whatever the driver threw. +- `retainedPendingCause` rides beside `retainedExecution` everywhere that field goes: `Settled`'s + down arm, the `reconciled` and terminal journal records (through the one shared field spread, + so the released record — written live or healed on resume — carries it forward), `NodeSnapshot` + live and materialized, replay, both `agent.child` payloads, and `PursuitNodeProjection`. A + reader never splits the population on the reason's wording. +- **Not covered, stated rather than faked:** #1204's exhibits 3 (`Tangle session event arrived + without a stable id`) and 6 (`value exceeds its JSON bound`) are thrown by agent-provider-tangle + as plain `Error`s with no code, so a structure-only classifier cannot name them; they land on + `'unobservable'`, the safe side, until the provider types them. Where they arrive as an + observation failure beside a failed result read, the read decides. +- The refusal behaviour is unchanged: every cause still settles the child `down`, `infra: true`, + retained-pending, and counts in `fleetYield.releasedUnrecovered` once released. The cause says + what to do next, not whether the fleet lost the child. Journals written before this release + carry no cause; a reader treats absence as unknown, not as unobservable. + ## 0.234.0 **A resume heals the 0.233.0 crash window.** The `reconciled` record now carries the settlement diff --git a/api-surface.json b/api-surface.json index c87ac1b2..a5cfe0ab 100644 --- a/api-surface.json +++ b/api-surface.json @@ -505,7 +505,7 @@ "PursuitNodeCost": "type 2ae084b090ba", "PursuitNodePlacement": "type 744e7b45dc01", "PursuitNodePlatform": "type 1d5dc2e2dd89", - "PursuitNodeProjection": "type 6d2b6c593df9", + "PursuitNodeProjection": "type a17725838ecf", "PursuitNodeTiming": "type 8cb37b8dca25", "PursuitNodeUsage": "type 951e957775ab", "PursuitProjection": "type 8d4e8a08d707", @@ -1044,7 +1044,7 @@ "NoWinnerError": "type adcf245eb125", "NodeExecutionIdentity": "type 76b27bc384d1", "NodeId": "type ba5552a40a12", - "NodeSnapshot": "type 7a1dd9177e44", + "NodeSnapshot": "type 881433f52643", "NodeStatus": "type 2073c1387091", "Observation": "type c276b53b9eb0", "ObservationAnalysis": "type b23fcc6368fb", @@ -1150,6 +1150,7 @@ "RetainedInteractiveRunHandle": "type dca0bd0f359f", "RetainedInteractiveStartMaterial": "type 82f6c8325ca2", "RetainedInteractiveStartedAdmission": "type 58531f291358", + "RetainedPendingCause": "type 9aefb24afb94", "RetainedRunAdmission": "type b1333206214a", "RetainedRunAdmissionHook": "type aa9c0342cbfc", "RetainedRunCancelOptions": "type c9bf674c803e", @@ -1211,13 +1212,13 @@ "SessionCapableBox": "type 9db08b40cab8", "SessionMessageLike": "type 726215634de7", "SessionTraceBox": "type b8d989093064", - "Settled": "type 24da40c1d786", + "Settled": "type d79f74b03f4d", "ShapeBudget": "type a1cb1a8d0995", "ShapeContext": "type a266c1b7e746", "ShapeRegistry": "type b567a5be55ec", "Shell": "type c676fe970f79", "ShotSpec": "type bf645fd74234", - "SpawnEvent": "type 1497e8b7c300", + "SpawnEvent": "type cf743d02f97e", "SpawnForest": "type 7308f34da226", "SpawnForestEvent": "type 230e369c6548", "SpawnForestInDoubtNode": "type 4e4bddd6a7d9", diff --git a/docs/api/durable.md b/docs/api/durable.md index 0f95b433..9f73b564 100644 --- a/docs/api/durable.md +++ b/docs/api/durable.md @@ -1041,6 +1041,12 @@ Recorded by Runtime on the `agent.child` payload: `'pending'` at a retained chil fold overwrites in observed order; that event's `settledAt` is the original settlement, so `settledAt` and `timing` do not move. +##### retainedPendingCause? + +> `readonly` `optional` **retainedPendingCause?**: [`RetainedPendingCause`](runtime.md#retainedpendingcause-1) + +Why a retained child has no accepted result; see `RetainedPendingCause`. + ##### releasedAt? > `readonly` `optional` **releasedAt?**: `number` diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index b3d6c0b2..e93cc0cb 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.234.0` and `@tangle-network/agent-eval@0.182.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.235.0` and `@tangle-network/agent-eval@0.182.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -423,7 +423,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 167 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 956 exports. +Import from `@tangle-network/agent-runtime/kernel` — 957 exports. | Symbol | Kind | Summary | |---|---|---| @@ -1160,6 +1160,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 956 exports. | `RetainedInteractiveAdmission` | type | Durable records for one exact native coding-agent process. | | `RetainedInteractiveAdmissionHook` | type | Persist each exact interactive record before the runtime proceeds. | | `RetainedInteractiveEnvironmentInput` | type | Environment and exact AgentProfile used to start one native coding-agent process. | +| `RetainedPendingCause` | type | Why a retained execution has no accepted terminal result, classified where the cause is still | | `RetainedRunAdmission` | type | One detached-run admission record the runtime persists before creation or dispatch proceeds. | | `RetainedRunAdmissionHook` | type | Awaited durability hook for retained admission records. | | `RetainedRunEffect` | type | Effect recorded for one retained control operation. | diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 7acd26e8..e13e9aa6 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -1171,6 +1171,16 @@ Present on a retained child: `'pending'` while its cursor slot is open, `'releas [`NodeSnapshot`](#nodesnapshot).[`retainedExecution`](#retainedexecution-1) +##### retainedPendingCause? + +> `readonly` `optional` **retainedPendingCause?**: [`RetainedPendingCause`](#retainedpendingcause-1) + +Why a retained child has no accepted result; see `RetainedPendingCause`. + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`retainedPendingCause`](#retainedpendingcause-2) + *** ### SpawnForestInDoubtNode @@ -22502,6 +22512,12 @@ Present on a retained child: `'pending'` while its cursor slot is open, `'releas (`materializeTreeView`) state the same fact, so a settle record's `tree` answers the retained-vs-down question without the observer journal. +##### retainedPendingCause? + +> `readonly` `optional` **retainedPendingCause?**: [`RetainedPendingCause`](#retainedpendingcause-1) + +Why a retained child has no accepted result; see `RetainedPendingCause`. + *** ### TreeView @@ -27952,6 +27968,47 @@ Why an attempt did not reach a sibling. Each value is a fact the sender can read *** +### RetainedPendingCause + +> **RetainedPendingCause** = `"unobservable"` \| `"provider-contract"` \| `"request-rejected"` \| `"transport"` \| `"nested-recovery"` + +Why a retained execution has no accepted terminal result, classified where the cause is still +a typed value rather than a string in a journal. + +One reason string used to cover two situations that call for opposite operator responses +(#1204): an execution whose status genuinely cannot be determined — refusing to replace it is +correct, and the operator must reconcile before retrying or pay twice for one turn — and a +provider that broke its contract, where nothing needs reconciling and the right response is to +fix or report the provider. Six exhibits in three days wore the first name for the second fault. + +- `'unobservable'`: the execution may have run and nothing local can say. The safety refusal. + Anything unclassifiable lands here, and so does a 4xx, a not-found, or a client deadline hit + AFTER admission: the provider cannot resolve what it admitted, which is exactly the case the + refusal exists for, not a rejected request. +- `'provider-contract'`: the provider answered with something its own contract forbids — a + `RetainedRunProviderContractError` naming a broken answer (an `*_INVALID`, `*_CHANGED`, + `*_DUPLICATE`, `*_MISSING` code), an event bound to another run. +- `'request-rejected'`: the request itself was refused BEFORE it ran — a schema violation + (`ZodError`), an HTTP 4xx at admission. Never named after admission. +- `'transport'`: the provider or a gateway in front of it failed — an HTTP 5xx, a socket + error, a platform service answering with a server error. Status is in doubt only because + the transport was. +- `'nested-recovery'`: a nested manager's own recovery could not be reconstructed. Stated by + the thrower, never inferred. + +Classification reads the cause's structure — class name, `code`, HTTP `status`, a Zod issue +list — never its message text, because the provider is not a dependency of this package and +its messages are not a contract. Two exhibits (#1204's 3 and 6) are thrown by the provider as +plain `Error`s with no code, so a structure-only classifier cannot name them until the provider +types them; they land on `'unobservable'`, which is the safe side. + +One `RetainedRunProviderContractError` is NOT one meaning. The runtime mints it both when the +provider answered wrongly and when a READ of the provider failed (`RETAINED_RESULT_READ_FAILED`, +`RETAINED_CONTROL_REF_READ_FAILED`). The second kind is a wrapper: what it wraps decides, in a +post-admission context, and a wrapper around nothing classifiable is exhibit 4. + +*** + ### RunContext > **RunContext** = [`InMemoryRunContext`](#inmemoryruncontext) @@ -28586,7 +28643,7 @@ recovery before a replacement can run. ### Settled -> **Settled**\<`Out`\> = \{ `kind`: `"done"`; `handle`: [`Handle`](#handle-3)\<`Out`\>; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `settledAt?`: `number`; `seq`: `number`; \} \| \{ `kind`: `"down"`; `handle`: [`Handle`](#handle-3)\<`Out`\>; `reason`: `string`; `outRef?`: `string`; `infra`: `boolean`; `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: [`RetainedExecutionState`](#retainedexecutionstate); `settledAt?`: `number`; `seq`: `number`; \} +> **Settled**\<`Out`\> = \{ `kind`: `"done"`; `handle`: [`Handle`](#handle-3)\<`Out`\>; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `settledAt?`: `number`; `seq`: `number`; \} \| \{ `kind`: `"down"`; `handle`: [`Handle`](#handle-3)\<`Out`\>; `reason`: `string`; `outRef?`: `string`; `infra`: `boolean`; `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: [`RetainedExecutionState`](#retainedexecutionstate); `retainedPendingCause?`: [`RetainedPendingCause`](#retainedpendingcause-1); `settledAt?`: `number`; `seq`: `number`; \} A settled child, delivered by `scope.next()`. `seq` is the monotonic cursor order `next()` yielded this settlement (B2) — NOT wall-clock — and replay delivers strictly @@ -28670,7 +28727,7 @@ Epoch ms parsed from the durable settlement record when available. ##### Type Literal -\{ `kind`: `"down"`; `handle`: [`Handle`](#handle-3)\<`Out`\>; `reason`: `string`; `outRef?`: `string`; `infra`: `boolean`; `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: [`RetainedExecutionState`](#retainedexecutionstate); `settledAt?`: `number`; `seq`: `number`; \} +\{ `kind`: `"down"`; `handle`: [`Handle`](#handle-3)\<`Out`\>; `reason`: `string`; `outRef?`: `string`; `infra`: `boolean`; `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: [`RetainedExecutionState`](#retainedexecutionstate); `retainedPendingCause?`: [`RetainedPendingCause`](#retainedpendingcause-1); `settledAt?`: `number`; `seq`: `number`; \} ###### kind @@ -28735,6 +28792,16 @@ Present only when this child's provider execution was RETAINED (see states, so a reader never splits this population on `reason` text — which is identical on every one of these children. +###### retainedPendingCause? + +> `optional` **retainedPendingCause?**: [`RetainedPendingCause`](#retainedpendingcause-1) + +WHY the retained execution has no accepted result, as a value: the safety refusal + (`'unobservable'`) against a provider contract violation, a rejected request, a lost + transport, or a nested recovery that could not be reconstructed. Present iff + `retainedExecution` is; the `reason` text names the same thing, but a reader must never + have to parse it (#1204). + ###### settledAt? > `optional` **settledAt?**: `number` @@ -28749,7 +28816,7 @@ Epoch ms parsed from the durable settlement/cancellation record when available. ### SpawnEvent -> **SpawnEvent** = \{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-6); `parent?`: [`NodeId`](#nodeid-6); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](#budget-18); `runtime`: [`Runtime`](#runtime-7); `ownedTreeRoot?`: [`NodeId`](#nodeid-6); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `profileRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-input"`; `id`: [`NodeId`](#nodeid-6); `taskRef`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-admitted"`; `id`: [`NodeId`](#nodeid-6); `admission`: [`RetainedRunAdmission`](#retainedrunadmission); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-result"`; `outcome?`: `Pick`\<`AgentTurnResult`, `"success"` \| `"error"`\>; `id`: [`NodeId`](#nodeid-6); `outRef`: `string`; `spent`: [`Spend`](#spend); `verdict?`: `DefaultVerdict`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-6); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-6); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-6); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\>; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-6); `reason`: `string`; `source?`: `string`; `infra?`: `boolean`; `spent?`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `outRef?`: `string`; `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\>; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"node-inputs-resolved"`; `id`: [`NodeId`](#nodeid-6); `node`: `string`; `instance`: `string`; `inputRef`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"edge-verdict"`; `id`: [`NodeId`](#nodeid-6); `edge`: `string`; `fired`: `boolean`; `sourceStatus`: `"done"` \| `"down"` \| `"invalid"`; `capped?`: `boolean`; `inputRef?`: `string`; `toInstance?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"join-state"`; `id`: [`NodeId`](#nodeid-6); `node`: `string`; `rule`: `"all"` \| `"any"` \| `"any_failed"` \| `"all_done"`; `satisfiedBy`: `ReadonlyArray`\<`string`\>; `consumedPending`: `ReadonlyArray`\<`string`\>; `instance`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-6); `parent?`: [`NodeId`](#nodeid-6); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-6); `by`: `"fired"` \| `"timeout"` \| `"cancelled"` \| `"expired"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-6); `spend`: [`Spend`](#spend); `accountingOnly?`: `true`; `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"progress"`; `id`: [`NodeId`](#nodeid-6); `spend`: [`Spend`](#spend); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"reconciled"`; `id`: [`NodeId`](#nodeid-6); `spent`: [`Spend`](#spend); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `settledSeq?`: `number`; `reason?`: `string`; `infra?`: `boolean`; `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `outRef?`: `string`; `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `cancellation?`: \{ `source`: `string`; \}; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"teardown-unconfirmed"`; `id`: [`NodeId`](#nodeid-6); `label`: `string`; `runtime`: [`Runtime`](#runtime-7); `status`: [`NodeStatus`](#nodestatus); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"environment-teardown"`; `id`: [`NodeId`](#nodeid-6); `provider`: `string`; `environmentId`: `string`; `destroyed`: `boolean`; `detail?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"edge"`; `id`: [`NodeId`](#nodeid-6); `edge`: \{ `kind`: `"delegates"` \| `"analyzes"` \| `"data"`; `from`: `string`; `to`: `string`; `directive?`: `string`; `port?`: `string`; \}; `traversal`: `number`; `outcome`: `"delivered"` \| `"stripped"` \| `"empty"` \| `"unpropagated"`; `continuity?`: `"fresh"` \| `"resume"` \| `"steer"`; `bytes`: `number`; `reason?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"trace-unpropagated"`; `id`: [`NodeId`](#nodeid-6); `expectedTraceId`: `string`; `backend`: `string`; `reason`: `"no-env-channel"` \| `"no-worker-process"` \| `"caller-omitted"`; `seq`: `number`; `at`: `string`; \} +> **SpawnEvent** = \{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-6); `parent?`: [`NodeId`](#nodeid-6); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](#budget-18); `runtime`: [`Runtime`](#runtime-7); `ownedTreeRoot?`: [`NodeId`](#nodeid-6); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `profileRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-input"`; `id`: [`NodeId`](#nodeid-6); `taskRef`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-admitted"`; `id`: [`NodeId`](#nodeid-6); `admission`: [`RetainedRunAdmission`](#retainedrunadmission); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-result"`; `outcome?`: `Pick`\<`AgentTurnResult`, `"success"` \| `"error"`\>; `id`: [`NodeId`](#nodeid-6); `outRef`: `string`; `spent`: [`Spend`](#spend); `verdict?`: `DefaultVerdict`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-6); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-6); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-6); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\>; `retainedPendingCause?`: [`RetainedPendingCause`](#retainedpendingcause-1); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-6); `reason`: `string`; `source?`: `string`; `infra?`: `boolean`; `spent?`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `outRef?`: `string`; `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\>; `retainedPendingCause?`: [`RetainedPendingCause`](#retainedpendingcause-1); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"node-inputs-resolved"`; `id`: [`NodeId`](#nodeid-6); `node`: `string`; `instance`: `string`; `inputRef`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"edge-verdict"`; `id`: [`NodeId`](#nodeid-6); `edge`: `string`; `fired`: `boolean`; `sourceStatus`: `"done"` \| `"down"` \| `"invalid"`; `capped?`: `boolean`; `inputRef?`: `string`; `toInstance?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"join-state"`; `id`: [`NodeId`](#nodeid-6); `node`: `string`; `rule`: `"all"` \| `"any"` \| `"any_failed"` \| `"all_done"`; `satisfiedBy`: `ReadonlyArray`\<`string`\>; `consumedPending`: `ReadonlyArray`\<`string`\>; `instance`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-6); `parent?`: [`NodeId`](#nodeid-6); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-6); `by`: `"fired"` \| `"timeout"` \| `"cancelled"` \| `"expired"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-6); `spend`: [`Spend`](#spend); `accountingOnly?`: `true`; `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"progress"`; `id`: [`NodeId`](#nodeid-6); `spend`: [`Spend`](#spend); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"reconciled"`; `id`: [`NodeId`](#nodeid-6); `spent`: [`Spend`](#spend); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `settledSeq?`: `number`; `reason?`: `string`; `retainedPendingCause?`: [`RetainedPendingCause`](#retainedpendingcause-1); `infra?`: `boolean`; `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `outRef?`: `string`; `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `cancellation?`: \{ `source`: `string`; \}; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"teardown-unconfirmed"`; `id`: [`NodeId`](#nodeid-6); `label`: `string`; `runtime`: [`Runtime`](#runtime-7); `status`: [`NodeStatus`](#nodestatus); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"environment-teardown"`; `id`: [`NodeId`](#nodeid-6); `provider`: `string`; `environmentId`: `string`; `destroyed`: `boolean`; `detail?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"edge"`; `id`: [`NodeId`](#nodeid-6); `edge`: \{ `kind`: `"delegates"` \| `"analyzes"` \| `"data"`; `from`: `string`; `to`: `string`; `directive?`: `string`; `port?`: `string`; \}; `traversal`: `number`; `outcome`: `"delivered"` \| `"stripped"` \| `"empty"` \| `"unpropagated"`; `continuity?`: `"fresh"` \| `"resume"` \| `"steer"`; `bytes`: `number`; `reason?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"trace-unpropagated"`; `id`: [`NodeId`](#nodeid-6); `expectedTraceId`: `string`; `backend`: `string`; `reason`: `"no-env-channel"` \| `"no-worker-process"` \| `"caller-omitted"`; `seq`: `number`; `at`: `string`; \} Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO timestamp for human inspection only (NOT a replay input). @@ -28985,7 +29052,7 @@ Trusted runtime transformation from the authorized profile to actual wire bytes. ##### Type Literal -\{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-6); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\>; `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-6); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\>; `retainedPendingCause?`: [`RetainedPendingCause`](#retainedpendingcause-1); `seq`: `number`; `at`: `string`; \} ###### kind @@ -29070,6 +29137,10 @@ Written by the release sweep in the settling process, on the same tree, after ev instant, and the release instant is on the receipt immediately before it. Typed so a `'pending'` can never be journaled: the journal states that as `reconciled`. +###### retainedPendingCause? + +> `optional` **retainedPendingCause?**: [`RetainedPendingCause`](#retainedpendingcause-1) + ###### seq > **seq**: `number` @@ -29082,7 +29153,7 @@ Written by the release sweep in the settling process, on the same tree, after ev ##### Type Literal -\{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-6); `reason`: `string`; `source?`: `string`; `infra?`: `boolean`; `spent?`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `outRef?`: `string`; `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\>; `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-6); `reason`: `string`; `source?`: `string`; `infra?`: `boolean`; `spent?`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `outRef?`: `string`; `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\>; `retainedPendingCause?`: [`RetainedPendingCause`](#retainedpendingcause-1); `seq`: `number`; `at`: `string`; \} ###### kind @@ -29138,6 +29209,10 @@ The child's harness transcript receipt, when the executor could still be read at As on `settled`: a retained child that was cancelled settles `cancelled`, and the one builder writes whichever kind the settlement had. +###### retainedPendingCause? + +> `optional` **retainedPendingCause?**: [`RetainedPendingCause`](#retainedpendingcause-1) + ###### seq > **seq**: `number` @@ -29451,7 +29526,7 @@ without charging the same spend twice. ##### Type Literal -\{ `kind`: `"reconciled"`; `id`: [`NodeId`](#nodeid-6); `spent`: [`Spend`](#spend); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `settledSeq?`: `number`; `reason?`: `string`; `infra?`: `boolean`; `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `outRef?`: `string`; `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `cancellation?`: \{ `source`: `string`; \}; `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"reconciled"`; `id`: [`NodeId`](#nodeid-6); `spent`: [`Spend`](#spend); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `settledSeq?`: `number`; `reason?`: `string`; `retainedPendingCause?`: [`RetainedPendingCause`](#retainedpendingcause-1); `infra?`: `boolean`; `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `outRef?`: `string`; `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `cancellation?`: \{ `source`: `string`; \}; `seq`: `number`; `at`: `string`; \} ###### kind @@ -29505,6 +29580,10 @@ The cursor seq `next()` stamped on the delivery this floor stands in for — wha The settlement the driver received, verbatim, as `settled`/`cancelled` carry it. Optional only so journals written before these fields existed remain replayable. +###### retainedPendingCause? + +> `optional` **retainedPendingCause?**: [`RetainedPendingCause`](#retainedpendingcause-1) + ###### infra? > `optional` **infra?**: `boolean` diff --git a/docs/canonical-api.md b/docs/canonical-api.md index e9999b8d..fbdf173f 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.234.0.** +> **Version 0.235.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.182.0 <0.183.0`. > `sandbox` must satisfy `>=0.36.4 <0.42.0`. diff --git a/package.json b/package.json index e5dc8f2e..d8855398 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.234.0", + "version": "0.235.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/durable/observer-projection.ts b/src/durable/observer-projection.ts index 9c562518..48612fb8 100644 --- a/src/durable/observer-projection.ts +++ b/src/durable/observer-projection.ts @@ -5,6 +5,7 @@ import type { ProfileMaterializationReceipt, ProviderModelExecutionEvidence, RetainedExecutionState, + RetainedPendingCause, Spend, SpendChannel, SpendGap, @@ -193,6 +194,8 @@ export interface PursuitNodeProjection { * fold overwrites in observed order; that event's `settledAt` is the original settlement, so * `settledAt` and `timing` do not move. */ readonly retainedExecution?: RetainedExecutionState + /** Why a retained child has no accepted result; see `RetainedPendingCause`. */ + readonly retainedPendingCause?: RetainedPendingCause /** When the release sweep closed a retained node's slot; absent unless `'released'`. */ readonly releasedAt?: number /** Each channel on which the settled spend exceeded the node's reservation. The status is the @@ -279,6 +282,7 @@ type MutableNode = { reason?: string infra?: boolean retainedExecution?: RetainedExecutionState + retainedPendingCause?: RetainedPendingCause releasedAt?: number budgetViolation?: BudgetViolation wait?: unknown @@ -536,6 +540,15 @@ function projectNodeActivity(nodes: Map, record: ObserverRe if (infra !== undefined) node.infra = infra const retained = stringField(payload, 'retainedExecution') if (retained === 'pending' || retained === 'released') node.retainedExecution = retained + const cause = stringField(payload, 'retainedPendingCause') + if ( + cause === 'unobservable' || + cause === 'provider-contract' || + cause === 'request-rejected' || + cause === 'transport' || + cause === 'nested-recovery' + ) + node.retainedPendingCause = cause const releasedAt = numberField(payload, 'releasedAt') if (releasedAt !== undefined) node.releasedAt = releasedAt const budgetViolation = budgetViolationField(payload) diff --git a/src/durable/spawn-journal.ts b/src/durable/spawn-journal.ts index 29fcb96a..6c664fff 100644 --- a/src/durable/spawn-journal.ts +++ b/src/durable/spawn-journal.ts @@ -40,6 +40,7 @@ import type { ProviderModelExecutionEvidence, ResultBlobStore, RetainedExecutionState, + RetainedPendingCause, Runtime, Settled, SpawnEvent, @@ -1145,6 +1146,9 @@ export async function replaySpawnTree( trace: ev.trace ?? { status: 'unavailable', reason: 'execution-did-not-start' }, ...(ev.harnessTranscript === undefined ? {} : { harnessTranscript: ev.harnessTranscript }), ...(ev.retainedExecution === undefined ? {} : { retainedExecution: ev.retainedExecution }), + ...(ev.retainedPendingCause === undefined + ? {} + : { retainedPendingCause: ev.retainedPendingCause }), ...settlementTime(ev.at), seq: ev.seq, }) @@ -1172,6 +1176,9 @@ export async function replaySpawnTree( // Replay yields the settlement the driver saw except for the one deliberate difference: // live carried `'pending'` while recoverable; the journal records how the slot closed. ...(ev.retainedExecution === undefined ? {} : { retainedExecution: ev.retainedExecution }), + ...(ev.retainedPendingCause === undefined + ? {} + : { retainedPendingCause: ev.retainedPendingCause }), ...settlementTime(ev.at), seq: ev.seq, }) @@ -1316,6 +1323,7 @@ export function materializeTreeView(events: SpawnEvent[]): TreeView { node.trace = traceEvidenceFor(ev) node.budgetViolation = budgetViolationOf(ev).budgetViolation if (ev.retainedExecution !== undefined) node.retainedExecution = ev.retainedExecution + if (ev.retainedPendingCause !== undefined) node.retainedPendingCause = ev.retainedPendingCause const settledAt = Date.parse(ev.at) if (Number.isFinite(settledAt)) node.settledAt = settledAt } else if (ev.kind === 'woken') { @@ -1334,6 +1342,7 @@ export function materializeTreeView(events: SpawnEvent[]): TreeView { node.outRef = ev.outRef node.budgetViolation = budgetViolationOf(ev).budgetViolation if (ev.retainedExecution !== undefined) node.retainedExecution = ev.retainedExecution + if (ev.retainedPendingCause !== undefined) node.retainedPendingCause = ev.retainedPendingCause const settledAt = Date.parse(ev.at) if (Number.isFinite(settledAt)) node.settledAt = settledAt } @@ -1419,6 +1428,7 @@ interface MutableSnapshot { trace?: NodeSnapshot['trace'] budgetViolation?: BudgetViolation retainedExecution?: RetainedExecutionState + retainedPendingCause?: RetainedPendingCause settledAt?: number spawnedAt?: number } @@ -1491,6 +1501,9 @@ function freezeSnapshot(node: MutableSnapshot): NodeSnapshot { trace: node.trace, ...(node.budgetViolation === undefined ? {} : { budgetViolation: node.budgetViolation }), ...(node.retainedExecution === undefined ? {} : { retainedExecution: node.retainedExecution }), + ...(node.retainedPendingCause === undefined + ? {} + : { retainedPendingCause: node.retainedPendingCause }), settledAt: node.settledAt, spawnedAt: node.spawnedAt, } diff --git a/src/runtime/environment-provider.ts b/src/runtime/environment-provider.ts index 17966f30..f0353a65 100644 --- a/src/runtime/environment-provider.ts +++ b/src/runtime/environment-provider.ts @@ -74,6 +74,7 @@ import { assertEventBinding, awaitAbortable, exactSession, + RetainedRunProviderContractError, sameControlCoordinates, } from './retained-run-binding' import { @@ -1423,8 +1424,16 @@ async function providerExecutionSource( break } if (next.done) break - // A received event for another execution must never be accepted as evidence. - assertEventBinding(next.value, handle.controlRef) + // A received event for another execution must never be accepted as evidence — and + // it is the PROVIDER's contract that was broken, so it must classify as such (#1204). + try { + assertEventBinding(next.value, handle.controlRef) + } catch (error) { + throw new RetainedRunProviderContractError( + error instanceof Error ? error.message : 'provider event bound to another run', + { code: 'RETAINED_EVENT_BINDING_INVALID', cause: error }, + ) + } if (next.value.id !== undefined) lastReplayPosition = next.value.id if (isTerminalEnvironmentEvent(next.value)) sawTerminal = true yield next.value @@ -1501,7 +1510,8 @@ async function providerExecutionSource( } return { environment, events: events(), retained: true } } catch (error) { - if (admitted) throw new RetainedExecutionPendingError(error) + // Before `events()` is ever iterated, so a refused request still means refused: admission. + if (admitted) throw new RetainedExecutionPendingError(error, 'admission') throw error } } diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 46a6127e..a555843d 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -1111,6 +1111,7 @@ export type { ResumedKeyState, ResumedWork, RetainedExecutionState, + RetainedPendingCause, RootHandle, RootMaterialization, RootProviderModelEvidence, diff --git a/src/runtime/retained-run-events.ts b/src/runtime/retained-run-events.ts index 6d7f70ba..5ed7ff28 100644 --- a/src/runtime/retained-run-events.ts +++ b/src/runtime/retained-run-events.ts @@ -20,6 +20,24 @@ import { import type { RetainedRunEventOptions } from './retained-run-types' import { extractTransportEventIdentity, parseCanonicalTransportEvent } from './sandbox-events' +/** The runtime's own contract checks inside the stream loop, so the catch can tell a broken + * answer (a verdict) from a read the provider's iterator failed (a wrapper). */ +const runtimeChecks = new WeakSet() +function contractCheckFailed(message: string): never { + const error = new Error(message) + runtimeChecks.add(error) + throw error +} +/** Run one of the runtime's own checks; whatever it throws is a verdict, not a failed read. */ +function verdict(check: () => T): T { + try { + return check() + } catch (error) { + if (error instanceof Error) runtimeChecks.add(error) + throw error + } +} + export async function* retainedRunEvents( session: AgentSession, controlRef: AgentExactRunControlRef, @@ -52,14 +70,14 @@ export async function* retainedRunEvents( const source = next.value const identity = extractTransportEventIdentity(source) const sourceCursor = identity.cursor ?? identity.eventId - assertEventBinding(source, controlRef) + verdict(() => assertEventBinding(source, controlRef)) if (sourceCursor === after?.cursor) continue const event = canonicalEvent(source) if (!event) continue - assertCanonicalEventBinding(controlRef, event) + verdict(() => assertCanonicalEventBinding(controlRef, event)) if (firstAfterEvent && after !== undefined) { if (identity.sequence !== undefined && identity.sequence <= after.sequence) { - throw new Error( + contractCheckFailed( 'provider replay did not prove that the first event follows the requested cursor', ) } @@ -67,28 +85,30 @@ export async function* retainedRunEvents( } const eventId = identity.eventId ?? identity.cursor if (!eventId) { - throw new Error('replayable canonical event has no stable provider event id or cursor') + contractCheckFailed('replayable canonical event has no stable provider event id or cursor') } assertStableText(eventId, 'provider event id') - if (seen.has(eventId)) throw new Error(`provider replay repeated event id "${eventId}"`) + if (seen.has(eventId)) contractCheckFailed(`provider replay repeated event id "${eventId}"`) seen.add(eventId) const sourceSequence = identity.sequence const sequence = sourceSequence ?? nextSequence if (sequence <= lastSequence) { - throw new Error( + contractCheckFailed( `provider event sequence is not monotonic: ${sequence} follows ${lastSequence}`, ) } const occurredAt = identity.occurredAt - const envelope = RuntimeEventEnvelopeSchema.parse({ - runId: controlRef.runId, - eventId, - sequence, - cursor: identity.cursor ?? eventId, - ...(occurredAt === undefined ? {} : { occurredAt }), - receivedAt: new Date(now()).toISOString(), - event, - }) + const envelope = verdict(() => + RuntimeEventEnvelopeSchema.parse({ + runId: controlRef.runId, + eventId, + sequence, + cursor: identity.cursor ?? eventId, + ...(occurredAt === undefined ? {} : { occurredAt }), + receivedAt: new Date(now()).toISOString(), + event, + }), + ) yield envelope lastSequence = sequence nextSequence = sequence + 1 @@ -103,9 +123,16 @@ export async function* retainedRunEvents( throw abortError(options.signal.reason) } if (error instanceof RetainedRunProviderContractError) throw error + // Two facts share this catch: the runtime's own checks above threw (the provider's answer + // broke its contract), or the provider's iterator threw (the READ failed). Name each so a + // settlement can tell them apart (#1204); the read is a wrapper, the check is a verdict. + const fromProvider = error instanceof Error && !runtimeChecks.has(error) throw new RetainedRunProviderContractError( error instanceof Error ? error.message : 'provider retained event stream failed', - { code: 'RETAINED_EVENT_STREAM_INVALID', cause: error }, + { + code: fromProvider ? 'RETAINED_EVENT_STREAM_READ_FAILED' : 'RETAINED_EVENT_STREAM_INVALID', + cause: error, + }, ) } finally { if (!options?.signal?.aborted) await iterator.return?.() diff --git a/src/runtime/supervise/driver-executor.ts b/src/runtime/supervise/driver-executor.ts index 930876fa..21919928 100644 --- a/src/runtime/supervise/driver-executor.ts +++ b/src/runtime/supervise/driver-executor.ts @@ -266,6 +266,7 @@ export const driverExecutorFactory: ExecutorFactory = (rawSpec, ctx) => } else if (prior.length !== 1 && !recovering) { throw new RetainedExecutionPendingError( new ValidationError('driverExecutor: interrupted nested execution requires recovery'), + 'nested-recovery', ) } if (recovering && (!spec.recoverExecutor || !seam.restore)) { @@ -273,6 +274,7 @@ export const driverExecutorFactory: ExecutorFactory = (rawSpec, ctx) => new ValidationError( 'driverExecutor: nested recovery has no original executor reconstruction', ), + 'nested-recovery', ) } signal.throwIfAborted() @@ -289,7 +291,8 @@ export const driverExecutorFactory: ExecutorFactory = (rawSpec, ctx) => } catch (error) { controller.abort(error) signal.removeEventListener('abort', onParentAbort) - throw new RetainedExecutionPendingError(error) + // The nested tree's spawn record already exists; a replacement must recover it. + throw new RetainedExecutionPendingError(error, 'nested-recovery') } active = { controller, scope: nestedScope } nestedScopeHeld = nestedScope @@ -375,7 +378,8 @@ export const driverExecutorFactory: ExecutorFactory = (rawSpec, ctx) => ) { // An admitted backend or accepted backend result is not a finalized manager output. // Keep the parent's key in doubt until a supported nested recovery can run its finalizer. - throw new RetainedExecutionPendingError(err) + // The cause is journal state, not whatever the driver threw, so it is stated. + throw new RetainedExecutionPendingError(err, 'nested-recovery') } if (cleanupError !== undefined) throw cleanupError throw err diff --git a/src/runtime/supervise/recover-executors.ts b/src/runtime/supervise/recover-executors.ts index b27406ed..f48aa41d 100644 --- a/src/runtime/supervise/recover-executors.ts +++ b/src/runtime/supervise/recover-executors.ts @@ -144,6 +144,7 @@ export async function healReleasedSlots( ...(floor.outRef ? { outRef: floor.outRef } : {}), ...(floor.providerModel ? { providerModel: floor.providerModel } : {}), ...(floor.harnessTranscript ? { harnessTranscript: floor.harnessTranscript } : {}), + ...(floor.retainedPendingCause ? { retainedPendingCause: floor.retainedPendingCause } : {}), } await opts.journal.appendEvent( tree.root, diff --git a/src/runtime/supervise/retained-executor.ts b/src/runtime/supervise/retained-executor.ts index 5c33cdd4..bd9a7ae5 100644 --- a/src/runtime/supervise/retained-executor.ts +++ b/src/runtime/supervise/retained-executor.ts @@ -33,11 +33,168 @@ export function retainedExecutorContext(ctx: ExecutorContext): RetainedExecutorC return ctx.seams[retainedExecutorSeamKey] as RetainedExecutorContext | undefined } -/** Local observation stopped; the retained provider execution has no accepted terminal result. */ +/** + * Why a retained execution has no accepted terminal result, classified where the cause is still + * a typed value rather than a string in a journal. + * + * One reason string used to cover two situations that call for opposite operator responses + * (#1204): an execution whose status genuinely cannot be determined — refusing to replace it is + * correct, and the operator must reconcile before retrying or pay twice for one turn — and a + * provider that broke its contract, where nothing needs reconciling and the right response is to + * fix or report the provider. Six exhibits in three days wore the first name for the second fault. + * + * - `'unobservable'`: the execution may have run and nothing local can say. The safety refusal. + * Anything unclassifiable lands here, and so does a 4xx, a not-found, or a client deadline hit + * AFTER admission: the provider cannot resolve what it admitted, which is exactly the case the + * refusal exists for, not a rejected request. + * - `'provider-contract'`: the provider answered with something its own contract forbids — a + * `RetainedRunProviderContractError` naming a broken answer (an `*_INVALID`, `*_CHANGED`, + * `*_DUPLICATE`, `*_MISSING` code), an event bound to another run. + * - `'request-rejected'`: the request itself was refused BEFORE it ran — a schema violation + * (`ZodError`), an HTTP 4xx at admission. Never named after admission. + * - `'transport'`: the provider or a gateway in front of it failed — an HTTP 5xx, a socket + * error, a platform service answering with a server error. Status is in doubt only because + * the transport was. + * - `'nested-recovery'`: a nested manager's own recovery could not be reconstructed. Stated by + * the thrower, never inferred. + * + * Classification reads the cause's structure — class name, `code`, HTTP `status`, a Zod issue + * list — never its message text, because the provider is not a dependency of this package and + * its messages are not a contract. Two exhibits (#1204's 3 and 6) are thrown by the provider as + * plain `Error`s with no code, so a structure-only classifier cannot name them until the provider + * types them; they land on `'unobservable'`, which is the safe side. + * + * One `RetainedRunProviderContractError` is NOT one meaning. The runtime mints it both when the + * provider answered wrongly and when a READ of the provider failed (`RETAINED_RESULT_READ_FAILED`, + * `RETAINED_CONTROL_REF_READ_FAILED`). The second kind is a wrapper: what it wraps decides, in a + * post-admission context, and a wrapper around nothing classifiable is exhibit 4. + */ +export type RetainedPendingCause = + | 'unobservable' + | 'provider-contract' + | 'request-rejected' + | 'transport' + | 'nested-recovery' + +const causeMessages: Record = { + unobservable: 'retained provider execution requires reconciliation before replacement', + 'provider-contract': + 'retained provider execution ended on a provider contract violation; nothing to reconcile', + 'request-rejected': 'retained provider execution was refused before it ran; nothing to reconcile', + transport: 'retained provider execution lost its transport; status in doubt', + 'nested-recovery': 'retained nested execution requires recovery before replacement', +} + +/** The contract-error codes that wrap a failed READ rather than name a broken answer. */ +const readFailureCodes = new Set([ + 'RETAINED_RESULT_READ_FAILED', + 'RETAINED_CONTROL_REF_READ_FAILED', +]) + +/** Socket-level failures, by the codes Node and undici assign, plus the Sandbox SDK's own. */ +const transportCodes = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ETIMEDOUT', + 'ENOTFOUND', + 'EAI_AGAIN', + 'EPIPE', + 'UND_ERR_SOCKET', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_BODY_TIMEOUT', + 'NETWORK_ERROR', +]) + +function readNumber(value: object, key: string): number | undefined { + const read: unknown = Reflect.get(value, key) + return typeof read === 'number' && Number.isFinite(read) ? read : undefined +} + +function readString(value: object, key: string): string | undefined { + const read: unknown = Reflect.get(value, key) + return typeof read === 'string' ? read : undefined +} + +/** + * When the cause arose relative to admission. A read wrapper, an event-stream failure and the + * scope's own retained branch are all post-admission; only the bare admission path is not. + */ +export type RetainedPendingPhase = 'admission' | 'execution' + +/** + * Walk the cause chain (bounded, cycle-safe) and name the first cause whose structure decides. + * + * `phase` says whether a 4xx or a client deadline can mean "refused before it ran": only at + * admission. After admission the same status means the provider cannot resolve what it + * admitted, and the honest name is the refusal. + */ +export function classifyRetainedPendingCause( + cause: unknown, + phase: RetainedPendingPhase = 'execution', +): RetainedPendingCause { + const seen = new Set() + const queue: Array<{ value: unknown; phase: RetainedPendingPhase }> = [{ value: cause, phase }] + let steps = 0 + while (queue.length > 0 && steps < 12) { + steps += 1 + const { value, phase: at } = queue.shift()! + if (typeof value !== 'object' || value === null || seen.has(value)) continue + seen.add(value) + if (value instanceof RetainedExecutionPendingError) return value.pendingCause + const name = readString(value, 'name') + const code = readString(value, 'code') + const status = readNumber(value, 'status') ?? readNumber(value, 'statusCode') + if (name === 'RetainedRunProviderContractError') { + if (code === undefined || !readFailureCodes.has(code)) return 'provider-contract' + // A read that failed is a wrapper; what the read hit decides, after admission. + queue.unshift({ value: Reflect.get(value, 'cause'), phase: 'execution' }) + continue + } + if (name === 'ZodError' || Array.isArray(Reflect.get(value, 'issues'))) { + // A schema violation at admission is a rejected request. After admission it is the + // runtime refusing the PROVIDER's answer — a contract violation on the provider's side. + return at === 'admission' ? 'request-rejected' : 'provider-contract' + } + if (code !== undefined && transportCodes.has(code)) return 'transport' + if (name === 'NetworkError' || name === 'ServerError') return 'transport' + if (status !== undefined) { + if (status >= 500) return 'transport' + if (at === 'admission' && status !== 408) return 'request-rejected' + // 4xx after admission: the provider cannot resolve what it admitted. 408 is a client + // deadline at either phase and says nothing about whether the request was admitted. + return 'unobservable' + } + // An AggregateError carries its members in `errors`, with `cause` possibly the last one; + // the first classifiable member decides. + const errors: unknown = Reflect.get(value, 'errors') + if (Array.isArray(errors)) for (const member of errors) queue.push({ value: member, phase: at }) + queue.push({ value: Reflect.get(value, 'cause'), phase: at }) + } + return 'unobservable' +} + +/** + * Local observation stopped; the retained provider execution has no accepted terminal result. + * + * The message names WHICH of the situations in {@link RetainedPendingCause} this is, and the + * `pendingCause` field carries it as a value the settlement records, so a reader never splits + * the population on the message's wording. The cause chain is kept for the `caused by` receipt. + */ export class RetainedExecutionPendingError extends Error { - constructor(cause: unknown) { - super('retained provider execution requires reconciliation before replacement', { cause }) + readonly pendingCause: RetainedPendingCause + + constructor( + cause: unknown, + pendingCause: RetainedPendingCause | RetainedPendingPhase = 'execution', + ) { + const resolved = + pendingCause === 'admission' || pendingCause === 'execution' + ? classifyRetainedPendingCause(cause, pendingCause) + : pendingCause + super(causeMessages[resolved], { cause }) this.name = 'RetainedExecutionPendingError' + this.pendingCause = resolved } } diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index 0d8d8614..707f0179 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -93,6 +93,7 @@ import { type RetainedChildRecovery, RetainedExecutionPendingError, type RetainedExecutorContext, + type RetainedPendingCause, retainedExecutorSeamKey, } from './retained-executor' import { @@ -389,6 +390,7 @@ interface LiveChild { settledSeq?: number /** Mirrors the journal's statement about a retained execution so `makeTreeView` reports it. */ retainedExecution?: RetainedExecutionState + retainedPendingCause?: RetainedPendingCause /** Resolves with the terminal settlement WITHOUT a `seq` — `next()` stamps the seq. */ readonly settled: Promise /** Synchronous mirror of `settled`'s value once it has resolved (else `undefined`). */ @@ -480,6 +482,8 @@ export type PreSeqSettled = * the `settled` record it cannot write — otherwise every journal reader charges the ceiling * the pool just refunded. */ reconciled?: Spend + /** Why the retained execution is pending, as a value (#1204). Present iff `reconciled` is. */ + retainedPendingCause?: RetainedPendingCause } /** @@ -2704,6 +2708,7 @@ async function finalizeSettlement( // `recordCancellation` only assigns before `executorDone`. else { child.retainedExecution = 'pending' + child.retainedPendingCause = settlement.retainedPendingCause if (settlement.reconciled !== undefined) await appendReconciledFloor( args.journal, @@ -2726,7 +2731,14 @@ async function finalizeSettlement( // The in-memory down and the first `agent.child` are the only surfaces that exist while the // slot is open, so they are where `'pending'` lives; the journal states the same fact as the // `reconciled` record and replay never yields it. - const retainedExecution = retainedPending ? { retainedExecution: 'pending' as const } : {} + const retainedExecution = retainedPending + ? { + retainedExecution: 'pending' as const, + ...(settlement.retainedPendingCause === undefined + ? {} + : { retainedPendingCause: settlement.retainedPendingCause }), + } + : {} notifyRuntimeHookEvent( args.hooks, { @@ -3249,6 +3261,13 @@ async function runChild( await persistTranscriptOnce(), ), reconciled: live.spent, + // Classified HERE, from the typed cause, before `errMessage` flattens it to text: a + // provider that broke its contract and an execution nobody can observe both arrive on + // this branch, and an operator must act differently on each (#1204). `recoveryPending` + // without a RetainedExecutionPendingError is an admission that never got a result — by + // definition unobservable. + retainedPendingCause: + err instanceof RetainedExecutionPendingError ? err.pendingCause : 'unobservable', } } // A thrown executor has also finished its own work — only the down-record persistence @@ -3391,6 +3410,7 @@ function makeTreeView(root: NodeId, children: Map): TreeView ...(c.providerModel ? { providerModel: c.providerModel } : {}), ...(c.budgetViolation ? { budgetViolation: c.budgetViolation } : {}), ...(c.retainedExecution ? { retainedExecution: c.retainedExecution } : {}), + ...(c.retainedPendingCause ? { retainedPendingCause: c.retainedPendingCause } : {}), })) return { root, diff --git a/src/runtime/supervise/terminal-record.ts b/src/runtime/supervise/terminal-record.ts index 3f96cb35..2f2b2579 100644 --- a/src/runtime/supervise/terminal-record.ts +++ b/src/runtime/supervise/terminal-record.ts @@ -52,11 +52,15 @@ export function settlementFields( budgetViolation?: BudgetViolation trace: DownSettlement['trace'] harnessTranscript?: DownSettlement['harnessTranscript'] + retainedPendingCause?: DownSettlement['retainedPendingCause'] } { return { spent: subject.spent, infra: settlement.infra, reason: settlement.reason, + ...(settlement.retainedPendingCause === undefined + ? {} + : { retainedPendingCause: settlement.retainedPendingCause }), ...(settlement.outRef ? { outRef: settlement.outRef } : {}), ...(settlement.providerModel ? { providerModel: settlement.providerModel } : {}), ...(subject.budgetViolation ? { budgetViolation: subject.budgetViolation } : {}), @@ -149,6 +153,9 @@ export function releasedChildPayload( childId: subject.id, status: 'down', retainedExecution: 'released', + ...(settlement.retainedPendingCause === undefined + ? {} + : { retainedPendingCause: settlement.retainedPendingCause }), releasedAt, ...(settlement.outRef === undefined ? {} : { outRef: settlement.outRef }), reason: settlement.reason, diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index 2e7b2fa3..689216ef 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -40,6 +40,7 @@ import type { RetainedInteractiveRunHandle } from '../retained-interactive-types import type { RetainedRunEffect } from '../retained-run-types' import type { LoopTokenUsage } from '../types' import type { ExecutorProgress, WorkerProgress } from './progress' +import type { RetainedPendingCause } from './retained-executor' import type { TraceSource } from './trace-source' import type { PendingWait, WaitOutcome, WaitProbeRegistry, WaitRejection, WaitSpec } from './wait' import type { WorkerTraceResolver } from './worker-trace' @@ -1003,6 +1004,12 @@ export type Settled = * states, so a reader never splits this population on `reason` text — which is identical * on every one of these children. */ retainedExecution?: RetainedExecutionState + /** WHY the retained execution has no accepted result, as a value: the safety refusal + * (`'unobservable'`) against a provider contract violation, a rejected request, a lost + * transport, or a nested recovery that could not be reconstructed. Present iff + * `retainedExecution` is; the `reason` text names the same thing, but a reader must never + * have to parse it (#1204). */ + retainedPendingCause?: RetainedPendingCause /** Epoch ms parsed from the durable settlement/cancellation record when available. */ settledAt?: number seq: number @@ -1248,6 +1255,8 @@ export interface NodeSnapshot { * (`materializeTreeView`) state the same fact, so a settle record's `tree` answers the * retained-vs-down question without the observer journal. */ readonly retainedExecution?: RetainedExecutionState + /** Why a retained child has no accepted result; see `RetainedPendingCause`. */ + readonly retainedPendingCause?: RetainedPendingCause } /** The live tree — what `scope.view` / `RootHandle.view()` materialize for a viewer. */ @@ -1375,6 +1384,7 @@ export type SpawnEvent = * instant, and the release instant is on the receipt immediately before it. Typed so a * `'pending'` can never be journaled: the journal states that as `reconciled`. */ retainedExecution?: Extract + retainedPendingCause?: RetainedPendingCause seq: number at: string } @@ -1395,6 +1405,7 @@ export type SpawnEvent = /** As on `settled`: a retained child that was cancelled settles `cancelled`, and the one * builder writes whichever kind the settlement had. */ retainedExecution?: Extract + retainedPendingCause?: RetainedPendingCause seq: number at: string } @@ -1532,6 +1543,7 @@ export type SpawnEvent = /** The settlement the driver received, verbatim, as `settled`/`cancelled` carry it. Optional * only so journals written before these fields existed remain replayable. */ reason?: string + retainedPendingCause?: RetainedPendingCause infra?: boolean trace?: WorkerTraceEvidence outRef?: string @@ -1880,6 +1892,8 @@ export interface SpendGap { */ export type RetainedExecutionState = 'pending' | 'released' +export type { RetainedPendingCause } from './retained-executor' + /** * How this run's spawned CHILDREN ended, counted by node id off the complete journal FOREST at * root settlement — after the join barrier and the release sweep, so the terminal records the diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index a73b5484..518e2f4d 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:a58a7ee58716613986d3256e2cb47e6e64718e2f356ee86bf4932f194424eadd", + "digest": "sha256:1796588e4afa2c014ab97ea9f0bd890cfb52fe252a0368658701e76d30c13d5c", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.234.0" + "runtimeVersion": "0.235.0" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:b3d9230879ed4ffaa0e60253ed1868db36bda7913b479ae5af420d005bdde5c8", - "runId": "agent-runtime-0.234.0-proposal-fixture", + "recordDigest": "sha256:6354241d7ef4a495c5f49fa308f1e197d198d03d9b13f81bf95bc81cf23dedf4", + "runId": "agent-runtime-0.235.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.234.0-proposal-fixture" + "runId": "agent-runtime-0.235.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 271f8f14..e1b9ae63 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:59878d6e5cf517ac69c6432f0c5078756c4b40be4a222af55661bd411ab68e1b", + "digest": "sha256:abe63493ef0f10236a6a77c5eee3cf4ea771e02bd085735fd57eb6180c086c09", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.234.0" + "runtimeVersion": "0.235.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:c145255a0e299d8876bfa1285bef32c76391634c20c1e3785e25f5c01c71eebe", + "recordDigest": "sha256:dbc8aa9c080c23a0b1cf3ca9116767beb27f75f9b8313574942b6c5d33e8a78b", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } diff --git a/tests/kernel/retained-environment-release.test.ts b/tests/kernel/retained-environment-release.test.ts index 433524cd..ec3f5481 100644 --- a/tests/kernel/retained-environment-release.test.ts +++ b/tests/kernel/retained-environment-release.test.ts @@ -73,6 +73,9 @@ function retainedProvider(directory: string) { const stateFile = join(directory, 'provider.json') const state = { resultLost: true, + /** What the lost read throws: a plain Error is exhibit 4's shape, unobservable by design. */ + resultError: () => new Error('provider result read lost'), + foreignEvent: false, observe: undefined as ((signal: AbortSignal | undefined) => Promise) | undefined, destroyFailure: undefined as Error | undefined, destroys: 0, @@ -87,10 +90,15 @@ function retainedProvider(directory: string) { ...session, async *events(eventOptions) { if (state.observe) await state.observe(eventOptions?.signal) + if (state.foreignEvent) { + // An event bound to another run: the runtime's own identity check must refuse it + // as a contract violation, not as an unobservable execution (#1204). + yield { type: 'message.part.updated', data: { delta: 'x', runId: 'another-run' } } + } yield* session.events(eventOptions) }, result: async () => { - if (state.resultLost) throw new Error('provider result read lost') + if (state.resultLost) throw state.resultError() return { ...(await session.result()), usage: { inputTokens: 3, outputTokens: 2 }, @@ -237,6 +245,9 @@ describe('retained environments at root settlement', () => { expect(live?.kind).toBe('down') if (live?.kind !== 'down') return expect(live.retainedExecution).toBe('pending') + // A lost read on a plain Error is exhibit 4 of #1204: the execution may have run and nothing + // local can say, so the refusal keeps its name — as a VALUE, not only in the reason text. + expect(live.retainedPendingCause).toBe('unobservable') expect(live.harnessTranscript).toBeDefined() expect(live.reason).toContain('reconciliation') // The release closed the slot: exactly one terminal record, the settlement the driver @@ -250,6 +261,7 @@ describe('retained environments at root settlement', () => { status: 'down', id: 'release:s0', retainedExecution: 'released', + retainedPendingCause: 'unobservable', infra: true, reason: live.reason, spent: expect.objectContaining({ tokensKnown: false, usdKnown: false }), @@ -287,28 +299,36 @@ describe('retained environments at root settlement', () => { expect(result.tree.nodes.find((node) => node.id === 'release:s0')).toMatchObject({ status: 'failed', retainedExecution: 'released', + retainedPendingCause: 'unobservable', }) expect(await replaySpawnTree(context.journal, context.blobs, 'release')).toMatchObject([ { kind: 'down', retainedExecution: 'released', + retainedPendingCause: 'unobservable', settledAt: live.settledAt, seq: 0, harnessTranscript: live.harnessTranscript, }, ]) expect( - materializeTreeView(events).nodes.find((node) => node.id === 'release:s0')?.retainedExecution, - ).toBe('released') + materializeTreeView(events).nodes.find((node) => node.id === 'release:s0'), + ).toMatchObject({ retainedExecution: 'released', retainedPendingCause: 'unobservable' }) // Two `agent.child` events for one node: the settlement (pending, with the driver's metering // if any) and the release (released, no metering, the release instant beside the settlement). const payloads = childPayloads(hookEvents, 'release:s0') expect(payloads.map((entry) => entry.stepIndex)).toEqual([0, 0]) expect(payloads.map((entry) => entry.payload)).toMatchObject([ - { status: 'down', retainedExecution: 'pending', settledAt: live.settledAt }, + { + status: 'down', + retainedExecution: 'pending', + retainedPendingCause: 'unobservable', + settledAt: live.settledAt, + }, { status: 'down', retainedExecution: 'released', + retainedPendingCause: 'unobservable', releasedAt: expect.any(Number), settledAt: live.settledAt, }, @@ -316,6 +336,95 @@ describe('retained environments at root settlement', () => { expect(payloads[1]?.payload).not.toHaveProperty('metered') }) + it('names an event bound to another run a provider contract violation, on the live stream', async () => { + const fleet = retainedProvider(directory) + fleet.state.foreignEvent = true + const context = createInMemoryRunContext() + let live: Settled | undefined + const result = await createSupervisor().run( + { + name: 'root', + async act(_task, scope) { + live = await spawnAndAwait(scope, retainedWorker(providerAsExecutor(fleet.provider()))) + return 'finished' + }, + }, + 'task', + { ...context, runId: 'foreign', budget: { maxIterations: 4, maxTokens: 4000 } }, + ) + expect(result.kind).toBe('winner') + expect(live?.kind).toBe('down') + if (live?.kind !== 'down') return + expect(live.retainedExecution).toBe('pending') + expect(live.retainedPendingCause).toBe('provider-contract') + expect(live.reason).toMatch(/provider contract violation; nothing to reconcile/u) + expect(live.reason).toContain('provider returned an event for another retained run') + const events = (await context.journal.loadTree('foreign')) ?? [] + expect(terminalRecords(events, 'foreign:s0')).toMatchObject([ + { retainedExecution: 'released', retainedPendingCause: 'provider-contract' }, + ]) + }) + + it('names a lost transport as such on every surface, never as the safety refusal', async () => { + // Exhibit 2 of #1204: the gateway answered the result read with an HTML 502. The Sandbox + // SDK throws ServerError{status: 502}; the runtime wraps the failed read; the classifier + // looks through the wrapper. Nothing needs reconciling; the transport failed. + const fleet = retainedProvider(directory) + fleet.state.resultError = () => + Object.assign(new Error(' tangle.tools | 502: Bad gateway'), { + name: 'ServerError', + code: 'SERVER_ERROR', + status: 502, + }) + const context = createInMemoryRunContext() + const hookEvents: RuntimeHookEvent[] = [] + let live: Settled | undefined + const result = await createSupervisor().run( + { + name: 'root', + async act(_task, scope) { + live = await spawnAndAwait(scope, retainedWorker(providerAsExecutor(fleet.provider()))) + return 'finished' + }, + }, + 'task', + { + ...context, + runId: 'contract', + budget: { maxIterations: 4, maxTokens: 4000 }, + retainedAtSettlement: 'release', + hooks: { onEvent: (event) => void hookEvents.push(event) }, + }, + ) + expect(result.kind).toBe('winner') + expect(live?.kind).toBe('down') + if (live?.kind !== 'down') return + expect(live.retainedExecution).toBe('pending') + expect(live.retainedPendingCause).toBe('transport') + expect(live.reason).toMatch(/lost its transport; status in doubt/u) + expect(live.reason).not.toMatch(/requires reconciliation/u) + const events = (await context.journal.loadTree('contract')) ?? [] + expect(events.find((event) => event.kind === 'reconciled')).toMatchObject({ + retainedPendingCause: 'transport', + }) + expect(terminalRecords(events, 'contract:s0')).toMatchObject([ + { retainedExecution: 'released', retainedPendingCause: 'transport' }, + ]) + expect(result.tree.nodes.find((node) => node.id === 'contract:s0')).toMatchObject({ + retainedPendingCause: 'transport', + }) + expect(await replaySpawnTree(context.journal, context.blobs, 'contract')).toMatchObject([ + { retainedExecution: 'released', retainedPendingCause: 'transport' }, + ]) + expect(childPayloads(hookEvents, 'contract:s0').map((entry) => entry.payload)).toMatchObject([ + { retainedExecution: 'pending', retainedPendingCause: 'transport' }, + { retainedExecution: 'released', retainedPendingCause: 'transport' }, + ]) + // The released bucket counts it the same either way: the cause says what to do, not whether + // the fleet lost the child. + expect(result.fleetYield.releasedUnrecovered).toBe(1) + }) + it('does not trip an armed intensity breaker with the released record', async () => { // The released record is the driver's earlier down re-stated after the join barrier. Counting // it would abort a run that has already settled and reclassify a delivered winner. diff --git a/tests/kernel/retained-pending-cause.test.ts b/tests/kernel/retained-pending-cause.test.ts new file mode 100644 index 00000000..20153f9d --- /dev/null +++ b/tests/kernel/retained-pending-cause.test.ts @@ -0,0 +1,180 @@ +/** + * #1204: one reason string used to cover a safety refusal and a provider contract violation. + * The cause is now classified from the typed error's STRUCTURE and carried as a value. Every + * shape below is one the runtime or the Sandbox SDK actually produces; none is invented. + */ +import { describe, expect, it } from 'vitest' +import { ValidationError } from '../../src/errors' +import { RetainedRunProviderContractError } from '../../src/runtime/retained-run-binding' +import { + classifyRetainedPendingCause, + RetainedExecutionPendingError, +} from '../../src/runtime/supervise/retained-executor' + +/** The Sandbox SDK's error shape: a `name`, a `code`, and for HTTP failures a `status`. */ +const sdk = (name: string, code: string, status?: number, message = name) => + Object.assign(new Error(message), { name, code, ...(status === undefined ? {} : { status }) }) +const zod = () => Object.assign(new Error('maximum 16384'), { name: 'ZodError', issues: [{}] }) +/** What retained-run-handle.ts mints around a thrown `handle.result()`. */ +const readFailed = (cause: unknown) => + new RetainedRunProviderContractError(cause instanceof Error ? cause.message : 'read failed', { + code: 'RETAINED_RESULT_READ_FAILED', + cause, + }) + +describe('classifyRetainedPendingCause', () => { + it('names a broken provider answer by its code, whatever it wraps', () => { + for (const code of [ + 'RETAINED_RESULT_SCHEMA_INVALID', + 'RETAINED_RESULT_BINDING_INVALID', + 'RETAINED_CONTROL_REF_INVALID', + 'RETAINED_NATIVE_CONTINUATION_RESULT_CHANGED', + 'RETAINED_EVENT_BINDING_INVALID', + 'RETAINED_EVENT_STREAM_INVALID', + ]) { + expect( + classifyRetainedPendingCause(new RetainedRunProviderContractError('bad answer', { code })), + code, + ).toBe('provider-contract') + } + // No code is the class's own default, a contract violation; a wrapper keeps it. + const bare = new RetainedRunProviderContractError('bare') + expect(classifyRetainedPendingCause(new Error('outer', { cause: bare }))).toBe( + 'provider-contract', + ) + }) + + it('looks through a failed-read wrapper at what the read hit: the #1204 ambiguity itself', () => { + // Exhibit 4: the provider cannot resolve what it admitted. A 404 AFTER admission is not a + // rejected request; the execution may have run and nothing local can say. + expect(classifyRetainedPendingCause(readFailed(sdk('NotFoundError', 'NOT_FOUND', 404)))).toBe( + 'unobservable', + ) + expect(classifyRetainedPendingCause(readFailed(sdk('StateError', 'STATE', 409)))).toBe( + 'unobservable', + ) + expect(classifyRetainedPendingCause(readFailed(sdk('TimeoutError', 'TIMEOUT', 408)))).toBe( + 'unobservable', + ) + expect(classifyRetainedPendingCause(readFailed(new Error('provider result read lost')))).toBe( + 'unobservable', + ) + // Exhibits 2 and 5: an HTML 502 body, a platform ServerError. The transport, not the answer. + expect( + classifyRetainedPendingCause( + readFailed(sdk('ServerError', 'SERVER_ERROR', 502, ' 502: Bad gateway')), + ), + ).toBe('transport') + expect( + classifyRetainedPendingCause( + readFailed( + sdk('ServerError', 'SERVER_ERROR', undefined, 'Platform key verification unavailable'), + ), + ), + ).toBe('transport') + // The SDK's NetworkError after exhausted reconnects carries no status and no cause. + expect(classifyRetainedPendingCause(readFailed(sdk('NetworkError', 'NETWORK_ERROR')))).toBe( + 'transport', + ) + expect( + classifyRetainedPendingCause( + readFailed(Object.assign(new Error('reset'), { code: 'ECONNRESET' })), + ), + ).toBe('transport') + }) + + it('names a rejected request only at admission (exhibit 1)', () => { + expect(classifyRetainedPendingCause(zod(), 'admission')).toBe('request-rejected') + expect( + classifyRetainedPendingCause(sdk('ValidationError', 'VALIDATION', 400), 'admission'), + ).toBe('request-rejected') + // The same schema failure after admission is the runtime refusing the PROVIDER's answer. + expect(classifyRetainedPendingCause(zod(), 'execution')).toBe('provider-contract') + // A client deadline says nothing about admission at either phase. + expect(classifyRetainedPendingCause(sdk('TimeoutError', 'TIMEOUT', 408), 'admission')).toBe( + 'unobservable', + ) + }) + + it('reads the members of an AggregateError, not only its cause', () => { + // retained-run-start throws AggregateError([cause, cleanupError]) with no `cause` when the + // environment destroy also fails; the first classifiable member decides. + const aggregate = new AggregateError([zod(), new Error('DELETE 409')], 'start failed') + expect(classifyRetainedPendingCause(aggregate, 'admission')).toBe('request-rejected') + // environment-provider's observation+result AggregateError: the cause is the read. + const both = new AggregateError( + [ + new Error('event arrived without a stable id'), + readFailed(sdk('ServerError', 'SERVER_ERROR', 502)), + ], + 'observation failed', + { cause: readFailed(sdk('ServerError', 'SERVER_ERROR', 502)) }, + ) + expect(classifyRetainedPendingCause(both)).toBe('transport') + }) + + it('keeps the safety refusal for what a structure-only classifier cannot name', () => { + // #1204 exhibits 3 and 6 are thrown by the provider as plain Errors with no code; until the + // provider types them they land on the safe side, and nothing here pretends otherwise. + expect( + classifyRetainedPendingCause(new Error('Tangle session event arrived without a stable id')), + ).toBe('unobservable') + expect(classifyRetainedPendingCause(new Error('value exceeds its JSON bound'))).toBe( + 'unobservable', + ) + expect(classifyRetainedPendingCause(new Error('sandbox not found'))).toBe('unobservable') + expect(classifyRetainedPendingCause('string rejection')).toBe('unobservable') + expect(classifyRetainedPendingCause(undefined)).toBe('unobservable') + }) + + it('never reads message text', () => { + expect( + classifyRetainedPendingCause( + new Error('RetainedRunProviderContractError: 502 Bad gateway ZodError ECONNRESET'), + ), + ).toBe('unobservable') + }) + + it('does not infer nested recovery; the thrower states it', () => { + expect( + classifyRetainedPendingCause( + new ValidationError( + 'driverExecutor: nested recovery has no original executor reconstruction', + ), + ), + ).toBe('unobservable') + }) + + it('does not loop on a cyclic cause chain', () => { + const a = new Error('a') + const b = new Error('b', { cause: a }) + Object.assign(a, { cause: b }) + expect(classifyRetainedPendingCause(a)).toBe('unobservable') + }) +}) + +describe('RetainedExecutionPendingError', () => { + it('carries the cause as a value and says which one in its message', () => { + const contract = new RetainedExecutionPendingError( + new RetainedRunProviderContractError('invalid', { code: 'RETAINED_RESULT_SCHEMA_INVALID' }), + ) + expect(contract.pendingCause).toBe('provider-contract') + expect(contract.message).toMatch(/provider contract violation; nothing to reconcile/u) + const unknown = new RetainedExecutionPendingError(readFailed(new Error('lost'))) + expect(unknown.pendingCause).toBe('unobservable') + expect(unknown.message).toBe( + 'retained provider execution requires reconciliation before replacement', + ) + expect(new RetainedExecutionPendingError(contract).pendingCause).toBe('provider-contract') + }) + + it('takes a phase or an explicit cause from its thrower', () => { + expect(new RetainedExecutionPendingError(zod(), 'admission').pendingCause).toBe( + 'request-rejected', + ) + expect(new RetainedExecutionPendingError(zod()).pendingCause).toBe('provider-contract') + const nested = new RetainedExecutionPendingError(new Error('opaque'), 'nested-recovery') + expect(nested.pendingCause).toBe('nested-recovery') + expect(nested.message).toMatch(/nested execution requires recovery/u) + }) +})