diff --git a/CHANGELOG.md b/CHANGELOG.md index a2d271a3..22f22d57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 0.216.0 + +A provider-stated dollar with no billing receipt behind it is priced once, not twice. +The cost path priced every call a catalog could price and then added the provider's own stated +total, so a run whose provider reported dollars AND whose calls were catalogue-priced counted the +same work on both channels. +Only the calls no receipt covers are priced now, so `usd - usdEstimated` remains what a provider is +known to have billed. +A consumer reading `usdEstimated` on a run with partial receipts should expect a smaller figure. + +The durable settle record now retains a leaked reservation, so a run that hit the conserved-pool +invariant is diagnosable from its own record rather than only from the exception. +`runDirectoryHolderIsLive` is documented as advisory: it answers about the instant it was asked. + ## 0.215.0 A lost bridge run now waits before replaying, instead of burning every reconnect attempt back to diff --git a/api-surface.json b/api-surface.json index 121d2e87..b1ace7f1 100644 --- a/api-surface.json +++ b/api-surface.json @@ -969,7 +969,7 @@ "Interval": "type 7beacbf4a336", "IsolatedCheckOptions": "type c0648ebd04db", "IsolatedCheckResult": "type 105120ddd047", - "Iteration": "type 5370c086cdd2", + "Iteration": "type 60e6c09425fa", "KeyProvider": "type 0ad3a4b60a7f", "Leaderboard": "type 5664041325d1", "LeaderboardBenchScore": "type ef7cb230f39b", @@ -997,7 +997,7 @@ "LoopOptionsForDispatch": "type f04ec83b947a", "LoopPlanDescription": "type 76b21ce0ef26", "LoopPlanPayload": "type 43448c05358d", - "LoopResult": "type 25d8a3e038fb", + "LoopResult": "type 72710561eb38", "LoopSandboxPlacement": "type a99e4be1736f", "LoopShape": "type 8925f5e259c9", "LoopStartedPayload": "type a552ef7496eb", diff --git a/docs/api/durable.md b/docs/api/durable.md index 5c76334f..bb5288a3 100644 --- a/docs/api/durable.md +++ b/docs/api/durable.md @@ -2569,6 +2569,12 @@ the question instead of hand-rolling `process.kill(pid, 0)`. A bare signal probe live holder from an unrelated process that later took the same pid, which is the failure this lock's start token exists to prevent. +The answer is ADVISORY: it is read outside the mutation guard `acquireRunDirectoryLock` holds, +so a `false` can be stale by the time the caller acts on it and a concurrent acquire can take +the directory in between. Reporting and listing are what this is for. A caller that intends to +TAKE the directory calls `acquireRunDirectoryLock`, which evaluates the same rule under the +guard and refuses atomically. + #### Parameters ##### runDir diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 73c12283..03485ccc 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.215.0` and `@tangle-network/agent-eval@0.180.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.216.0` and `@tangle-network/agent-eval@0.180.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 54f959ce..61cbb5f2 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -24328,6 +24328,17 @@ Raw sandbox event stream collected for this iteration. Present on a failed itera False when `costUsd` is only the observed subtotal, not a complete bill. +##### unprovenCostUsd? + +> `optional` **unprovenCostUsd?**: `number` + +The part of `costUsd` that came from calls carrying no billing receipt, summed per call. + +`costUsdKnown` is an AND over the iteration, so it cannot say HOW MUCH of the total is +unproven: one receiptless call marks the whole iteration unknown. This is the amount that +belongs on `Spend.usdEstimated`, which keeps `usd - usdEstimated` reading as billed money on +a settlement that mixed both kinds. Absent when every dollar here carried a receipt. + ##### estimatedCostUsd? > `optional` **estimatedCostUsd?**: `number` @@ -24596,6 +24607,13 @@ Sum of every iteration's `costUsd`. False when `costUsd` is only the observed subtotal, not a complete bill. +##### unprovenCostUsd? + +> `optional` **unprovenCostUsd?**: `number` + +Sum of every iteration's `unprovenCostUsd` — the part of `costUsd` no billing receipt + covers. Absent when every dollar in the loop carried one. + ##### estimatedCostUsd? > `optional` **estimatedCostUsd?**: `number` diff --git a/docs/canonical-api.md b/docs/canonical-api.md index c2312a8c..48b98232 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.215.0.** +> **Version 0.216.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.180.0 <0.181.0`. > `sandbox` must satisfy `>=0.36.4 <0.39.0`. diff --git a/package.json b/package.json index c3a1a0a2..2527bb51 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.215.0", + "version": "0.216.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/durable/run-lock.ts b/src/durable/run-lock.ts index e4568fa9..26778ede 100644 --- a/src/durable/run-lock.ts +++ b/src/durable/run-lock.ts @@ -135,6 +135,12 @@ export interface RunDirectoryHolderLiveness { * the question instead of hand-rolling `process.kill(pid, 0)`. A bare signal probe cannot tell a * live holder from an unrelated process that later took the same pid, which is the failure this * lock's start token exists to prevent. + * + * The answer is ADVISORY: it is read outside the mutation guard `acquireRunDirectoryLock` holds, + * so a `false` can be stale by the time the caller acts on it and a concurrent acquire can take + * the directory in between. Reporting and listing are what this is for. A caller that intends to + * TAKE the directory calls `acquireRunDirectoryLock`, which evaluates the same rule under the + * guard and refuses atomically. */ export async function runDirectoryHolderIsLive( runDir: string, diff --git a/src/runtime/run-loop.ts b/src/runtime/run-loop.ts index d7bf2a0a..7b349968 100644 --- a/src/runtime/run-loop.ts +++ b/src/runtime/run-loop.ts @@ -722,7 +722,13 @@ async function executeIteration(args: ExecuteIterationArgs { sawLlmCall = true slot.costUsd += llmCall.costUsd ?? 0 - if (llmCall.usdKnown === false) slot.costUsdKnown = false + if (llmCall.usdKnown === false) { + slot.costUsdKnown = false + // Accumulated per CALL, because `costUsdKnown` is an AND over the whole iteration: one + // unproven call marks the iteration unknown, and a consumer that priced the iteration TOTAL + // from that flag would relabel every receipt-backed dollar beside it as a price. + slot.unprovenCostUsd = (slot.unprovenCostUsd ?? 0) + (llmCall.costUsd ?? 0) + } if (llmCall.estimatedCostUsd !== undefined) { slot.estimatedCostUsd = (slot.estimatedCostUsd ?? 0) + llmCall.estimatedCostUsd } @@ -1091,6 +1097,10 @@ function finalize( } const costUsd = args.iterations.reduce((sum, iter) => sum + iter.costUsd, 0) const costUsdKnown = args.iterations.every((iter) => iter.costUsdKnown !== false) + const unprovenCostUsd = args.iterations.reduce( + (sum, iter) => sum + (iter.unprovenCostUsd ?? 0), + 0, + ) const estimatedCostUsd = args.iterations.reduce( (sum, iter) => sum + (iter.estimatedCostUsd ?? 0), 0, @@ -1118,6 +1128,7 @@ function finalize( durationMs: args.now() - args.startMs, costUsd, ...(costUsdKnown ? {} : { costUsdKnown: false }), + ...(unprovenCostUsd > 0 ? { unprovenCostUsd } : {}), ...(estimatedCostUsd > 0 ? { estimatedCostUsd } : {}), ...(Object.keys(promptCache).length > 0 ? { promptCache } : {}), tokenUsage, diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index 45069768..e770ded0 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -1327,12 +1327,19 @@ async function* streamSandboxLeaf(args: StreamSandboxArgs): AsyncIterable 0 ? unprovenUsd : undefined const outWithUsage = { ...outWithOutcome, ...(result.estimatedCostUsd !== undefined diff --git a/src/runtime/types.ts b/src/runtime/types.ts index 35b91cd5..7b872887 100644 --- a/src/runtime/types.ts +++ b/src/runtime/types.ts @@ -232,6 +232,15 @@ export interface Iteration { costUsd: number /** False when `costUsd` is only the observed subtotal, not a complete bill. */ costUsdKnown?: false + /** + * The part of `costUsd` that came from calls carrying no billing receipt, summed per call. + * + * `costUsdKnown` is an AND over the iteration, so it cannot say HOW MUCH of the total is + * unproven: one receiptless call marks the whole iteration unknown. This is the amount that + * belongs on `Spend.usdEstimated`, which keeps `usd - usdEstimated` reading as billed money on + * a settlement that mixed both kinds. Absent when every dollar here carried a receipt. + */ + unprovenCostUsd?: number /** Local/catalog estimates remain separate from billed spend. */ estimatedCostUsd?: number /** Provider-reported prompt-cache fields; absent fields remain unknown. */ @@ -332,6 +341,9 @@ export interface LoopResult { costUsd: number /** False when `costUsd` is only the observed subtotal, not a complete bill. */ costUsdKnown?: false + /** Sum of every iteration's `unprovenCostUsd` — the part of `costUsd` no billing receipt + * covers. Absent when every dollar in the loop carried one. */ + unprovenCostUsd?: number /** Sum of separately-labelled local/catalog estimates. */ estimatedCostUsd?: number /** Aggregated provider-reported prompt-cache fields. */ diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 5746108e..480eca22 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:fc63774f0bfd14a14008e605b3af1d219c3dd000d23f8927f7099ed46bd43f11", + "digest": "sha256:cc5ee5b8cc2015ad99204d729011eb41a15d76fffd1d5fa5ef87e9d1ffc55d96", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.215.0" + "runtimeVersion": "0.216.0" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:76ae29ff744a977a353aa6b4086b677af665427976e12b43e53c216e7d7cb17c", - "runId": "agent-runtime-0.215.0-proposal-fixture", + "recordDigest": "sha256:475bd7945cfe8c4d711f3a7d79fd44c8ee84ac73a3eb6a10ab769d7b74f34a05", + "runId": "agent-runtime-0.216.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.215.0-proposal-fixture" + "runId": "agent-runtime-0.216.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index ecb5eb3d..dbe3ba15 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:de5a5c471910fccf7867732f50a239a7fbe9b8fdbc2caf314afc18cecd039832", + "digest": "sha256:ca79a4342e47b309b84836b6deded31fb51520d41f4b839472af99784daa9eca", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.215.0" + "runtimeVersion": "0.216.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:46dfa991cc83075144c7ed26278f8e0a844c5f9c8f8c91396e2282103d96c3e4", + "recordDigest": "sha256:94e79e0bad096e4d0d5d9459f46f85589d00a584f9fde10cad2d0068bd9d3bc9", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } diff --git a/tests/runtime/leaked-reservation-settlement.test.ts b/tests/runtime/leaked-reservation-settlement.test.ts index 0d5e872b..8a3fc9c2 100644 --- a/tests/runtime/leaked-reservation-settlement.test.ts +++ b/tests/runtime/leaked-reservation-settlement.test.ts @@ -13,6 +13,7 @@ */ import { describe, expect, it } from 'vitest' +import { settleRecordJson } from '../../src/durable/settle-record' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' import { createBudgetPool } from '../../src/runtime/supervise/budget' import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' @@ -115,6 +116,22 @@ describe('a reservation leaked at the join barrier', () => { expect(typeof leak?.ticketId).toBe('number') }) + it('survives into the durable settle record, which is what an autopsy reads', async () => { + const result = await runRoot(async (task, scope) => { + await spawnLeakingChild(task, scope) + return undefined + }) + + // `result.json` holds the whole settled result, so the finding outlives the process that made + // it. The leak was the reason #1181 left a `failure.json` with `nodes: 0` and no settle record + // at all; the evidence is worth nothing if it only exists in memory. + const recorded = JSON.parse(settleRecordJson(result)) as { + leakedReservations?: ReadonlyArray<{ childId?: string; assignment?: string }> + } + expect(recorded.leakedReservations).toHaveLength(1) + expect(recorded.leakedReservations?.[0]?.assignment).toBe('extract-ghrist') + }) + it('still fails loud on the success path, where a corrupt total would travel as a winner', async () => { await expect( runRoot(async (task, scope) => { diff --git a/tests/runtime/unproven-dollars-are-estimates.test.ts b/tests/runtime/unproven-dollars-are-estimates.test.ts index 36227454..66e17f02 100644 --- a/tests/runtime/unproven-dollars-are-estimates.test.ts +++ b/tests/runtime/unproven-dollars-are-estimates.test.ts @@ -18,8 +18,14 @@ import { type ObserverRecord, observerRecordDigest } from '../../src/durable/obs import { projectPursuit } from '../../src/durable/observer-projection' import { spendFromUsageEvents } from '../../src/runtime/supervise/budget' import { createInbox } from '../../src/runtime/supervise/inbox' +import { createExecutor } from '../../src/runtime/supervise/runtime' import { createSteerableSandboxSession } from '../../src/runtime/supervise/sandbox-session' -import type { Spend, UsageEvent } from '../../src/runtime/supervise/types' +import type { + AgentSpec, + ExecutorContext, + Spend, + UsageEvent, +} from '../../src/runtime/supervise/types' import type { SandboxClient } from '../../src/runtime/types' import { testAgentProfile } from '../kernel/test-agent-profile' @@ -201,3 +207,87 @@ describe('a sandbox child whose box states a dollar figure', () => { expect(projectedCost(spent as Spend)?.provenance).toBe('unknown') }) }) + +const leafSpec: AgentSpec = { + profile: testAgentProfile('leaf', { harness: 'opencode' }), + harness: 'opencode', +} + +/** Settle the cloud-child leaf executor over a scripted box stream — the path a spawned Tangle + * Sandbox child really settles on, and the one the #1175 measurement was taken from. */ +async function settleLeaf(frames: ReadonlyArray) { + const executor = createExecutor({ backend: 'sandbox', sandboxClient: boxClient(frames) })( + leafSpec, + { signal: new AbortController().signal, seams: {} } as ExecutorContext, + ) + const events: UsageEvent[] = [] + for await (const event of executor.execute( + 'extract', + new AbortController().signal, + ) as AsyncIterable) { + events.push(event) + } + return { events, spent: executor.resultArtifact().spent } +} + +/** One model call the box prices without a receipt behind it. */ +function unprovenCall(costUsd: number): SandboxEvent { + return { + type: 'llm_call', + data: { costUsd, usdKnown: false, tokensIn: 100, tokensOut: 10 }, + } as unknown as SandboxEvent +} + +/** The box's terminal frame. Its `totalCostUsd` is the platform's own cumulative receipt, so the + * ledger credits the part above what the priced calls already reported. */ +function terminal(totalCostUsd?: number): SandboxEvent { + return { + type: 'done', + data: { + outcome: { type: 'completed' }, + tokenUsage: { inputTokens: 100, outputTokens: 10 }, + ...(totalCostUsd === undefined ? {} : { totalCostUsd }), + }, + } as unknown as SandboxEvent +} + +describe('the cloud-child leaf, whose settlement #1175 was measured on', () => { + it('prices only the calls no receipt covered, keeping a mixed settlement separable', async () => { + // $0.50 priced with no receipt, then a $1.50 cumulative platform receipt: the ledger credits + // the $1.00 residual as billed money. Before the partition this reported the whole $1.50 as a + // price, so `usd - usdEstimated` read $0.00 billed against a receipt that really stated $1.00. + const { events, spent } = await settleLeaf([unprovenCall(0.5), terminal(1.5)]) + + expect(spent.usd).toBe(1.5) + expect(spent.usdEstimated).toBe(0.5) + expect(spent.usd - (spent.usdEstimated ?? 0)).toBe(1) + // One unproven call marks the whole channel unknown; it does not make every dollar a price. + expect(spent.usdKnown).toBe(false) + expect(events.find((event) => event.kind === 'cost')).toMatchObject({ + usd: 1.5, + usdKnown: false, + usdEstimated: 0.5, + }) + expect(projectedCost(spent)?.provenance).toBe('partial') + }) + + it('prices the whole figure when no call carried a receipt', async () => { + const { spent } = await settleLeaf([unprovenCall(0.37), terminal()]) + + expect(spent.usd).toBe(0.37) + expect(spent.usdEstimated).toBe(0.37) + expect(spent.usdKnown).toBe(false) + expect(projectedCost(spent)?.provenance).toBe('estimated') + }) + + it('names no estimate when every dollar the box reported carried a receipt', async () => { + const { spent } = await settleLeaf([terminal(0.045)]) + + expect(spent.usd).toBe(0.045) + // Nothing was priced, so the estimate channel stays empty and the receipt reads as billed. + expect(spent.usdEstimated).toBeUndefined() + // Absence is how this type states a known dollar channel; only `false` marks it unproven. + expect(spent.usdKnown).not.toBe(false) + expect(projectedCost(spent)?.provenance).toBe('reported') + }) +})