From b6362ed2062eb2c50b72dc52c7551ad51ab086cc Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Thu, 6 Aug 2026 12:22:52 +0530 Subject: [PATCH 1/4] fix: honor cancellation mid-attack and fix extension report gaps on stop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CLI's AbortSignal was only checked between attacks/evaluators, so a long multi-turn attack (e.g. turns: 100) couldn't be interrupted until every turn finished. The extension had a parallel gap: its own stop signal wasn't recognized by runAllBrowser, so a cancelled run returned no report at all — dropping token/cost data and leaving a fake blank turn in the HTML report. Co-Authored-By: Claude Sonnet 5 --- core/src/execute/attackRunner.ts | 10 +++++- core/src/execute/evaluatorLoop.ts | 12 +++++-- core/src/execute/mcpAttackDriver.ts | 6 ++-- core/src/execute/runAgentLoop.ts | 6 ++-- core/src/execute/runAllBrowser.ts | 11 +++++++ core/tests/attackRunner.test.ts | 49 +++++++++++++++++++++++++++++ runners/extension/orchestrator.js | 7 +++++ runners/extension/popup.html | 13 ++++++++ runners/extension/popup.js | 43 +++++++++++++++++++++++-- 9 files changed, 148 insertions(+), 9 deletions(-) diff --git a/core/src/execute/attackRunner.ts b/core/src/execute/attackRunner.ts index 4d483864..30bb2265 100644 --- a/core/src/execute/attackRunner.ts +++ b/core/src/execute/attackRunner.ts @@ -31,11 +31,19 @@ export interface AttackDriver { /** * Template Method for running one attack: the invariant skeleton every attack * kind shares. The `driver` supplies the kind-specific behavior. + * + * `signal`, when given, is checked before each turn so a long multi-turn attack + * (e.g. `turns: 100`) can be interrupted between turns rather than only between + * whole attacks/evaluators — `runAll`'s cancellation contract promises "finishes + * in-flight work", and a single attack's turn loop is exactly the granularity + * that needs to honor it. */ export async function runAttack( - driver: AttackDriver + driver: AttackDriver, + signal?: AbortSignal ): Promise { for (let turnNo = driver.startTurn; turnNo <= driver.totalTurns; turnNo++) { + if (signal?.aborted) break; const input = await driver.buildTurn(turnNo); const output = await driver.execute(input); driver.record(turnNo, input, output); diff --git a/core/src/execute/evaluatorLoop.ts b/core/src/execute/evaluatorLoop.ts index 1de6b0b6..83f9ee4d 100644 --- a/core/src/execute/evaluatorLoop.ts +++ b/core/src/execute/evaluatorLoop.ts @@ -179,7 +179,14 @@ export async function runEvaluatorAttacks( try { result = attack.kind === "mcp" - ? await runMcpAttack(attack, mcpTarget!, attackModel, judgeLlmConfig, evalTracker) + ? await runMcpAttack( + attack, + mcpTarget!, + attackModel, + judgeLlmConfig, + evalTracker, + signal + ) : await runAgentAttack( attack, attackModel, @@ -191,7 +198,8 @@ export async function runEvaluatorAttacks( targetConfig: config.target, telemetry: config.telemetry, tokenTracker: evalTracker, - } + }, + signal ); } catch (err) { const makeFailedResult = (reason: string): AttackResult => diff --git a/core/src/execute/mcpAttackDriver.ts b/core/src/execute/mcpAttackDriver.ts index 03ce7db8..acddef4b 100644 --- a/core/src/execute/mcpAttackDriver.ts +++ b/core/src/execute/mcpAttackDriver.ts @@ -169,7 +169,8 @@ export async function runMcpAttack( target: McpTarget, attackModel: LanguageModel, judgeLlm: LlmConfig, - tokenTracker?: TokenTracker + tokenTracker?: TokenTracker, + signal?: AbortSignal ): Promise { if (!attack.toolName) { return { @@ -185,6 +186,7 @@ export async function runMcpAttack( }; } return runAttack( - new McpAttackDriver(attack, target, attack.toolName, attackModel, judgeLlm, tokenTracker) + new McpAttackDriver(attack, target, attack.toolName, attackModel, judgeLlm, tokenTracker), + signal ); } diff --git a/core/src/execute/runAgentLoop.ts b/core/src/execute/runAgentLoop.ts index 2f552b03..3413e597 100644 --- a/core/src/execute/runAgentLoop.ts +++ b/core/src/execute/runAgentLoop.ts @@ -19,9 +19,11 @@ export async function runAgentAttack( attackIndex: string, patterns: AttackPattern[], target: AgentTarget, - context?: AgentAttackContext + context?: AgentAttackContext, + signal?: AbortSignal ): Promise { return runAttack( - new AgentAttackDriver(attack, attackModel, judgeModel, attackIndex, patterns, target, context) + new AgentAttackDriver(attack, attackModel, judgeModel, attackIndex, patterns, target, context), + signal ); } diff --git a/core/src/execute/runAllBrowser.ts b/core/src/execute/runAllBrowser.ts index bad15ce0..d228a242 100644 --- a/core/src/execute/runAllBrowser.ts +++ b/core/src/execute/runAllBrowser.ts @@ -209,6 +209,17 @@ export async function runAllBrowser( pushPartialResult(stopReason); break evaluatorLoop; } + // The extension's DomTarget throws a plain Error tagged `code: "OPFOR_STOP"` + // on user cancel/pause (see runners/extension/domTarget.js) — core can't + // import that class, so recognize it structurally instead. Without this, + // a user-cancelled run threw past every handler below and returned no + // report at all, silently dropping the TokenTracker totals collected so far. + if ((err as { code?: string })?.code === "OPFOR_STOP") { + stopReason = err instanceof Error ? err.message : "Run stopped by user."; + notify({ type: "run_stopped", reason: stopReason }); + pushPartialResult(stopReason); + break evaluatorLoop; + } throw err; } diff --git a/core/tests/attackRunner.test.ts b/core/tests/attackRunner.test.ts index 1c37e080..4c4589fa 100644 --- a/core/tests/attackRunner.test.ts +++ b/core/tests/attackRunner.test.ts @@ -101,3 +101,52 @@ test("runs zero turns when startTurn exceeds totalTurns, still finalizes", async await runAttack(driver); assert.deepStrictEqual(driver.calls, ["finalize"]); }); + +test("an already-aborted signal stops before turn 1, still finalizes", async () => { + const driver = trackingDriver({ startTurn: 1, totalTurns: 100 }); + const ac = new AbortController(); + ac.abort(); + await runAttack(driver, ac.signal); + assert.deepStrictEqual(driver.calls, ["finalize"]); +}); + +test("a signal aborted mid-run stops before the next turn, not after totalTurns", async () => { + const driver = trackingDriver({ startTurn: 1, totalTurns: 100 }); + const ac = new AbortController(); + const originalShouldEarlyStop = driver.shouldEarlyStop.bind(driver); + driver.shouldEarlyStop = async (t, input, output) => { + const result = await originalShouldEarlyStop(t, input, output); + if (t === 2) ac.abort(); // simulate Ctrl+C landing mid-attack + return result; + }; + await runAttack(driver, ac.signal); + // Turn 3 never builds — the loop only reached turns 1 and 2. + assert.deepStrictEqual(driver.calls, [ + "build:1", + "execute:in:1", + "record:1:in:1:out:in:1", + "stop?:1", + "build:2", + "execute:in:2", + "record:2:in:2:out:in:2", + "stop?:2", + "finalize", + ]); +}); + +test("no signal behaves exactly as before (backward compatible)", async () => { + const driver = trackingDriver({ startTurn: 1, totalTurns: 2 }); + const result = await runAttack(driver, undefined); + assert.strictEqual(result, RESULT); + assert.deepStrictEqual(driver.calls, [ + "build:1", + "execute:in:1", + "record:1:in:1:out:in:1", + "stop?:1", + "build:2", + "execute:in:2", + "record:2:in:2:out:in:2", + "stop?:2", + "finalize", + ]); +}); diff --git a/runners/extension/orchestrator.js b/runners/extension/orchestrator.js index 45f7b420..500e3222 100644 --- a/runners/extension/orchestrator.js +++ b/runners/extension/orchestrator.js @@ -913,6 +913,9 @@ export async function executeAdaptiveRedTeamRun(sendResponse, message, resume) { transcript: fullTranscript, turns: fullTurnLog, judgment: cancelledJudgment(fullTranscript), + tokenUsage: report?.summary?.tokenUsage ?? report?.evaluators?.[0]?.tokenUsage, + tokenUsageByModel: + report?.summary?.tokenUsageByModel ?? report?.evaluators?.[0]?.tokenUsageByModel, }, }); sendResponse(response); @@ -944,6 +947,8 @@ export async function executeAdaptiveRedTeamRun(sendResponse, message, resume) { turns: fullTurnLog, judgment: errorJudgment, tokenUsage: report?.summary?.tokenUsage ?? report?.evaluators?.[0]?.tokenUsage, + tokenUsageByModel: + report?.summary?.tokenUsageByModel ?? report?.evaluators?.[0]?.tokenUsageByModel, }; await persistPartialResult(partialResult); try { @@ -998,6 +1003,8 @@ export async function executeAdaptiveRedTeamRun(sendResponse, message, resume) { turns: fullTurnLog, judgment, tokenUsage: report.summary?.tokenUsage ?? report.evaluators?.[0]?.tokenUsage, + tokenUsageByModel: + report.summary?.tokenUsageByModel ?? report.evaluators?.[0]?.tokenUsageByModel, }; await persistPartialResult(finalResult); try { diff --git a/runners/extension/popup.html b/runners/extension/popup.html index 04b59ee3..c2e41fc8 100644 --- a/runners/extension/popup.html +++ b/runners/extension/popup.html @@ -2613,6 +2613,19 @@ " > + diff --git a/runners/extension/popup.js b/runners/extension/popup.js index 7d62296c..3f3a5140 100644 --- a/runners/extension/popup.js +++ b/runners/extension/popup.js @@ -11,6 +11,8 @@ import { PROVIDER_CAPABILITIES, PROVIDER_DISPLAY_NAMES, renderReport, + estimateRunCost, + formatUsd, } from "./dist/core.bundle.js"; const PROVIDER_OPTIONS = Object.values(PROVIDERS).map((value) => ({ @@ -1065,6 +1067,18 @@ function renderDone() { tokenEl.style.display = "none"; } + const costEl = $("statCost"); + const cost = state.lastReport?.summary?.cost; + if (cost) { + // Testing cost, not total cost — the target's own inference spend is + // invisible to opfor, and an unpriced model makes this a lower bound. + $("statCostValue").textContent = (cost.complete ? "" : "≥") + formatUsd(cost.totalUsd); + $("statCostSub").textContent = cost.complete ? "estimated" : "lower bound"; + costEl.style.display = ""; + } else { + costEl.style.display = "none"; + } + $("resultsCountLabel").textContent = `Evaluators · ${state.results.length}`; const list = $("resultsList"); list.innerHTML = ""; @@ -1257,17 +1271,34 @@ function buildReport() { let aggInput = 0; let aggOutput = 0; + // Merge each evaluator's per-model breakdown by key ("provider:model") so a + // model used across multiple (single-evaluator) runAllBrowser calls prices + // once as one bucket instead of once per evaluator. + const byModelMap = new Map(); for (const r of state.results) { const tu = r.raw?.tokenUsage; if (tu) { aggInput += tu.inputTokens ?? 0; aggOutput += tu.outputTokens ?? 0; } + for (const m of r.raw?.tokenUsageByModel ?? []) { + const existing = byModelMap.get(m.key); + if (existing) { + existing.inputTokens += m.inputTokens; + existing.outputTokens += m.outputTokens; + existing.totalTokens += m.totalTokens; + existing.calls += m.calls; + existing.roles = [...new Set([...existing.roles, ...m.roles])].sort(); + } else { + byModelMap.set(m.key, { ...m, roles: [...m.roles] }); + } + } } const tokenUsage = aggInput + aggOutput > 0 ? { inputTokens: aggInput, outputTokens: aggOutput, totalTokens: aggInput + aggOutput } : undefined; + const cost = estimateRunCost([...byModelMap.values()]); const now = new Date(); const stamp = now.toISOString().replace(/[-:]/g, "").replace(/\..+/, "").replace("T", "-"); @@ -1316,6 +1347,7 @@ function buildReport() { criticalFindings: criticalFindings.length, highFindings: highFindings.length, tokenUsage, + cost, durationMs, }, cancelled: state.runCancelled, @@ -1388,6 +1420,12 @@ function toTurnViewModels(raw) { function toResultViewModel(tr, evaluator) { const cancelled = tr.verdict === "CANCELLED"; + const turns = toTurnViewModels(evaluator.raw); + // Mirror core's buildReport.ts contract: `detail` is the last attempted + // prompt/response, used as the transcript fallback when `turns` is empty + // (e.g. cancelled before any turn completed). Hardcoding blanks here made a + // cancelled-with-zero-turns evaluator render a fake "1 turn" with no content. + const last = turns[turns.length - 1]?.detail; return { id: evaluator.id + (tr.testNumber > 1 ? `-${tr.testNumber}` : ""), label: tr.pattern, @@ -1401,8 +1439,8 @@ function toResultViewModel(tr, evaluator) { : (tr.reasoning ?? ""), errorMessage: cancelled ? "Cancelled by user" : undefined, }, - detail: { kind: "prompt", prompt: "", response: "" }, - turns: toTurnViewModels(evaluator.raw), + detail: { kind: "prompt", prompt: last?.prompt ?? "", response: last?.response ?? "" }, + turns, }; } @@ -1447,6 +1485,7 @@ function toReportViewModel(report) { safetyScore: summary.safetyScore ?? 0, attackSuccessRate: summary.attackSuccessRate ?? 0, tokenUsage: summary.tokenUsage, + cost: summary.cost, durationMs: summary.durationMs, }, evaluators: report.evaluatorResults.map(toEvaluatorViewModel), From f0a92a1efea1fe6c170185f65354910b51821220 Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Thu, 6 Aug 2026 15:55:57 +0530 Subject: [PATCH 2/4] fix: price cached input tokens at the provider's cache rate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cost estimation charged every input token at the full input rate, but `inputTokens` is the inclusive total — it already contains the cached tokens providers bill far more cheaply. The token counter was also discarding the cache split the AI SDK already reports. Input is now divided across its tiers and each priced at its own rate. Validated against LiteLLM billing for a 24-request run: previously reported $0.070749, actually billed $0.057939, now reports $0.057939 — every request agreeing to 12 decimal places. Reported fields are unchanged; only the cost figure moves. A run whose provider reports no cache split prices exactly as before, and a tier with no published rate falls back to the full input rate rather than to free. Co-Authored-By: Claude Opus 5 --- core/src/execute/tokenTracker.ts | 98 +++++++++++++++++++++++++++--- core/src/pricing/estimateCost.ts | 26 ++++++-- core/src/pricing/types.ts | 5 +- core/tests/pricing.test.ts | 100 +++++++++++++++++++++++++++++++ core/tests/tokenTracker.test.ts | 98 ++++++++++++++++++++++++++++++ 5 files changed, 311 insertions(+), 16 deletions(-) diff --git a/core/src/execute/tokenTracker.ts b/core/src/execute/tokenTracker.ts index ed7ba6a7..c39eb969 100644 --- a/core/src/execute/tokenTracker.ts +++ b/core/src/execute/tokenTracker.ts @@ -27,6 +27,26 @@ export interface TokenUsage { totalTokens: number; } +/** + * How one call's input tokens split across cache tiers. + * + * `inputTokens` is the **inclusive** total — providers report it as + * `noCache + cacheRead + cacheWrite`, not as the fresh-text count alone. Pricing + * therefore has to divide that total between the tiers, never add a cache charge + * on top of it (which would bill cached tokens twice). + * + * Carried only so {@link estimateRunCost} can apply each tier's rate; it is not + * reported on its own. + */ +export interface InputCacheSplit { + /** Input tokens processed fresh, billed at the full input rate. */ + noCache: number; + /** Input tokens served from cache, billed at the provider's (much lower) read rate. */ + cacheRead: number; + /** Input tokens written to cache, billed at the provider's write rate. */ + cacheWrite: number; +} + /** Token usage attributed to one provider/model pair. */ export interface ModelTokenUsage extends TokenUsage { /** `":"`, or `"unknown"` for usage that could not be attributed. */ @@ -37,6 +57,14 @@ export interface ModelTokenUsage extends TokenUsage { roles: string[]; /** Number of LLM calls recorded against this model. */ calls: number; + /** + * Cache split of {@link TokenUsage.inputTokens}, present only when this model + * actually hit a cache. Exists so each tier can be priced at its own rate; the + * three fields sum to `inputTokens`. See {@link InputCacheSplit}. + */ + noCacheInputTokens?: number; + cacheReadInputTokens?: number; + cacheWriteInputTokens?: number; } /** Optional provenance supplied alongside a usage recording. */ @@ -65,20 +93,43 @@ export const LlmUsageSchema = z inputTokens: z.number().int().min(0).optional().default(0), outputTokens: z.number().int().min(0).optional().default(0), totalTokens: z.number().int().min(0).optional().default(0), + // Provider-agnostic cache split, supplied by the AI SDK as part of `usage`. + // Absent on providers (or call paths) that don't report it — see the + // transform below, which then treats every input token as uncached. + inputTokenDetails: z + .object({ + noCacheTokens: z.number().int().min(0).optional(), + cacheReadTokens: z.number().int().min(0).optional(), + cacheWriteTokens: z.number().int().min(0).optional(), + }) + .passthrough() + .optional(), }) .passthrough() - .transform((u) => ({ - inputTokens: u.inputTokens, - outputTokens: u.outputTokens, - totalTokens: u.totalTokens > 0 ? u.totalTokens : u.inputTokens + u.outputTokens, - })); + .transform((u) => { + const cacheRead = u.inputTokenDetails?.cacheReadTokens ?? 0; + const cacheWrite = u.inputTokenDetails?.cacheWriteTokens ?? 0; + // `inputTokens` already includes the cached tokens, so the fresh count is the + // remainder — derived rather than trusted so a provider that reports only + // some of the three fields still leaves the tiers summing to inputTokens. + const noCache = + u.inputTokenDetails?.noCacheTokens ?? Math.max(0, u.inputTokens - cacheRead - cacheWrite); + return { + inputTokens: u.inputTokens, + outputTokens: u.outputTokens, + totalTokens: u.totalTokens > 0 ? u.totalTokens : u.inputTokens + u.outputTokens, + // Omitted entirely when nothing was cached, so a non-caching run records + // and prices exactly as it did before this field existed. + ...(cacheRead > 0 || cacheWrite > 0 ? { cache: { noCache, cacheRead, cacheWrite } } : {}), + }; + }); /** * Validate and normalize a raw usage object (from any provider/SDK) into a * clean {@link TokenUsage}. Returns `undefined` when the input is falsy or * fails validation so callers can safely discard garbage. */ -export function parseUsage(raw: unknown): TokenUsage | undefined { +export function parseUsage(raw: unknown): (TokenUsage & { cache?: InputCacheSplit }) | undefined { if (!raw || typeof raw !== "object") return undefined; const result = LlmUsageSchema.safeParse(raw); return result.success ? result.data : undefined; @@ -94,6 +145,11 @@ interface ModelBucket { input: number; output: number; total: number; + // Cache tiers of `input`. Calls that report no split count entirely as + // noCache, so these three always sum to `input`. + noCache: number; + cacheRead: number; + cacheWrite: number; } /** @@ -119,7 +175,12 @@ export class TokenTracker { * call site degrades to today's behavior rather than losing tokens. */ record( - usage?: { inputTokens?: number; outputTokens?: number; totalTokens?: number }, + usage?: { + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + cache?: InputCacheSplit; + }, attribution?: RecordAttribution ): void { if (!usage) return; @@ -132,7 +193,7 @@ export class TokenTracker { this.output += out; this.total += resolvedTotal; - this.recordToBucket(inp, out, resolvedTotal, attribution); + this.recordToBucket(inp, out, resolvedTotal, attribution, usage.cache); } /** Fold one call's usage into its per-model bucket, creating the bucket on first sight. */ @@ -140,7 +201,8 @@ export class TokenTracker { inp: number, out: number, total: number, - attribution?: RecordAttribution + attribution?: RecordAttribution, + cache?: InputCacheSplit ): void { const identity = resolveModelIdentity(attribution?.model); const key = identity ? modelKey(identity) : UNKNOWN_MODEL_KEY; @@ -156,6 +218,9 @@ export class TokenTracker { input: 0, output: 0, total: 0, + noCache: 0, + cacheRead: 0, + cacheWrite: 0, }; this.buckets.set(key, bucket); } @@ -166,6 +231,11 @@ export class TokenTracker { bucket.input += inp; bucket.output += out; bucket.total += total; + // No split reported → the whole call was uncached, keeping the three tiers + // summing to `input` even when only some calls to this model were cached. + bucket.noCache += cache?.noCache ?? inp; + bucket.cacheRead += cache?.cacheRead ?? 0; + bucket.cacheWrite += cache?.cacheWrite ?? 0; } /** Current accumulated totals. Uses the provider-supplied total when available. */ @@ -193,6 +263,15 @@ export class TokenTracker { inputTokens: b.input, outputTokens: b.output, totalTokens: b.total, + // Omitted when this model never hit a cache, so the shape is unchanged + // for runs where caching never applied. + ...(b.cacheRead > 0 || b.cacheWrite > 0 + ? { + noCacheInputTokens: b.noCache, + cacheReadInputTokens: b.cacheRead, + cacheWriteInputTokens: b.cacheWrite, + } + : {}), })) .sort((a, b) => b.totalTokens - a.totalTokens || a.key.localeCompare(b.key)); } @@ -222,6 +301,7 @@ class ChildTracker extends TokenTracker { inputTokens?: number; outputTokens?: number; totalTokens?: number; + cache?: InputCacheSplit; }, attribution?: RecordAttribution ): void { diff --git a/core/src/pricing/estimateCost.ts b/core/src/pricing/estimateCost.ts index 4452a0a3..945780d7 100644 --- a/core/src/pricing/estimateCost.ts +++ b/core/src/pricing/estimateCost.ts @@ -6,9 +6,11 @@ * whenever `complete` is false, and the UI must say so — a report that quietly * treats an unknown model as $0 understates spend while looking authoritative. * - * Repeat ("cached") input tokens are not yet separated by the token counter, so - * all input is priced at the full input rate. For multi-turn runs against - * providers that discount repeated context, that makes this an over-estimate. + * Input is priced per cache tier. `inputTokens` is the inclusive total, so the + * tiers are *divided out of* it rather than charged on top — adding a cache + * charge to the full input charge would bill every cached token twice. A tier + * whose rate the price table doesn't publish falls back to the full input rate, + * keeping the same never-quietly-free stance as an unpriced model. */ import type { ModelTokenUsage } from "../execute/tokenTracker.js"; @@ -33,11 +35,25 @@ function costOne(usage: ModelTokenUsage): ModelCost { if (!found) return base; + const { inputPerToken, outputPerToken, cacheReadPerToken, cacheWritePerToken } = found.price; + + // Split the (inclusive) input total across cache tiers. A model that reported + // no split priced every input token at the full rate before this existed, and + // still does: cacheRead/cacheWrite fall to 0 and noCache absorbs the total. + const cacheRead = usage.cacheReadInputTokens ?? 0; + const cacheWrite = usage.cacheWriteInputTokens ?? 0; + const noCache = + usage.noCacheInputTokens ?? Math.max(0, usage.inputTokens - cacheRead - cacheWrite); + + // `??` not `||`: a published rate of 0 is real (some providers don't charge + // for cache writes) and must not be mistaken for a missing rate. return { ...base, usd: - usage.inputTokens * found.price.inputPerToken + - usage.outputTokens * found.price.outputPerToken, + noCache * inputPerToken + + cacheRead * (cacheReadPerToken ?? inputPerToken) + + cacheWrite * (cacheWritePerToken ?? inputPerToken) + + usage.outputTokens * outputPerToken, source: "table", matchedKey: found.matchedKey, }; diff --git a/core/src/pricing/types.ts b/core/src/pricing/types.ts index 0287615b..5b72d617 100644 --- a/core/src/pricing/types.ts +++ b/core/src/pricing/types.ts @@ -14,10 +14,11 @@ export interface ModelPrice { outputPerToken: number; /** * USD per cached (repeat) input token, when the provider publishes one. - * Not applied yet — token counting does not separate repeat tokens today. + * Falls back to {@link inputPerToken} when absent, so an unpublished cache + * rate over-estimates rather than under-charges. */ cacheReadPerToken?: number; - /** USD per cache-write token, when the provider publishes one. Not applied yet. */ + /** USD per cache-write token, when the provider publishes one. Same fallback. */ cacheWritePerToken?: number; } diff --git a/core/tests/pricing.test.ts b/core/tests/pricing.test.ts index 7e710a59..ecba6a82 100644 --- a/core/tests/pricing.test.ts +++ b/core/tests/pricing.test.ts @@ -178,6 +178,106 @@ test("attacker and judge are priced separately at their own rates", () => { assert.ok(Math.abs(cost.totalUsd - (judge.usd + attacker.usd)) < 1e-9); }); +// --------------------------------------------------------------------------- +// Cache-tier pricing +// +// `inputTokens` is the inclusive total, so the tiers must be divided out of it. +// Charging a cache rate *on top of* the full input charge would bill every +// cached token twice — these pin the split against that. +// --------------------------------------------------------------------------- + +/** A breakdown row whose input tokens carry a cache split. */ +function cachedUsage( + provider: string, + model: string, + noCache: number, + cacheRead: number, + cacheWrite: number, + outputTokens: number +): ModelTokenUsage { + const inputTokens = noCache + cacheRead + cacheWrite; + return { + ...usage(provider, model, inputTokens, outputTokens), + noCacheInputTokens: noCache, + cacheReadInputTokens: cacheRead, + cacheWriteInputTokens: cacheWrite, + }; +} + +test("cached input is billed at the cache rate, not the full input rate", () => { + const price = lookupPrice("anthropic", "claude-opus-5"); + assert.ok(price?.price.cacheReadPerToken); + const cost = estimateRunCost([cachedUsage("anthropic", "claude-opus-5", 2000, 8000, 0, 500)]); + assert.ok(cost); + const expected = + 2000 * price.price.inputPerToken + + 8000 * price.price.cacheReadPerToken! + + 500 * price.price.outputPerToken; + assert.ok(Math.abs(cost.totalUsd - expected) < 1e-12); +}); + +test("a cache hit costs strictly less than the same tokens uncached", () => { + const cached = estimateRunCost([cachedUsage("anthropic", "claude-opus-5", 2000, 8000, 0, 500)]); + const uncached = estimateRunCost([usage("anthropic", "claude-opus-5", 10_000, 500)]); + assert.ok(cached && uncached); + // Same 10k input tokens either way — the split is the only difference. + assert.equal(cached.byModel[0].inputTokens, uncached.byModel[0].inputTokens); + assert.ok(cached.totalUsd < uncached.totalUsd); +}); + +test("cached tokens are billed once, not once per tier", () => { + // The double-count bug would price the 8000 cache-read tokens at both the + // full input rate and the cache rate, landing above the all-uncached figure. + const price = lookupPrice("anthropic", "claude-opus-5"); + assert.ok(price); + const cost = estimateRunCost([cachedUsage("anthropic", "claude-opus-5", 2000, 8000, 0, 0)]); + assert.ok(cost); + assert.ok(cost.totalUsd < 10_000 * price.price.inputPerToken); +}); + +test("cache writes are priced at their own published rate", () => { + const price = lookupPrice("anthropic", "claude-opus-5"); + assert.ok(price?.price.cacheWritePerToken); + const cost = estimateRunCost([cachedUsage("anthropic", "claude-opus-5", 0, 0, 4000, 0)]); + assert.ok(cost); + assert.ok(Math.abs(cost.totalUsd - 4000 * price.price.cacheWritePerToken!) < 1e-12); + // Anthropic charges a premium to write the cache — above the input rate. + assert.ok(price.price.cacheWritePerToken! > price.price.inputPerToken); +}); + +test("a published cache rate of zero is honored, not treated as missing", () => { + // deepseek publishes cw: 0. A `||` fallback would silently reprice these at + // the full input rate; `??` keeps them free. + const price = lookupPrice("openai-compatible", "deepseek/deepseek-v4-pro"); + assert.ok(price); + assert.equal(price.price.cacheWritePerToken, 0); + const cost = estimateRunCost([ + cachedUsage("openai-compatible", "deepseek/deepseek-v4-pro", 0, 0, 50_000, 0), + ]); + assert.ok(cost); + assert.equal(cost.totalUsd, 0); +}); + +test("a model with no published cache rate falls back to the full input rate", () => { + // Never-quietly-free: an unpublished cache tier over-estimates rather than + // under-charging, matching how an unpriced model is handled. + const price = lookupPrice("azure", "command-r-plus"); + assert.ok(price); + assert.equal(price.price.cacheReadPerToken, undefined); + const cost = estimateRunCost([cachedUsage("azure", "command-r-plus", 1000, 9000, 0, 0)]); + assert.ok(cost); + assert.ok(Math.abs(cost.totalUsd - 10_000 * price.price.inputPerToken) < 1e-12); +}); + +test("a breakdown with no cache split prices exactly as before", () => { + const price = lookupPrice("openai", "gpt-4o-mini"); + assert.ok(price); + const cost = estimateRunCost([usage("openai", "gpt-4o-mini", 10_000, 1000)]); + assert.ok(cost); + const expected = 10_000 * price.price.inputPerToken + 1000 * price.price.outputPerToken; + assert.ok(Math.abs(cost.totalUsd - expected) < 1e-12); +}); + test("an unpriced model is reported, not silently counted as free", () => { const cost = estimateRunCost([ usage("openai", "gpt-4o-mini", 1000, 100), diff --git a/core/tests/tokenTracker.test.ts b/core/tests/tokenTracker.test.ts index 9ed369a9..94fc1f0f 100644 --- a/core/tests/tokenTracker.test.ts +++ b/core/tests/tokenTracker.test.ts @@ -127,6 +127,104 @@ test("parseUsage strips unknown provider metadata and returns normalized usage", assert.deepStrictEqual(result, { inputTokens: 100, outputTokens: 20, totalTokens: 150 }); }); +// Cache split. `inputTokens` is the inclusive total the provider reports, so the +// three tiers must always divide it — never extend it. + +test("parseUsage extracts the cache split the SDK reports", () => { + const result = parseUsage({ + inputTokens: 10_000, + outputTokens: 500, + inputTokenDetails: { noCacheTokens: 2000, cacheReadTokens: 8000, cacheWriteTokens: 0 }, + }); + assert.deepStrictEqual(result, { + inputTokens: 10_000, + outputTokens: 500, + totalTokens: 10_500, + cache: { noCache: 2000, cacheRead: 8000, cacheWrite: 0 }, + }); +}); + +test("parseUsage derives the uncached remainder when the SDK omits it", () => { + const result = parseUsage({ + inputTokens: 10_000, + outputTokens: 0, + inputTokenDetails: { cacheReadTokens: 6000, cacheWriteTokens: 1000 }, + }); + assert.deepStrictEqual(result?.cache, { noCache: 3000, cacheRead: 6000, cacheWrite: 1000 }); +}); + +test("parseUsage omits the cache split entirely when nothing was cached", () => { + const result = parseUsage({ + inputTokens: 100, + outputTokens: 20, + inputTokenDetails: { noCacheTokens: 100, cacheReadTokens: 0, cacheWriteTokens: 0 }, + }); + // Same shape as a provider that reports no details at all — a non-caching run + // records and prices exactly as it did before the split existed. + assert.deepStrictEqual(result, { inputTokens: 100, outputTokens: 20, totalTokens: 120 }); +}); + +test("tracker keeps the cache tiers summing to inputTokens", () => { + const tracker = new TokenTracker(); + tracker.record({ + inputTokens: 10_000, + outputTokens: 500, + totalTokens: 10_500, + cache: { noCache: 2000, cacheRead: 8000, cacheWrite: 0 }, + }); + const [bucket] = tracker.breakdown; + assert.equal(bucket.inputTokens, 10_000); + assert.equal( + bucket.noCacheInputTokens! + bucket.cacheReadInputTokens! + bucket.cacheWriteInputTokens!, + bucket.inputTokens + ); + // Run totals are untouched by the split — the report shows what it always did. + assert.deepStrictEqual(tracker.totals, { + inputTokens: 10_000, + outputTokens: 500, + totalTokens: 10_500, + }); +}); + +test("a model mixing cached and uncached calls still balances", () => { + const tracker = new TokenTracker(); + tracker.record({ inputTokens: 500, outputTokens: 10 }); // no split reported + tracker.record({ + inputTokens: 10_000, + outputTokens: 20, + cache: { noCache: 2000, cacheRead: 8000, cacheWrite: 0 }, + }); + const [bucket] = tracker.breakdown; + assert.equal(bucket.inputTokens, 10_500); + // The uncached call folds into noCache, so the tiers still divide the total. + assert.equal(bucket.noCacheInputTokens, 2500); + assert.equal(bucket.cacheReadInputTokens, 8000); + assert.equal( + bucket.noCacheInputTokens! + bucket.cacheReadInputTokens! + bucket.cacheWriteInputTokens!, + bucket.inputTokens + ); +}); + +test("breakdown omits cache fields for a model that never hit a cache", () => { + const tracker = new TokenTracker(); + tracker.record({ inputTokens: 100, outputTokens: 20 }); + const [bucket] = tracker.breakdown; + assert.equal(bucket.cacheReadInputTokens, undefined); + assert.equal(bucket.noCacheInputTokens, undefined); +}); + +test("a child tracker propagates the cache split to its parent", () => { + const parent = new TokenTracker(); + const child = parent.child(); + child.record({ + inputTokens: 10_000, + outputTokens: 0, + cache: { noCache: 2000, cacheRead: 8000, cacheWrite: 0 }, + }); + assert.equal(parent.breakdown[0].cacheReadInputTokens, 8000); + assert.equal(child.breakdown[0].cacheReadInputTokens, 8000); +}); + // --------------------------------------------------------------------------- // Per-model attribution // --------------------------------------------------------------------------- From c481c023a7244962b9ff4f147300304a4525a026 Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Thu, 6 Aug 2026 16:06:39 +0530 Subject: [PATCH 3/4] fix: enforce the cache-split invariant instead of trusting the provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The transform read `noCacheTokens` verbatim, so a provider reporting inputTokens 100 with noCache 100 and cacheRead 50 produced a 150-token split against a 100-token call — and estimateRunCost billed all 150. The invariant was asserted in the docs and tests but never enforced. Derive the fresh count from inputTokens instead. For a well-formed provider the two agree (the AI SDK builds inputTokens as the sum), so this costs nothing and makes the invariant hold by construction. A split claiming more cached tokens than there was input can't be divided at all, so it is dropped and the call prices at the full input rate. Addresses CodeRabbit review on #236. Co-Authored-By: Claude Opus 5 --- core/src/execute/tokenTracker.ts | 23 +++++++++++++++-------- core/tests/tokenTracker.test.ts | 25 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/core/src/execute/tokenTracker.ts b/core/src/execute/tokenTracker.ts index c39eb969..72e9da68 100644 --- a/core/src/execute/tokenTracker.ts +++ b/core/src/execute/tokenTracker.ts @@ -109,18 +109,25 @@ export const LlmUsageSchema = z .transform((u) => { const cacheRead = u.inputTokenDetails?.cacheReadTokens ?? 0; const cacheWrite = u.inputTokenDetails?.cacheWriteTokens ?? 0; - // `inputTokens` already includes the cached tokens, so the fresh count is the - // remainder — derived rather than trusted so a provider that reports only - // some of the three fields still leaves the tiers summing to inputTokens. - const noCache = - u.inputTokenDetails?.noCacheTokens ?? Math.max(0, u.inputTokens - cacheRead - cacheWrite); + const cached = cacheRead + cacheWrite; + // `inputTokens` already includes the cached tokens, so the fresh count is + // always the remainder. Derived rather than read from `noCacheTokens`: cost + // divides inputTokens between the tiers, so a reported split that doesn't add + // up would bill tokens the call never used. For a well-formed provider the + // two agree — the AI SDK builds inputTokens as noCache + cacheRead + + // cacheWrite — so deriving costs nothing and makes the invariant hold. + // + // A split claiming more cached tokens than there was input can't be divided + // at all; drop it and let the call price at the full input rate, the same + // conservative direction taken for an unpriced model. + const usable = cached > 0 && cached <= u.inputTokens; return { inputTokens: u.inputTokens, outputTokens: u.outputTokens, totalTokens: u.totalTokens > 0 ? u.totalTokens : u.inputTokens + u.outputTokens, - // Omitted entirely when nothing was cached, so a non-caching run records - // and prices exactly as it did before this field existed. - ...(cacheRead > 0 || cacheWrite > 0 ? { cache: { noCache, cacheRead, cacheWrite } } : {}), + // Omitted when nothing was cached, so a non-caching run records and prices + // exactly as it did before this field existed. + ...(usable ? { cache: { noCache: u.inputTokens - cached, cacheRead, cacheWrite } } : {}), }; }); diff --git a/core/tests/tokenTracker.test.ts b/core/tests/tokenTracker.test.ts index 94fc1f0f..110b6888 100644 --- a/core/tests/tokenTracker.test.ts +++ b/core/tests/tokenTracker.test.ts @@ -153,6 +153,31 @@ test("parseUsage derives the uncached remainder when the SDK omits it", () => { assert.deepStrictEqual(result?.cache, { noCache: 3000, cacheRead: 6000, cacheWrite: 1000 }); }); +test("parseUsage ignores a reported noCacheTokens that breaks the invariant", () => { + // inputTokens 100 with noCache 100 AND cacheRead 50 sums to 150. Trusting the + // reported figure would price 150 tokens against a 100-token call. + const result = parseUsage({ + inputTokens: 100, + outputTokens: 0, + inputTokenDetails: { noCacheTokens: 100, cacheReadTokens: 50 }, + }); + assert.deepStrictEqual(result?.cache, { noCache: 50, cacheRead: 50, cacheWrite: 0 }); + const { noCache, cacheRead, cacheWrite } = result!.cache!; + assert.equal(noCache + cacheRead + cacheWrite, result!.inputTokens); +}); + +test("parseUsage drops a split claiming more cached tokens than input", () => { + // Undividable — fall back to pricing the whole call at the full input rate + // rather than inventing a negative fresh-token count. + const result = parseUsage({ + inputTokens: 100, + outputTokens: 0, + inputTokenDetails: { cacheReadTokens: 400, cacheWriteTokens: 0 }, + }); + assert.equal(result?.cache, undefined); + assert.equal(result?.inputTokens, 100); +}); + test("parseUsage omits the cache split entirely when nothing was cached", () => { const result = parseUsage({ inputTokens: 100, From a14c5818deefe7a054c64fa0ffdf2541c05ebd84 Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Thu, 6 Aug 2026 16:15:26 +0530 Subject: [PATCH 4/4] fix: omit the transcript when an attack captured no exchange MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An attack interrupted before its first turn completed has no turns and an empty detail card, but the renderer emitted a transcript anyway — headed "1 turn" with two blank bubbles. That reads as an exchange that happened and came back empty, rather than as nothing having run. Skip the transcript and its toggle when there is no content on either the turns list or the detail card. The judge's error message still renders, so the reason the attack produced nothing is not lost with it. Docs: the cache-aware pricing landed earlier in this PR made the "multi-turn runs are over-estimated" caveat wrong in README.md and docs/cli.md; replaced with what actually happens now and the cases that still over-estimate. AGENTS.md gains the turn-granular cancellation contract and the inclusive-split invariant behind the cost maths. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 6 +- README.md | 4 +- core/src/report/render.ts | 44 ++++++++++--- core/tests/render.test.ts | 131 ++++++++++++++++++++++++++++++++++++++ docs/cli.md | 4 +- 5 files changed, 176 insertions(+), 13 deletions(-) create mode 100644 core/tests/render.test.ts diff --git a/AGENTS.md b/AGENTS.md index 942408de..92639ba2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -277,10 +277,14 @@ There is no longer a separate `generate` step. `opfor run --config ` does **MCP targets** additionally run baseline pre-flight scans (`runBaselineScans`) before the evaluator loop — these enumerate `tools/list` + `resources/list` and judge them for poisoning / leakage independent of any evaluator. Throughout the run, `runAll` fans lifecycle events to registered `RunListener`s (progress reporting, NDJSON streaming) rather than only a callback. -**Cancellation.** `RunAllOptions` accepts an optional `signal?: AbortSignal`. When aborted, the evaluator loop finishes the in-flight attack, skips remaining evaluators/attacks, and returns a partial report with `stopReason: "user-interrupted"`. The CLI wires this to SIGINT (first Ctrl+C = graceful stop, second = force kill). The SDK can reuse the same mechanism for programmatic cancellation. +**Cancellation.** `RunAllOptions` accepts an optional `signal?: AbortSignal`. It is threaded down to `runAttack` and checked **before each turn**, not just between attacks — otherwise a single high-`turns` attack would run to completion before the stop took effect. When aborted, the in-flight turn finishes, the attack is finalized (judging whatever transcript exists), remaining evaluators/attacks are skipped, and a partial report is returned with `stopReason: "user-interrupted"`. The CLI wires this to SIGINT (first Ctrl+C = graceful stop, second = force kill). The SDK can reuse the same mechanism for programmatic cancellation. + +The browser extension reaches the same granularity by a different route: `DomTarget.send()` checks `state.OPFOR_STOP` around each send/extract and throws an error tagged `code: "OPFOR_STOP"`, which `runAllBrowser` recognizes as a clean stop (structurally, since core cannot import the extension's class) and turns into a partial report rather than an unhandled throw. **Token usage tracking.** `runAll` creates a `TokenTracker` (see `core/src/execute/tokenTracker.ts`) and threads it through the evaluator loop → attack drivers → `withRetry` / `generateText` / `chatCompletionJsonContent` call sites. Every LLM call auto-records its `usage` (input/output tokens). Per-evaluator child trackers aggregate into evaluator-level totals (`EvaluatorResult.tokenUsage`); the parent accumulates run-level totals (`UnifiedRunReport.summary.tokenUsage`). The CLI prints a summary line, the HTML report shows a stat card, and the JSON report includes the data for CI. `runAllBrowser` follows the same pattern so the extension popup can show a token count. +**Cost is cache-aware.** `inputTokens` is the **inclusive** total — providers report it as `noCache + cacheRead + cacheWrite`. `estimateRunCost` (`core/src/pricing/estimateCost.ts`) therefore _divides_ it across those tiers and prices each at its own published rate; adding a cache charge on top of the full input charge would bill cached tokens twice. The tracker reads the split from the AI SDK's provider-agnostic `usage.inputTokenDetails`, so Anthropic's `cache_read_input_tokens` and OpenAI/DeepSeek's `prompt_tokens_details.cached_tokens` both work without provider-specific code. Two invariants worth preserving when touching this: the fresh count is always _derived_ from `inputTokens` (a split that doesn't add up is dropped rather than trusted), and a tier with no published rate falls back to the full input rate — never to free. + `runAllBrowser` is the same loop in browser-safe form: takes preloaded `EvaluatorSpec[]` + a pre-built `AgentTarget` (e.g. `DomTarget`), skips disk reads. --- diff --git a/README.md b/README.md index d6867fc9..f8385b35 100644 --- a/README.md +++ b/README.md @@ -124,9 +124,9 @@ Testing cost: $0.18 This is **opfor's own spend** — the attacker and judge LLMs. It excludes your target's inference cost, which opfor cannot see from the outside. The per-model split is the useful part: the judge is often the bigger share, and pointing it at a cheaper model is usually the easiest saving. -Prices come from a snapshot of LiteLLM's public price map that ships with the package, so runs work offline and a report re-rendered later produces the same figure. Two caveats worth knowing: +Prices come from a snapshot of LiteLLM's public price map that ships with the package, so runs work offline and a report re-rendered later produces the same figure. Cached input is billed at the provider's cache rate — multi-turn attacks re-send the conversation each turn, and that repeated prefix is often ~100× cheaper than fresh text, so the figure tracks the real bill rather than a worst case. Caveats worth knowing: -- **Multi-turn runs read high.** Providers discount repeated context, and a multi-turn attack re-sends the conversation each turn — opfor prices every input token at full rate, so the real bill is usually lower. +- **Caching is only credited when it's reported.** A provider that doesn't break out cached tokens, or a model with no published cache rate, is charged at the full input rate — an over-estimate, chosen over quietly under-reporting. - **Unknown models are never counted as free.** A model missing from the price table is reported as unpriced and the total is marked a lower bound, rather than silently reading as $0. - **A few helper calls aren't metered yet.** Trace curation, session summarisation and one JSON helper don't report token usage, so their spend is missing from the total. Treat the figure as a floor. diff --git a/core/src/report/render.ts b/core/src/report/render.ts index 14d33528..577a84bd 100644 --- a/core/src/report/render.ts +++ b/core/src/report/render.ts @@ -682,6 +682,26 @@ function singleTurnTranscript(detail: DetailCard): string { `; } +/** + * Whether there is any exchange worth rendering a transcript for. + * + * An attack interrupted before its first turn completed has no turns and an + * empty detail card. Rendering that anyway produced a transcript headed + * "1 turn" containing two blank bubbles, which reads as an exchange that was + * captured but came back empty — rather than as nothing having run at all. + */ +function hasTranscriptContent(r: ResultViewModel): boolean { + if (r.turns && r.turns.length > 0) return true; + const d = r.detail; + if (d.kind === "prompt") return Boolean(d.prompt?.trim() || d.response?.trim()); + return Boolean( + d.toolName?.trim() || + d.response?.trim() || + d.error?.trim() || + Object.keys(d.args ?? {}).length > 0 + ); +} + /** Render one attack result: reasoning/evidence, confidence + standards, and a collapsible transcript. */ function resultDetailCard( r: ResultViewModel, @@ -738,14 +758,10 @@ function resultDetailCard( ` : ""; - return ` - ${showTestHeading ? `
Test ${index + 1} — ${esc(r.label)}
` : ""} - ${reasoningHtml} - ${evidenceHtml} -
- ${confidenceCol} - ${standardsCol} -
+ // Nothing was exchanged (e.g. cancelled before the first turn finished), so + // drop the transcript and its toggle rather than show an empty one. + const transcriptHtml = hasTranscriptContent(r) + ? `
Conversation Transcript ${turnCount} turn${turnCount === 1 ? "" : "s"}
@@ -758,5 +774,15 @@ function resultDetailCard( `; + ` + : ""; + + return ` + ${showTestHeading ? `
Test ${index + 1} — ${esc(r.label)}
` : ""} + ${reasoningHtml} + ${evidenceHtml} +
+ ${confidenceCol} + ${standardsCol} +
${transcriptHtml}`; } diff --git a/core/tests/render.test.ts b/core/tests/render.test.ts new file mode 100644 index 00000000..ff1bf130 --- /dev/null +++ b/core/tests/render.test.ts @@ -0,0 +1,131 @@ +/** + * Report rendering — transcript presence. + * + * An attack interrupted before its first turn completed carries no turns and an + * empty detail card. The renderer used to emit a transcript anyway, headed + * "1 turn" with two blank bubbles, which reads as an exchange that happened and + * came back empty rather than as nothing having run. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { renderReport } from "../src/report/render.js"; +import type { DetailCard, ReportViewModel, TurnViewModel } from "../src/report/types.js"; + +/** A report carrying exactly one attack result, for transcript assertions. */ +function reportWith(detail: DetailCard, turns?: TurnViewModel[]): string { + const model: ReportViewModel = { + mode: "agent", + reportId: "test-report", + generatedAt: new Date("2026-01-01T00:00:00Z").toISOString(), + generatorModel: "attacker-model", + judgeModel: "judge-model", + target: { name: "target" }, + summary: { + total: 1, + passed: 0, + failed: 0, + errors: 1, + safetyScore: 0, + attackSuccessRate: 0, + }, + evaluators: [ + { + evaluatorId: "ev", + evaluatorName: "Evaluator", + severity: "critical", + total: 1, + passed: 0, + failed: 0, + errors: 1, + passRate: 0, + results: [ + { + id: "a1", + label: "Pattern", + judge: { + verdict: "ERROR", + score: 0, + confidence: 0, + evidence: "", + reasoning: "", + errorMessage: "Cancelled by user", + }, + detail, + turns, + }, + ], + }, + ], + }; + return renderReport(model); +} + +const EMPTY_PROMPT: DetailCard = { kind: "prompt", prompt: "", response: "" }; + +// The bare class names appear in the page's stylesheet and its inline script, +// so assert on the rendered elements rather than the class alone. +const TOGGLE_BUTTON = '