feat: estimate and report run cost per model

README.md

@@ -111,6 +111,27 @@ When you run a scan, opfor:
 
 Each run lands in its own subfolder under `.opfor/reports/run-report-<compactTs>-<slug>-<shortId>/` containing `<slug>-report.html` and `<slug>-report.json`. Autonomous `opfor hunt` runs use the same layout under `hunt-report-<compactTs>-<slug>-<shortId>/`.
 
+### Testing cost
+
+Every run reports what its instrumented LLM calls cost, broken down by model:
+
+```text
+Token usage: 51,323 input / 6,057 output (57,380 total)
+Testing cost: $0.18
+   deepseek/deepseek-v4-pro [attacker]: $0.037
+   anthropic/claude-opus-5 [judge]: $0.14
+```
+
+This is **opfor's own spend** — the attacker and judge LLMs. It excludes your target's inference cost, which opfor cannot see from the outside. The per-model split is the useful part: the judge is often the bigger share, and pointing it at a cheaper model is usually the easiest saving.
+
+Prices come from a snapshot of LiteLLM's public price map that ships with the package, so runs work offline and a report re-rendered later produces the same figure. Two caveats worth knowing:
+
+- **Multi-turn runs read high.** Providers discount repeated context, and a multi-turn attack re-sends the conversation each turn — opfor prices every input token at full rate, so the real bill is usually lower.
+- **Unknown models are never counted as free.** A model missing from the price table is reported as unpriced and the total is marked a lower bound, rather than silently reading as $0.
+- **A few helper calls aren't metered yet.** Trace curation, session summarisation and one JSON helper don't report token usage, so their spend is missing from the total. Treat the figure as a floor.
+
+→ [Token usage and testing cost](docs/cli.md#token-usage-and-testing-cost)
+
 ## Evaluator coverage
 
 Opfor ships with curated suites that map to industry standards. Pick a suite or run individual evaluators.

core/package.json

@@ -43,6 +43,10 @@
       "types": "./dist/providers/*.d.ts",
       "default": "./dist/providers/*.js"
     },
+    "./pricing/*.js": {
+      "types": "./dist/pricing/*.d.ts",
+      "default": "./dist/pricing/*.js"
+    },
     "./evaluators/*.js": {
       "types": "./dist/evaluators/*.d.ts",
       "default": "./dist/evaluators/*.js"

core/src/browser.ts

@@ -69,6 +69,14 @@ export {
 } from "./providers/factory.js";
 export type { LlmConfig, ProviderName } from "./config/types.js";
 
+// Cost estimation. The vendored price table is a plain inlined module with no
+// Node imports, so esbuild bundles it for the extension like any other source.
+export { estimateRunCost, formatUsd } from "./pricing/estimateCost.js";
+export { lookupPrice } from "./pricing/lookupPrice.js";
+export { PRICE_TABLE_VERSION } from "./pricing/priceTable.generated.js";
+export type { RunCost, ModelCost, ModelPrice, CostSource } from "./pricing/types.js";
+export type { ModelTokenUsage, TokenUsage } from "./execute/tokenTracker.js";
+
 export { getAdapter } from "./telemetry/adapter.js";
 
 export { renderReport } from "./report/render.js";

core/src/evaluators/judge.ts

@@ -206,7 +206,7 @@ export async function judgeResponse(
   try {
     const result = await withRetry(
       () => generateText({ model, system: JUDGE_SYSTEM, prompt: judgePrompt }),
-      { context: "Judge", maxRetries: 3, tokenTracker }
+      { context: "Judge", maxRetries: 3, tokenTracker, model }
     );
     return parseJudgeOutput(result.text);
   } catch (err) {

core/src/execute/baselineScanner.ts

@@ -8,6 +8,7 @@ import { createHash } from "node:crypto";
 import { z } from "zod";
 import { randomUUID } from "../lib/random.js";
 import { judgeToolResponse } from "../run/judge.js";
+import type { TokenTracker } from "./tokenTracker.js";
 import { errorJudge as mcpErrorJudge } from "../lib/judgeTypes.js";
 import { toEvaluatorResult } from "./aggregate.js";
 import { log } from "../lib/logger.js";
@@ -24,6 +25,12 @@ export interface BaselineScanContext {
   config: RunConfig;
   outputDir?: string;
   notify: (event: ProgressEvent) => void;
+  /**
+   * Run-level token accumulator. Baseline scans judge every tool description and
+   * resource before the evaluator loop starts, so without this their spend is
+   * invisible to the run's token count and cost estimate.
+   */
+  tokenTracker?: TokenTracker;
 }
 
 /**
@@ -132,6 +139,7 @@ async function scanResources(ctx: BaselineScanContext): Promise<AttackResult[]>
     try {
       judgeResult = await judgeToolResponse({
         model: judgeModelConfig,
+        tokenTracker: ctx.tokenTracker,
         evaluator: {
           id: evalId,
           name: "MCP Resource Exposure",
@@ -186,6 +194,7 @@ async function scanToolDescriptions(ctx: BaselineScanContext): Promise<AttackRes
     try {
       judgeResult = await judgeToolResponse({
         model: judgeModelConfig,
+        tokenTracker: ctx.tokenTracker,
         evaluator: {
           id: evalId,
           name: "Tool Description Poisoning Scan",

core/src/execute/evaluatorLoop.ts

@@ -17,6 +17,7 @@ import { verdictIcon } from "../lib/verdictIcon.js";
 import { TurnPlan } from "./turnPlan.js";
 import { isStopError, getStopReason } from "../lib/llmRetry.js";
 import type { TokenTracker } from "./tokenTracker.js";
+import { estimateRunCost } from "../pricing/estimateCost.js";
 import { log } from "../lib/logger.js";
 import type {
   RunConfig,
@@ -226,7 +227,11 @@ export async function runEvaluatorAttacks(
         attackResults.push(makeFailedResult(stopReason));
         notify({ type: "attack_done", attackId: attack.id, verdict: "ERROR" });
         const partialResult = toEvaluatorResult(evaluatorMeta, attackResults);
-        if (evalTracker) partialResult.tokenUsage = evalTracker.totals;
+        if (evalTracker) {
+          partialResult.tokenUsage = evalTracker.totals;
+          partialResult.tokenUsageByModel = evalTracker.breakdown;
+          partialResult.cost = estimateRunCost(partialResult.tokenUsageByModel);
+        }
         evaluatorResults.push(partialResult);
         return { evaluatorResults, stopReason };
       }
@@ -243,7 +248,11 @@ export async function runEvaluatorAttacks(
     notify({ type: "evaluator_done", evaluatorId: evaluator.id, passed, failed, errors });
 
     const evResult = toEvaluatorResult(evaluatorMeta, attackResults);
-    if (evalTracker) evResult.tokenUsage = evalTracker.totals;
+    if (evalTracker) {
+      evResult.tokenUsage = evalTracker.totals;
+      evResult.tokenUsageByModel = evalTracker.breakdown;
+      evResult.cost = estimateRunCost(evResult.tokenUsageByModel);
+    }
     evaluatorResults.push(evResult);
     sessionMap.set(evaluator.id, captureSessionContext(evaluator, attackResults));
   }

core/src/execute/runAll.ts

@@ -17,6 +17,7 @@ import type { LlmConfig } from "../config/types.js";
 import { getAdapter } from "../telemetry/adapter.js";
 import { runSetupTraceCuration } from "../telemetry/curation.js";
 import { TokenTracker } from "./tokenTracker.js";
+import { estimateRunCost } from "../pricing/estimateCost.js";
 import { log } from "../lib/logger.js";
 
 export interface RunAllOptions {
@@ -117,6 +118,7 @@ export async function runAll(
           config,
           outputDir: options?.outputDir,
           notify,
+          tokenTracker,
         }))
       );
     }
@@ -144,6 +146,8 @@ export async function runAll(
     const usage = tokenTracker.totals;
     if (usage.totalTokens > 0) {
       report.summary.tokenUsage = usage;
+      report.summary.tokenUsageByModel = tokenTracker.breakdown;
+      report.summary.cost = estimateRunCost(report.summary.tokenUsageByModel);
     }
     report.summary.durationMs = Date.now() - runStartedAt;
     if (stopReason) {

core/src/execute/runAllBrowser.ts

@@ -20,6 +20,7 @@ import {
   modelLabel,
 } from "./aggregate.js";
 import { TokenTracker } from "./tokenTracker.js";
+import { estimateRunCost } from "../pricing/estimateCost.js";
 import type {
   AgentAttackSpec,
   AttackResult,
@@ -189,6 +190,8 @@ export async function runAllBrowser(
             attackResults
           );
           partialResult.tokenUsage = evalTracker.totals;
+          partialResult.tokenUsageByModel = evalTracker.breakdown;
+          partialResult.cost = estimateRunCost(partialResult.tokenUsageByModel);
           evaluatorResults.push(partialResult);
         };
 
@@ -230,13 +233,17 @@ export async function runAllBrowser(
       attackResults
     );
     evResult.tokenUsage = evalTracker.totals;
+    evResult.tokenUsageByModel = evalTracker.breakdown;
+    evResult.cost = estimateRunCost(evResult.tokenUsageByModel);
     evaluatorResults.push(evResult);
   }
 
   const report = buildBrowserReport(config, evaluatorResults, stopReason);
   const usage = tokenTracker.totals;
   if (usage.totalTokens > 0) {
     report.summary.tokenUsage = usage;
+    report.summary.tokenUsageByModel = tokenTracker.breakdown;
+    report.summary.cost = estimateRunCost(report.summary.tokenUsageByModel);
   }
   report.summary.durationMs = Date.now() - runStartedAt;
   return report;

core/src/execute/tokenTracker.ts

@@ -5,9 +5,20 @@
  * attack drivers. Each `generateText` / `generateObject` call site records its
  * usage after the call completes (including retries). Aggregated totals are
  * surfaced in the CLI summary, HTML/JSON report, and extension popup.
+ *
+ * Usage is recorded twice: once into flat run totals, and once into a per-model
+ * bucket. The per-model breakdown exists because a run can mix models — the
+ * judge may be a different (and far more expensive) model than the attacker —
+ * so a single combined total cannot be priced. See {@link ModelTokenUsage}.
  */
 
 import { z } from "zod";
+import {
+  modelKey,
+  resolveModelIdentity,
+  UNKNOWN_MODEL_KEY,
+  type ModelRef,
+} from "../providers/modelIdentity.js";
 
 /** Aggregated input/output/total token counts from LLM calls. */
 export interface TokenUsage {
@@ -16,6 +27,26 @@ export interface TokenUsage {
   totalTokens: number;
 }
 
+/** Token usage attributed to one provider/model pair. */
+export interface ModelTokenUsage extends TokenUsage {
+  /** `"<provider>:<model>"`, or `"unknown"` for usage that could not be attributed. */
+  key: string;
+  provider: string;
+  model: string;
+  /** Which phases used this model — `"attacker"`, `"judge"`. Sorted, deduped. */
+  roles: string[];
+  /** Number of LLM calls recorded against this model. */
+  calls: number;
+}
+
+/** Optional provenance supplied alongside a usage recording. */
+export interface RecordAttribution {
+  /** The model the call was made against — an AI SDK model, an LlmConfig, or an identity. */
+  model?: ModelRef;
+  /** Run phase, e.g. `"attacker"` or `"judge"`. Case-insensitive. */
+  role?: string;
+}
+
 /** Shared zero-value constant to avoid re-allocating empty usage objects. */
 export const ZERO_USAGE: TokenUsage = Object.freeze({
   inputTokens: 0,
@@ -53,6 +84,18 @@ export function parseUsage(raw: unknown): TokenUsage | undefined {
   return result.success ? result.data : undefined;
 }
 
+/** Mutable per-model accumulator; projected to {@link ModelTokenUsage} on read. */
+interface ModelBucket {
+  key: string;
+  provider: string;
+  model: string;
+  roles: Set<string>;
+  calls: number;
+  input: number;
+  output: number;
+  total: number;
+}
+
 /**
  * Accumulator for LLM token usage.
  *
@@ -64,20 +107,65 @@ export class TokenTracker {
   private input = 0;
   private output = 0;
   private total = 0;
+  private readonly buckets = new Map<string, ModelBucket>();
 
   /**
    * Record usage from a single LLM call. Safe to call with undefined/partial
    * usage. When `totalTokens` is supplied it is preserved; otherwise it falls
    * back to `inputTokens + outputTokens`.
+   *
+   * `attribution` is optional — usage recorded without it lands in the
+   * `"unknown"` bucket and still counts toward run totals, so an un-migrated
+   * call site degrades to today's behavior rather than losing tokens.
    */
-  record(usage?: { inputTokens?: number; outputTokens?: number; totalTokens?: number }): void {
+  record(
+    usage?: { inputTokens?: number; outputTokens?: number; totalTokens?: number },
+    attribution?: RecordAttribution
+  ): void {
     if (!usage) return;
     const inp = usage.inputTokens ?? 0;
     const out = usage.outputTokens ?? 0;
     const tot = usage.totalTokens ?? 0;
+    const resolvedTotal = tot > 0 ? tot : inp + out;
+
     this.input += inp;
     this.output += out;
-    this.total += tot > 0 ? tot : inp + out;
+    this.total += resolvedTotal;
+
+    this.recordToBucket(inp, out, resolvedTotal, attribution);
+  }
+
+  /** Fold one call's usage into its per-model bucket, creating the bucket on first sight. */
+  private recordToBucket(
+    inp: number,
+    out: number,
+    total: number,
+    attribution?: RecordAttribution
+  ): void {
+    const identity = resolveModelIdentity(attribution?.model);
+    const key = identity ? modelKey(identity) : UNKNOWN_MODEL_KEY;
+
+    let bucket = this.buckets.get(key);
+    if (!bucket) {
+      bucket = {
+        key,
+        provider: identity?.provider ?? "unknown",
+        model: identity?.model ?? "unknown",
+        roles: new Set<string>(),
+        calls: 0,
+        input: 0,
+        output: 0,
+        total: 0,
+      };
+      this.buckets.set(key, bucket);
+    }
+
+    const role = attribution?.role?.trim().toLowerCase();
+    if (role) bucket.roles.add(role);
+    bucket.calls += 1;
+    bucket.input += inp;
+    bucket.output += out;
+    bucket.total += total;
   }
 
   /** Current accumulated totals. Uses the provider-supplied total when available. */
@@ -89,6 +177,26 @@ export class TokenTracker {
     };
   }
 
+  /**
+   * Per-model usage, heaviest first. Always sums to {@link totals}; models that
+   * could not be attributed appear under the `"unknown"` key rather than being
+   * dropped.
+   */
+  get breakdown(): ModelTokenUsage[] {
+    return [...this.buckets.values()]
+      .map((b) => ({
+        key: b.key,
+        provider: b.provider,
+        model: b.model,
+        roles: [...b.roles].sort(),
+        calls: b.calls,
+        inputTokens: b.input,
+        outputTokens: b.output,
+        totalTokens: b.total,
+      }))
+      .sort((a, b) => b.totalTokens - a.totalTokens || a.key.localeCompare(b.key));
+  }
+
   /**
    * Create a child tracker whose recordings propagate to this parent.
    * Used per-evaluator so individual usage is readable while the parent
@@ -109,12 +217,15 @@ class ChildTracker extends TokenTracker {
     super();
   }
 
-  override record(usage?: {
-    inputTokens?: number;
-    outputTokens?: number;
-    totalTokens?: number;
-  }): void {
-    super.record(usage);
-    this.parent.record(usage);
+  override record(
+    usage?: {
+      inputTokens?: number;
+      outputTokens?: number;
+      totalTokens?: number;
+    },
+    attribution?: RecordAttribution
+  ): void {
+    super.record(usage, attribution);
+    this.parent.record(usage, attribution);
   }
 }