From a6f9d27b37d4cac0fe18861baae74922815688a7 Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Mon, 3 Aug 2026 16:42:47 +0530 Subject: [PATCH 1/6] feat(pricing): estimate run cost from per-model token usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reports tracked total tokens but had no way to turn that into a cost. A single combined total is unpriceable in principle: a run can use a different model for the judge than the attacker, and their rates differ by more than an order of magnitude. Two parts: 1. Attribute tokens to models. TokenTracker now keeps a per-model breakdown alongside the existing flat totals, which are unchanged. Built AI SDK models no longer carry the configured provider name (openai-compatible surfaces as "custom.chat"), so createModel records identity in a WeakMap that recording sites resolve from the model object they already hold — no new parameters through call chains. Run phase is derived from the context label withRetry already takes. 2. Price it. scripts/build-pricing.ts prunes LiteLLM's community price map to the providers opfor can reach and vendors it as an inlined module (557 entries, 43 KB, no Node imports, so the extension bundles it). Lookup tries a short ladder of model-name forms, then verifies the matched row belongs to the configured provider — without that guard a mistyped Groq model would silently price at OpenAI rates. Vendored rather than fetched at runtime: opfor runs in air-gapped CI, and a security report should produce the same numbers when re-run later. Unpriceable models are reported as such, never counted as free; the report and CLI both mark the total a lower bound and name what was missed. Estimates were validated against a LiteLLM proxy's own billed cost across three vendors (0.0% delta). Labelled "Testing cost" throughout — the target's own inference spend is not observable from opfor and is excluded. Repeat (cached) input tokens are not yet counted separately, so multi-turn runs against providers that discount repeated context are over-estimated. Left for a follow-up. Co-Authored-By: Claude Opus 5 --- core/package.json | 4 + core/src/browser.ts | 7 + core/src/evaluators/judge.ts | 2 +- core/src/execute/evaluatorLoop.ts | 13 +- core/src/execute/runAll.ts | 3 + core/src/execute/runAllBrowser.ts | 7 + core/src/execute/tokenTracker.ts | 129 +++- core/src/execute/types.ts | 13 + core/src/generate/generateAttacks.ts | 3 + core/src/generate/generateNextTurn.ts | 4 +- core/src/lib/llmRetry.ts | 23 +- core/src/llm/openaiCompatible.ts | 4 +- core/src/pricing/estimateCost.ts | 86 +++ core/src/pricing/lookupPrice.ts | 85 +++ core/src/pricing/priceTable.generated.ts | 749 +++++++++++++++++++++++ core/src/pricing/providerAliases.ts | 45 ++ core/src/pricing/types.ts | 60 ++ core/src/providers/factory.ts | 7 +- core/src/providers/modelIdentity.ts | 88 +++ core/src/report/buildReport.ts | 2 + core/src/report/render.ts | 40 +- core/src/report/types.ts | 13 + core/src/run/judge.ts | 1 + core/tests/modelIdentity.test.ts | 93 +++ core/tests/pricing.test.ts | 231 +++++++ core/tests/tokenTracker.test.ts | 106 ++++ package.json | 1 + runners/cli/src/commands/run.ts | 14 + scripts/build-pricing.ts | 211 +++++++ 29 files changed, 2026 insertions(+), 18 deletions(-) create mode 100644 core/src/pricing/estimateCost.ts create mode 100644 core/src/pricing/lookupPrice.ts create mode 100644 core/src/pricing/priceTable.generated.ts create mode 100644 core/src/pricing/providerAliases.ts create mode 100644 core/src/pricing/types.ts create mode 100644 core/src/providers/modelIdentity.ts create mode 100644 core/tests/modelIdentity.test.ts create mode 100644 core/tests/pricing.test.ts create mode 100644 scripts/build-pricing.ts diff --git a/core/package.json b/core/package.json index a370ed84..367942c8 100644 --- a/core/package.json +++ b/core/package.json @@ -43,6 +43,10 @@ "types": "./dist/providers/*.d.ts", "default": "./dist/providers/*.js" }, + "./pricing/*.js": { + "types": "./dist/pricing/*.d.ts", + "default": "./dist/pricing/*.js" + }, "./evaluators/*.js": { "types": "./dist/evaluators/*.d.ts", "default": "./dist/evaluators/*.js" diff --git a/core/src/browser.ts b/core/src/browser.ts index 61fa9d15..8281a17e 100644 --- a/core/src/browser.ts +++ b/core/src/browser.ts @@ -69,6 +69,13 @@ export { } from "./providers/factory.js"; export type { LlmConfig, ProviderName } from "./config/types.js"; +// Cost estimation. The vendored price table is a plain inlined module with no +// Node imports, so esbuild bundles it for the extension like any other source. +export { estimateRunCost, formatUsd } from "./pricing/estimateCost.js"; +export { lookupPrice, PRICE_TABLE_VERSION } from "./pricing/lookupPrice.js"; +export type { RunCost, ModelCost, ModelPrice, CostSource } from "./pricing/types.js"; +export type { ModelTokenUsage, TokenUsage } from "./execute/tokenTracker.js"; + export { getAdapter } from "./telemetry/adapter.js"; export { renderReport } from "./report/render.js"; diff --git a/core/src/evaluators/judge.ts b/core/src/evaluators/judge.ts index c55a5ffe..e1a42217 100644 --- a/core/src/evaluators/judge.ts +++ b/core/src/evaluators/judge.ts @@ -206,7 +206,7 @@ export async function judgeResponse( try { const result = await withRetry( () => generateText({ model, system: JUDGE_SYSTEM, prompt: judgePrompt }), - { context: "Judge", maxRetries: 3, tokenTracker } + { context: "Judge", maxRetries: 3, tokenTracker, model } ); return parseJudgeOutput(result.text); } catch (err) { diff --git a/core/src/execute/evaluatorLoop.ts b/core/src/execute/evaluatorLoop.ts index 9963bd1c..1de6b0b6 100644 --- a/core/src/execute/evaluatorLoop.ts +++ b/core/src/execute/evaluatorLoop.ts @@ -17,6 +17,7 @@ import { verdictIcon } from "../lib/verdictIcon.js"; import { TurnPlan } from "./turnPlan.js"; import { isStopError, getStopReason } from "../lib/llmRetry.js"; import type { TokenTracker } from "./tokenTracker.js"; +import { estimateRunCost } from "../pricing/estimateCost.js"; import { log } from "../lib/logger.js"; import type { RunConfig, @@ -226,7 +227,11 @@ export async function runEvaluatorAttacks( attackResults.push(makeFailedResult(stopReason)); notify({ type: "attack_done", attackId: attack.id, verdict: "ERROR" }); const partialResult = toEvaluatorResult(evaluatorMeta, attackResults); - if (evalTracker) partialResult.tokenUsage = evalTracker.totals; + if (evalTracker) { + partialResult.tokenUsage = evalTracker.totals; + partialResult.tokenUsageByModel = evalTracker.breakdown; + partialResult.cost = estimateRunCost(partialResult.tokenUsageByModel); + } evaluatorResults.push(partialResult); return { evaluatorResults, stopReason }; } @@ -243,7 +248,11 @@ export async function runEvaluatorAttacks( notify({ type: "evaluator_done", evaluatorId: evaluator.id, passed, failed, errors }); const evResult = toEvaluatorResult(evaluatorMeta, attackResults); - if (evalTracker) evResult.tokenUsage = evalTracker.totals; + if (evalTracker) { + evResult.tokenUsage = evalTracker.totals; + evResult.tokenUsageByModel = evalTracker.breakdown; + evResult.cost = estimateRunCost(evResult.tokenUsageByModel); + } evaluatorResults.push(evResult); sessionMap.set(evaluator.id, captureSessionContext(evaluator, attackResults)); } diff --git a/core/src/execute/runAll.ts b/core/src/execute/runAll.ts index 86f4fc32..b1a01735 100644 --- a/core/src/execute/runAll.ts +++ b/core/src/execute/runAll.ts @@ -17,6 +17,7 @@ import type { LlmConfig } from "../config/types.js"; import { getAdapter } from "../telemetry/adapter.js"; import { runSetupTraceCuration } from "../telemetry/curation.js"; import { TokenTracker } from "./tokenTracker.js"; +import { estimateRunCost } from "../pricing/estimateCost.js"; import { log } from "../lib/logger.js"; export interface RunAllOptions { @@ -144,6 +145,8 @@ export async function runAll( const usage = tokenTracker.totals; if (usage.totalTokens > 0) { report.summary.tokenUsage = usage; + report.summary.tokenUsageByModel = tokenTracker.breakdown; + report.summary.cost = estimateRunCost(report.summary.tokenUsageByModel); } report.summary.durationMs = Date.now() - runStartedAt; if (stopReason) { diff --git a/core/src/execute/runAllBrowser.ts b/core/src/execute/runAllBrowser.ts index 763d8376..bad15ce0 100644 --- a/core/src/execute/runAllBrowser.ts +++ b/core/src/execute/runAllBrowser.ts @@ -20,6 +20,7 @@ import { modelLabel, } from "./aggregate.js"; import { TokenTracker } from "./tokenTracker.js"; +import { estimateRunCost } from "../pricing/estimateCost.js"; import type { AgentAttackSpec, AttackResult, @@ -189,6 +190,8 @@ export async function runAllBrowser( attackResults ); partialResult.tokenUsage = evalTracker.totals; + partialResult.tokenUsageByModel = evalTracker.breakdown; + partialResult.cost = estimateRunCost(partialResult.tokenUsageByModel); evaluatorResults.push(partialResult); }; @@ -230,6 +233,8 @@ export async function runAllBrowser( attackResults ); evResult.tokenUsage = evalTracker.totals; + evResult.tokenUsageByModel = evalTracker.breakdown; + evResult.cost = estimateRunCost(evResult.tokenUsageByModel); evaluatorResults.push(evResult); } @@ -237,6 +242,8 @@ export async function runAllBrowser( const usage = tokenTracker.totals; if (usage.totalTokens > 0) { report.summary.tokenUsage = usage; + report.summary.tokenUsageByModel = tokenTracker.breakdown; + report.summary.cost = estimateRunCost(report.summary.tokenUsageByModel); } report.summary.durationMs = Date.now() - runStartedAt; return report; diff --git a/core/src/execute/tokenTracker.ts b/core/src/execute/tokenTracker.ts index 2bf2d01c..ed7ba6a7 100644 --- a/core/src/execute/tokenTracker.ts +++ b/core/src/execute/tokenTracker.ts @@ -5,9 +5,20 @@ * attack drivers. Each `generateText` / `generateObject` call site records its * usage after the call completes (including retries). Aggregated totals are * surfaced in the CLI summary, HTML/JSON report, and extension popup. + * + * Usage is recorded twice: once into flat run totals, and once into a per-model + * bucket. The per-model breakdown exists because a run can mix models — the + * judge may be a different (and far more expensive) model than the attacker — + * so a single combined total cannot be priced. See {@link ModelTokenUsage}. */ import { z } from "zod"; +import { + modelKey, + resolveModelIdentity, + UNKNOWN_MODEL_KEY, + type ModelRef, +} from "../providers/modelIdentity.js"; /** Aggregated input/output/total token counts from LLM calls. */ export interface TokenUsage { @@ -16,6 +27,26 @@ export interface TokenUsage { totalTokens: number; } +/** Token usage attributed to one provider/model pair. */ +export interface ModelTokenUsage extends TokenUsage { + /** `":"`, or `"unknown"` for usage that could not be attributed. */ + key: string; + provider: string; + model: string; + /** Which phases used this model — `"attacker"`, `"judge"`. Sorted, deduped. */ + roles: string[]; + /** Number of LLM calls recorded against this model. */ + calls: number; +} + +/** Optional provenance supplied alongside a usage recording. */ +export interface RecordAttribution { + /** The model the call was made against — an AI SDK model, an LlmConfig, or an identity. */ + model?: ModelRef; + /** Run phase, e.g. `"attacker"` or `"judge"`. Case-insensitive. */ + role?: string; +} + /** Shared zero-value constant to avoid re-allocating empty usage objects. */ export const ZERO_USAGE: TokenUsage = Object.freeze({ inputTokens: 0, @@ -53,6 +84,18 @@ export function parseUsage(raw: unknown): TokenUsage | undefined { return result.success ? result.data : undefined; } +/** Mutable per-model accumulator; projected to {@link ModelTokenUsage} on read. */ +interface ModelBucket { + key: string; + provider: string; + model: string; + roles: Set; + calls: number; + input: number; + output: number; + total: number; +} + /** * Accumulator for LLM token usage. * @@ -64,20 +107,65 @@ export class TokenTracker { private input = 0; private output = 0; private total = 0; + private readonly buckets = new Map(); /** * Record usage from a single LLM call. Safe to call with undefined/partial * usage. When `totalTokens` is supplied it is preserved; otherwise it falls * back to `inputTokens + outputTokens`. + * + * `attribution` is optional — usage recorded without it lands in the + * `"unknown"` bucket and still counts toward run totals, so an un-migrated + * call site degrades to today's behavior rather than losing tokens. */ - record(usage?: { inputTokens?: number; outputTokens?: number; totalTokens?: number }): void { + record( + usage?: { inputTokens?: number; outputTokens?: number; totalTokens?: number }, + attribution?: RecordAttribution + ): void { if (!usage) return; const inp = usage.inputTokens ?? 0; const out = usage.outputTokens ?? 0; const tot = usage.totalTokens ?? 0; + const resolvedTotal = tot > 0 ? tot : inp + out; + this.input += inp; this.output += out; - this.total += tot > 0 ? tot : inp + out; + this.total += resolvedTotal; + + this.recordToBucket(inp, out, resolvedTotal, attribution); + } + + /** Fold one call's usage into its per-model bucket, creating the bucket on first sight. */ + private recordToBucket( + inp: number, + out: number, + total: number, + attribution?: RecordAttribution + ): void { + const identity = resolveModelIdentity(attribution?.model); + const key = identity ? modelKey(identity) : UNKNOWN_MODEL_KEY; + + let bucket = this.buckets.get(key); + if (!bucket) { + bucket = { + key, + provider: identity?.provider ?? "unknown", + model: identity?.model ?? "unknown", + roles: new Set(), + calls: 0, + input: 0, + output: 0, + total: 0, + }; + this.buckets.set(key, bucket); + } + + const role = attribution?.role?.trim().toLowerCase(); + if (role) bucket.roles.add(role); + bucket.calls += 1; + bucket.input += inp; + bucket.output += out; + bucket.total += total; } /** Current accumulated totals. Uses the provider-supplied total when available. */ @@ -89,6 +177,26 @@ export class TokenTracker { }; } + /** + * Per-model usage, heaviest first. Always sums to {@link totals}; models that + * could not be attributed appear under the `"unknown"` key rather than being + * dropped. + */ + get breakdown(): ModelTokenUsage[] { + return [...this.buckets.values()] + .map((b) => ({ + key: b.key, + provider: b.provider, + model: b.model, + roles: [...b.roles].sort(), + calls: b.calls, + inputTokens: b.input, + outputTokens: b.output, + totalTokens: b.total, + })) + .sort((a, b) => b.totalTokens - a.totalTokens || a.key.localeCompare(b.key)); + } + /** * Create a child tracker whose recordings propagate to this parent. * Used per-evaluator so individual usage is readable while the parent @@ -109,12 +217,15 @@ class ChildTracker extends TokenTracker { super(); } - override record(usage?: { - inputTokens?: number; - outputTokens?: number; - totalTokens?: number; - }): void { - super.record(usage); - this.parent.record(usage); + override record( + usage?: { + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + }, + attribution?: RecordAttribution + ): void { + super.record(usage, attribution); + this.parent.record(usage, attribution); } } diff --git a/core/src/execute/types.ts b/core/src/execute/types.ts index 1a84ea62..e9980373 100644 --- a/core/src/execute/types.ts +++ b/core/src/execute/types.ts @@ -1,5 +1,7 @@ import type { LlmConfig, TelemetryConfig } from "../config/types.js"; import type { JudgeResult } from "../run/types.js"; +import type { ModelTokenUsage } from "./tokenTracker.js"; +import type { RunCost } from "../pricing/types.js"; export type Effort = "adaptive" | "comprehensive"; @@ -249,6 +251,10 @@ export interface EvaluatorResult { passRate: number; attacks: AttackResult[]; tokenUsage?: { inputTokens: number; outputTokens: number; totalTokens: number }; + /** Same tokens as `tokenUsage`, split by the model that spent them. */ + tokenUsageByModel?: ModelTokenUsage[]; + /** Estimated USD cost of this evaluator's attacker + judge calls. */ + cost?: RunCost; } export interface UnifiedRunReport { @@ -273,6 +279,13 @@ export interface UnifiedRunReport { safetyScore: number; attackSuccessRate: number; tokenUsage?: { inputTokens: number; outputTokens: number; totalTokens: number }; + /** Same tokens as `tokenUsage`, split by the model that spent them. */ + tokenUsageByModel?: ModelTokenUsage[]; + /** + * Estimated USD cost of the attacker + judge LLM calls. Excludes the + * target's own inference cost, which opfor cannot observe. + */ + cost?: RunCost; /** Wall-clock time for the whole run, from the first evaluator to the last. */ durationMs?: number; }; diff --git a/core/src/generate/generateAttacks.ts b/core/src/generate/generateAttacks.ts index d7b5976f..d4768670 100644 --- a/core/src/generate/generateAttacks.ts +++ b/core/src/generate/generateAttacks.ts @@ -183,6 +183,7 @@ async function generatePatternAgentAttack( context: "Attacker", maxRetries: 3, tokenTracker, + model, }); return result.text.trim(); } catch (err) { @@ -387,6 +388,7 @@ async function generateSingleMcpAttack( context: "Attacker (MCP)", maxRetries: 3, tokenTracker, + model, }); return parseMcpAttackJson(result.text, tools[0].name); } catch (err) { @@ -437,6 +439,7 @@ async function generatePatternMcpAttack( context: "Attacker (MCP)", maxRetries: 3, tokenTracker, + model, }); return parseMcpAttackJson(result.text, tools[0].name); } catch (err) { diff --git a/core/src/generate/generateNextTurn.ts b/core/src/generate/generateNextTurn.ts index df8a0f73..e11c1c19 100644 --- a/core/src/generate/generateNextTurn.ts +++ b/core/src/generate/generateNextTurn.ts @@ -194,7 +194,7 @@ export async function generateNextAdaptiveTurn(params: { const result = await generateText({ model, system, prompt: userBlock }); const usage1 = parseUsage(result.usage); - if (usage1) params.tokenTracker?.record(usage1); + if (usage1) params.tokenTracker?.record(usage1, { model, role: "attacker" }); const parsed = parseAttackerOutput(result.text); if (!parsed.message) throw new Error("generateNextAdaptiveTurn: empty model response"); const message = @@ -343,7 +343,7 @@ export async function generateNextMcpTurn( const result = await generateText({ model, system, prompt: user }); const usage2 = parseUsage(result.usage); - if (usage2) tokenTracker?.record(usage2); + if (usage2) tokenTracker?.record(usage2, { model, role: "attacker" }); try { const cleaned = result.text diff --git a/core/src/lib/llmRetry.ts b/core/src/lib/llmRetry.ts index 680ac7ae..90430477 100644 --- a/core/src/lib/llmRetry.ts +++ b/core/src/lib/llmRetry.ts @@ -6,6 +6,7 @@ import { log } from "./logger.js"; import type { TokenTracker } from "../execute/tokenTracker.js"; import { parseUsage } from "../execute/tokenTracker.js"; +import type { ModelRef } from "../providers/modelIdentity.js"; export interface LlmError { isRetryable: boolean; @@ -127,6 +128,25 @@ export interface RetryOptions { context?: string; // e.g., "attacker", "judge" for logging /** When set, usage from a successful result's `.usage` field is auto-recorded. */ tokenTracker?: TokenTracker; + /** + * The model the wrapped call runs against. Used only to attribute recorded + * token usage — without it the usage still counts, but lands in the + * unattributed bucket and cannot be priced. + */ + model?: ModelRef; +} + +/** + * Derive a run phase from the free-text `context` label already passed for + * logging (`"Attacker"`, `"Attacker (MCP)"`, `"Judge"`), so call sites don't + * have to repeat themselves. Unrecognized labels yield no role. + */ +export function roleFromContext(context?: string): string | undefined { + const c = context?.trim().toLowerCase(); + if (!c) return undefined; + if (c.startsWith("attacker")) return "attacker"; + if (c.startsWith("judge")) return "judge"; + return undefined; } /** @@ -140,6 +160,7 @@ export async function withRetry(fn: () => Promise, options: RetryOptions = maxDelayMs = 30000, context = "LLM", tokenTracker, + model, } = options; let lastError: LlmError | null = null; @@ -149,7 +170,7 @@ export async function withRetry(fn: () => Promise, options: RetryOptions = const result = await fn(); if (tokenTracker && result && typeof result === "object" && "usage" in result) { const validated = parseUsage((result as { usage?: unknown }).usage); - if (validated) tokenTracker.record(validated); + if (validated) tokenTracker.record(validated, { model, role: roleFromContext(context) }); } return result; } catch (err) { diff --git a/core/src/llm/openaiCompatible.ts b/core/src/llm/openaiCompatible.ts index 484c099e..32f178ee 100644 --- a/core/src/llm/openaiCompatible.ts +++ b/core/src/llm/openaiCompatible.ts @@ -108,6 +108,8 @@ export async function chatCompletionJsonContent(args: { user: string; isAcceptableJson?: (json: string) => boolean; tokenTracker?: TokenTracker; + /** Run phase this call belongs to, used to attribute recorded token usage. */ + role?: string; }): Promise { const apiKey = resolveApiKey(args.model); if (!apiKey) { @@ -210,7 +212,7 @@ export async function chatCompletionJsonContent(args: { outputTokens: data.usage.completion_tokens ?? 0, totalTokens: data.usage.total_tokens ?? 0, }); - if (validated) args.tokenTracker.record(validated); + if (validated) args.tokenTracker.record(validated, { model: args.model, role: args.role }); } const content = data.choices?.[0]?.message?.content; if (typeof content !== "string" || !content.trim()) { diff --git a/core/src/pricing/estimateCost.ts b/core/src/pricing/estimateCost.ts new file mode 100644 index 00000000..eb8eadb4 --- /dev/null +++ b/core/src/pricing/estimateCost.ts @@ -0,0 +1,86 @@ +/** + * Turn a per-model token breakdown into a run cost. + * + * Deliberately conservative: a model whose price cannot be found is reported as + * unpriced rather than counted as free. `totalUsd` is therefore a lower bound + * whenever `complete` is false, and the UI must say so — a report that quietly + * treats an unknown model as $0 understates spend while looking authoritative. + * + * Repeat ("cached") input tokens are not yet separated by the token counter, so + * all input is priced at the full input rate. For multi-turn runs against + * providers that discount repeated context, that makes this an over-estimate. + */ + +import type { ModelTokenUsage } from "../execute/tokenTracker.js"; +import { lookupPrice, PRICE_TABLE_VERSION } from "./lookupPrice.js"; +import type { ModelCost, RunCost } from "./types.js"; + +/** Price one model's usage. */ +function costOne(usage: ModelTokenUsage): ModelCost { + const found = lookupPrice(usage.provider, usage.model); + + const base: ModelCost = { + key: usage.key, + provider: usage.provider, + model: usage.model, + roles: usage.roles, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + totalTokens: usage.totalTokens, + source: "unknown", + }; + + if (!found) return base; + + return { + ...base, + usd: + usage.inputTokens * found.price.inputPerToken + + usage.outputTokens * found.price.outputPerToken, + source: "table", + matchedKey: found.matchedKey, + }; +} + +/** + * Estimate the cost of a run from its per-model token breakdown. + * + * Returns undefined when there is nothing to price, so callers can omit the + * field entirely rather than render a meaningless $0.00. + */ +export function estimateRunCost(breakdown?: ModelTokenUsage[]): RunCost | undefined { + const spending = (breakdown ?? []).filter((b) => b.totalTokens > 0); + if (spending.length === 0) return undefined; + + const byModel = spending.map(costOne); + const unpricedModels = byModel.filter((c) => c.usd === undefined).map((c) => c.key); + const totalUsd = byModel.reduce((sum, c) => sum + (c.usd ?? 0), 0); + + return { + totalUsd, + currency: "USD", + byModel, + unpricedModels, + complete: unpricedModels.length === 0, + priceTableVersion: PRICE_TABLE_VERSION, + }; +} + +/** + * Format a USD amount for display. + * + * Fractions of a cent are the norm here — a smoke suite on a cheap model lands + * around $0.00003 — and a fixed number of decimals renders those as "$0.0000", + * which reads as free. So small amounts are shown to two significant figures + * instead, and anything below a millionth of a dollar is labelled as such rather + * than rounded away. + */ +export function formatUsd(usd: number): string { + if (usd === 0) return "$0.00"; + if (usd < 0.000001) return "<$0.000001"; + // Number() strips the trailing zeros toPrecision leaves behind (0.0034 stays + // "0.0034" rather than becoming "0.0034000"). + if (usd < 0.01) return `$${Number(usd.toPrecision(2))}`; + if (usd < 1) return `$${usd.toFixed(3)}`; + return `$${usd.toFixed(2)}`; +} diff --git a/core/src/pricing/lookupPrice.ts b/core/src/pricing/lookupPrice.ts new file mode 100644 index 00000000..562bc797 --- /dev/null +++ b/core/src/pricing/lookupPrice.ts @@ -0,0 +1,85 @@ +/** + * Resolve a configured provider/model pair to per-token prices. + * + * The same model is listed upstream under several names depending on how it is + * reached (`claude-opus-5`, `vertex_ai/claude-opus-5`, `anthropic/claude-opus-5`), + * while users type whatever their setup expects. So rather than one lookup, we + * try a short ladder of name forms and take the first that matches. + * + * Every match is then checked against the configured provider. Without that + * guard, someone running a mistyped model name on Groq would silently match + * OpenAI's row and be priced at OpenAI's rates — a wrong number presented with + * full confidence, which is worse than reporting nothing. + */ + +import { PRICE_TABLE, type CompactPrice } from "./priceTable.generated.js"; +import { LITELLM_PROVIDER_ALIASES } from "./providerAliases.js"; +import type { ModelPrice } from "./types.js"; + +/** A resolved price plus the table key it came from, for auditability. */ +export interface PriceLookupResult { + price: ModelPrice; + matchedKey: string; +} + +/** + * Name forms to try, in order. + * + * The provider-scoped form goes first because some vendors resell the same model + * at a different rate — `azure/gpt-4o-mini` costs more than OpenAI's own + * `gpt-4o-mini`. Trying it first picks the right row outright instead of relying + * on the guard below to reject the wrong one. Most providers have no such row, + * in which case the lookup simply falls through to the bare name. + */ +export function priceCandidates(provider: string, model: string): string[] { + const m = model.trim(); + if (!m) return []; + const lower = m.toLowerCase(); + + const forms = [`${provider}/${m}`, m]; + if (m.includes("/")) { + // "anthropic/claude-opus-5" -> "claude-opus-5"; upstream lists direct-API + // models bare, so a prefixed name only matches once the prefix is dropped. + forms.push(m.slice(m.indexOf("/") + 1)); + forms.push(m.slice(m.lastIndexOf("/") + 1)); + } + if (lower !== m) forms.push(lower); + + return [...new Set(forms.filter(Boolean))]; +} + +/** Whether a table row may legitimately price this provider. */ +function providerAccepts(provider: string, entry: CompactPrice): boolean { + const accepted = LITELLM_PROVIDER_ALIASES[provider]; + // Unknown to the alias map, or explicitly unverifiable (an OpenAI-compatible + // proxy can serve any vendor) — nothing to check against, so accept the match. + if (accepted === undefined || accepted === null) return true; + return accepted.includes(entry.p); +} + +function toModelPrice(entry: CompactPrice): ModelPrice { + return { + inputPerToken: entry.i, + outputPerToken: entry.o, + ...(entry.cr !== undefined ? { cacheReadPerToken: entry.cr } : {}), + ...(entry.cw !== undefined ? { cacheWritePerToken: entry.cw } : {}), + }; +} + +/** + * Look up prices for one provider/model pair. + * + * Returns undefined when no name form matches, or when every match belongs to a + * different vendor. Callers must treat that as "unpriced", never as free. + */ +export function lookupPrice(provider: string, model: string): PriceLookupResult | undefined { + for (const key of priceCandidates(provider, model)) { + const entry = PRICE_TABLE[key]; + if (!entry) continue; + if (!providerAccepts(provider, entry)) continue; + return { price: toModelPrice(entry), matchedKey: key }; + } + return undefined; +} + +export { PRICE_TABLE_VERSION } from "./priceTable.generated.js"; diff --git a/core/src/pricing/priceTable.generated.ts b/core/src/pricing/priceTable.generated.ts new file mode 100644 index 00000000..d195c0e4 --- /dev/null +++ b/core/src/pricing/priceTable.generated.ts @@ -0,0 +1,749 @@ +/** + * GENERATED FILE — do not edit by hand. + * + * Model prices in USD per token, pruned from LiteLLM's community price map: + * https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json + * + * Regenerate with: npm run build:pricing + * + * Entries: 557 + * + * Contains no Node imports so the browser extension can bundle it. + */ + +/** Compact price row. Short keys keep the generated table small. */ +export interface CompactPrice { + /** Upstream `litellm_provider` — used to reject a match from the wrong vendor. */ + p: string; + /** USD per input token. */ + i: number; + /** USD per output token. */ + o: number; + /** USD per cached input token, when published. */ + cr?: number; + /** USD per cache-write token, when published. */ + cw?: number; +} + +/** Identifies this snapshot; recorded in reports so a cost figure is reproducible. */ +export const PRICE_TABLE_VERSION = "litellm-0933d4a0129c"; + +export const PRICE_TABLE: Record = { + "azure/codex-mini": { p: "azure", i: 0.0000015, o: 0.000006, cr: 3.75e-7 }, + "azure/command-r-plus": { p: "azure", i: 0.000003, o: 0.000015 }, + "azure/computer-use-preview": { p: "azure", i: 0.000003, o: 0.000012 }, + "azure/eu/gpt-4o-2024-08-06": { p: "azure", i: 0.00000275, o: 0.000011, cr: 0.000001375 }, + "azure/eu/gpt-4o-2024-11-20": { p: "azure", i: 0.00000275, o: 0.000011, cw: 0.00000138 }, + "azure/eu/gpt-4o-mini-2024-07-18": { p: "azure", i: 1.65e-7, o: 6.6e-7, cr: 8.3e-8 }, + "azure/eu/gpt-5-2025-08-07": { p: "azure", i: 0.000001375, o: 0.000011, cr: 1.375e-7 }, + "azure/eu/gpt-5-mini-2025-08-07": { p: "azure", i: 2.75e-7, o: 0.0000022, cr: 2.75e-8 }, + "azure/eu/gpt-5-nano-2025-08-07": { p: "azure", i: 5.5e-8, o: 4.4e-7, cr: 5.5e-9 }, + "azure/eu/gpt-5.1": { p: "azure", i: 0.00000138, o: 0.000011, cr: 1.4e-7 }, + "azure/eu/gpt-5.1-chat": { p: "azure", i: 0.00000138, o: 0.000011, cr: 1.4e-7 }, + "azure/eu/gpt-5.1-codex": { p: "azure", i: 0.00000138, o: 0.000011, cr: 1.4e-7 }, + "azure/eu/gpt-5.1-codex-mini": { p: "azure", i: 2.75e-7, o: 0.0000022, cr: 2.8e-8 }, + "azure/eu/gpt-5.4": { p: "azure", i: 0.00000275, o: 0.0000165, cr: 2.8e-7 }, + "azure/eu/gpt-5.4-2026-03-05": { p: "azure", i: 0.00000275, o: 0.0000165, cr: 2.8e-7 }, + "azure/eu/gpt-5.5": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/eu/gpt-5.5-2026-04-23": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/eu/gpt-5.6": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/eu/gpt-5.6-luna": { p: "azure", i: 0.0000011, o: 0.0000066, cr: 1.1e-7 }, + "azure/eu/gpt-5.6-sol": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/eu/gpt-5.6-terra": { p: "azure", i: 0.00000275, o: 0.0000165, cr: 2.75e-7 }, + "azure/eu/o1-2024-12-17": { p: "azure", i: 0.0000165, o: 0.000066, cr: 0.00000825 }, + "azure/eu/o1-mini-2024-09-12": { p: "azure", i: 0.00000121, o: 0.00000484, cr: 6.05e-7 }, + "azure/eu/o1-preview-2024-09-12": { p: "azure", i: 0.0000165, o: 0.000066, cr: 0.00000825 }, + "azure/eu/o3-mini-2025-01-31": { p: "azure", i: 0.00000121, o: 0.00000484, cr: 6.05e-7 }, + "azure/global-standard/gpt-4o-2024-08-06": { + p: "azure", + i: 0.0000025, + o: 0.00001, + cr: 0.00000125, + }, + "azure/global-standard/gpt-4o-2024-11-20": { + p: "azure", + i: 0.0000025, + o: 0.00001, + cr: 0.00000125, + }, + "azure/global-standard/gpt-4o-mini": { p: "azure", i: 1.5e-7, o: 6e-7 }, + "azure/global/gpt-4o-2024-08-06": { p: "azure", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "azure/global/gpt-4o-2024-11-20": { p: "azure", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "azure/global/gpt-5.1": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/global/gpt-5.1-chat": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/global/gpt-5.1-codex": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/global/gpt-5.1-codex-mini": { p: "azure", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "azure/gpt-3.5-turbo": { p: "azure", i: 5e-7, o: 0.0000015 }, + "azure/gpt-3.5-turbo-0125": { p: "azure", i: 5e-7, o: 0.0000015 }, + "azure/gpt-35-turbo": { p: "azure", i: 5e-7, o: 0.0000015 }, + "azure/gpt-35-turbo-0125": { p: "azure", i: 5e-7, o: 0.0000015 }, + "azure/gpt-35-turbo-1106": { p: "azure", i: 0.000001, o: 0.000002 }, + "azure/gpt-35-turbo-16k": { p: "azure", i: 0.000003, o: 0.000004 }, + "azure/gpt-35-turbo-16k-0613": { p: "azure", i: 0.000003, o: 0.000004 }, + "azure/gpt-4": { p: "azure", i: 0.00003, o: 0.00006 }, + "azure/gpt-4-0125-preview": { p: "azure", i: 0.00001, o: 0.00003 }, + "azure/gpt-4-0613": { p: "azure", i: 0.00003, o: 0.00006 }, + "azure/gpt-4-1106-preview": { p: "azure", i: 0.00001, o: 0.00003 }, + "azure/gpt-4-32k": { p: "azure", i: 0.00006, o: 0.00012 }, + "azure/gpt-4-32k-0613": { p: "azure", i: 0.00006, o: 0.00012 }, + "azure/gpt-4-turbo": { p: "azure", i: 0.00001, o: 0.00003 }, + "azure/gpt-4-turbo-2024-04-09": { p: "azure", i: 0.00001, o: 0.00003 }, + "azure/gpt-4-turbo-vision-preview": { p: "azure", i: 0.00001, o: 0.00003 }, + "azure/gpt-4.1": { p: "azure", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "azure/gpt-4.1-2025-04-14": { p: "azure", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "azure/gpt-4.1-mini": { p: "azure", i: 4e-7, o: 0.0000016, cr: 1e-7 }, + "azure/gpt-4.1-mini-2025-04-14": { p: "azure", i: 4e-7, o: 0.0000016, cr: 1e-7 }, + "azure/gpt-4.1-nano": { p: "azure", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "azure/gpt-4.1-nano-2025-04-14": { p: "azure", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "azure/gpt-4.5-preview": { p: "azure", i: 0.000075, o: 0.00015, cr: 0.0000375 }, + "azure/gpt-4o": { p: "azure", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "azure/gpt-4o-2024-05-13": { p: "azure", i: 0.000005, o: 0.000015 }, + "azure/gpt-4o-2024-08-06": { p: "azure", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "azure/gpt-4o-2024-11-20": { p: "azure", i: 0.00000275, o: 0.000011, cr: 0.00000125 }, + "azure/gpt-4o-audio-preview-2024-12-17": { p: "azure", i: 0.0000025, o: 0.00001 }, + "azure/gpt-4o-mini": { p: "azure", i: 1.65e-7, o: 6.6e-7, cr: 7.5e-8 }, + "azure/gpt-4o-mini-2024-07-18": { p: "azure", i: 1.65e-7, o: 6.6e-7, cr: 7.5e-8 }, + "azure/gpt-4o-mini-audio-preview-2024-12-17": { p: "azure", i: 0.0000025, o: 0.00001 }, + "azure/gpt-5": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5-2025-08-07": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5-chat": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5-chat-latest": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5-codex": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5-mini": { p: "azure", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "azure/gpt-5-mini-2025-08-07": { p: "azure", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "azure/gpt-5-nano": { p: "azure", i: 5e-8, o: 4e-7, cr: 5e-9 }, + "azure/gpt-5-nano-2025-08-07": { p: "azure", i: 5e-8, o: 4e-7, cr: 5e-9 }, + "azure/gpt-5-pro": { p: "azure", i: 0.000015, o: 0.00012 }, + "azure/gpt-5.1": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-2025-11-13": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-chat": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-chat-2025-11-13": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-codex": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-codex-2025-11-13": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-codex-max": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-codex-mini": { p: "azure", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "azure/gpt-5.1-codex-mini-2025-11-13": { p: "azure", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "azure/gpt-5.2": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.2-2025-12-11": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.2-chat": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.2-chat-2025-12-11": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.2-codex": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.2-pro": { p: "azure", i: 0.000021, o: 0.000168 }, + "azure/gpt-5.2-pro-2025-12-11": { p: "azure", i: 0.000021, o: 0.000168 }, + "azure/gpt-5.3-chat": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.3-codex": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.4": { p: "azure", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "azure/gpt-5.4-2026-03-05": { p: "azure", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "azure/gpt-5.4-mini": { p: "azure", i: 7.5e-7, o: 0.0000045, cr: 7.5e-8 }, + "azure/gpt-5.4-mini-2026-03-17": { p: "azure", i: 7.5e-7, o: 0.0000045, cr: 7.5e-8 }, + "azure/gpt-5.4-nano": { p: "azure", i: 2e-7, o: 0.00000125, cr: 2e-8 }, + "azure/gpt-5.4-nano-2026-03-17": { p: "azure", i: 2e-7, o: 0.00000125, cr: 2e-8 }, + "azure/gpt-5.4-pro": { p: "azure", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "azure/gpt-5.4-pro-2026-03-05": { p: "azure", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "azure/gpt-5.5": { p: "azure", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "azure/gpt-5.5-2026-04-23": { p: "azure", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "azure/gpt-5.5-pro": { p: "azure", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "azure/gpt-5.5-pro-2026-04-23": { p: "azure", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "azure/gpt-5.6": { p: "azure", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "azure/gpt-5.6-luna": { p: "azure", i: 0.000001, o: 0.000006, cr: 1e-7 }, + "azure/gpt-5.6-sol": { p: "azure", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "azure/gpt-5.6-terra": { p: "azure", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "azure/gpt-audio-1.5-2026-02-23": { p: "azure", i: 0.0000025, o: 0.00001 }, + "azure/gpt-audio-2025-08-28": { p: "azure", i: 0.0000025, o: 0.00001 }, + "azure/gpt-audio-mini-2025-10-06": { p: "azure", i: 6e-7, o: 0.0000024 }, + "azure/mistral-large-2402": { p: "azure", i: 0.000008, o: 0.000024 }, + "azure/mistral-large-latest": { p: "azure", i: 0.000008, o: 0.000024 }, + "azure/o1": { p: "azure", i: 0.000015, o: 0.00006, cr: 0.0000075 }, + "azure/o1-2024-12-17": { p: "azure", i: 0.000015, o: 0.00006, cr: 0.0000075 }, + "azure/o1-mini": { p: "azure", i: 0.00000121, o: 0.00000484, cr: 6.05e-7 }, + "azure/o1-mini-2024-09-12": { p: "azure", i: 0.0000011, o: 0.0000044, cr: 5.5e-7 }, + "azure/o1-preview": { p: "azure", i: 0.000015, o: 0.00006, cr: 0.0000075 }, + "azure/o1-preview-2024-09-12": { p: "azure", i: 0.000015, o: 0.00006, cr: 0.0000075 }, + "azure/o3": { p: "azure", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "azure/o3-2025-04-16": { p: "azure", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "azure/o3-deep-research": { p: "azure", i: 0.00001, o: 0.00004, cr: 0.0000025 }, + "azure/o3-mini": { p: "azure", i: 0.0000011, o: 0.0000044, cr: 5.5e-7 }, + "azure/o3-mini-2025-01-31": { p: "azure", i: 0.0000011, o: 0.0000044, cr: 5.5e-7 }, + "azure/o3-pro": { p: "azure", i: 0.00002, o: 0.00008 }, + "azure/o3-pro-2025-06-10": { p: "azure", i: 0.00002, o: 0.00008 }, + "azure/o4-mini": { p: "azure", i: 0.0000011, o: 0.0000044, cr: 2.75e-7 }, + "azure/o4-mini-2025-04-16": { p: "azure", i: 0.0000011, o: 0.0000044, cr: 2.75e-7 }, + "azure/us/gpt-4.1-2025-04-14": { p: "azure", i: 0.0000022, o: 0.0000088, cr: 5.5e-7 }, + "azure/us/gpt-4.1-mini-2025-04-14": { p: "azure", i: 4.4e-7, o: 0.00000176, cr: 1.1e-7 }, + "azure/us/gpt-4.1-nano-2025-04-14": { p: "azure", i: 1.1e-7, o: 4.4e-7, cr: 2.5e-8 }, + "azure/us/gpt-4o-2024-08-06": { p: "azure", i: 0.00000275, o: 0.000011, cr: 0.000001375 }, + "azure/us/gpt-4o-2024-11-20": { p: "azure", i: 0.00000275, o: 0.000011, cw: 0.00000138 }, + "azure/us/gpt-4o-mini-2024-07-18": { p: "azure", i: 1.65e-7, o: 6.6e-7, cr: 8.3e-8 }, + "azure/us/gpt-5-2025-08-07": { p: "azure", i: 0.000001375, o: 0.000011, cr: 1.375e-7 }, + "azure/us/gpt-5-mini-2025-08-07": { p: "azure", i: 2.75e-7, o: 0.0000022, cr: 2.75e-8 }, + "azure/us/gpt-5-nano-2025-08-07": { p: "azure", i: 5.5e-8, o: 4.4e-7, cr: 5.5e-9 }, + "azure/us/gpt-5.1": { p: "azure", i: 0.00000138, o: 0.000011, cr: 1.4e-7 }, + "azure/us/gpt-5.1-chat": { p: "azure", i: 0.00000138, o: 0.000011, cr: 1.4e-7 }, + "azure/us/gpt-5.1-codex": { p: "azure", i: 0.00000138, o: 0.000011, cr: 1.4e-7 }, + "azure/us/gpt-5.1-codex-mini": { p: "azure", i: 2.75e-7, o: 0.0000022, cr: 2.8e-8 }, + "azure/us/gpt-5.4": { p: "azure", i: 0.00000275, o: 0.0000165, cr: 2.8e-7 }, + "azure/us/gpt-5.4-2026-03-05": { p: "azure", i: 0.00000275, o: 0.0000165, cr: 2.8e-7 }, + "azure/us/gpt-5.5": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/us/gpt-5.5-2026-04-23": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/us/gpt-5.6": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/us/gpt-5.6-luna": { p: "azure", i: 0.0000011, o: 0.0000066, cr: 1.1e-7 }, + "azure/us/gpt-5.6-sol": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/us/gpt-5.6-terra": { p: "azure", i: 0.00000275, o: 0.0000165, cr: 2.75e-7 }, + "azure/us/o1-2024-12-17": { p: "azure", i: 0.0000165, o: 0.000066, cr: 0.00000825 }, + "azure/us/o1-mini-2024-09-12": { p: "azure", i: 0.00000121, o: 0.00000484, cr: 6.05e-7 }, + "azure/us/o1-preview-2024-09-12": { p: "azure", i: 0.0000165, o: 0.000066, cr: 0.00000825 }, + "azure/us/o3-2025-04-16": { p: "azure", i: 0.0000022, o: 0.0000088, cr: 5.5e-7 }, + "azure/us/o3-mini-2025-01-31": { p: "azure", i: 0.00000121, o: 0.00000484, cr: 6.05e-7 }, + "azure/us/o4-mini-2025-04-16": { p: "azure", i: 0.00000121, o: 0.00000484, cr: 3.1e-7 }, + "azure_ai/Llama-3.2-11B-Vision-Instruct": { p: "azure_ai", i: 3.7e-7, o: 3.7e-7 }, + "azure_ai/Llama-3.2-90B-Vision-Instruct": { p: "azure_ai", i: 0.00000204, o: 0.00000204 }, + "azure_ai/Llama-3.3-70B-Instruct": { p: "azure_ai", i: 7.1e-7, o: 7.1e-7 }, + "azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8": { p: "azure_ai", i: 0.00000141, o: 3.5e-7 }, + "azure_ai/Llama-4-Scout-17B-16E-Instruct": { p: "azure_ai", i: 2e-7, o: 7.8e-7 }, + "azure_ai/MAI-DS-R1": { p: "azure_ai", i: 0.00000135, o: 0.0000054 }, + "azure_ai/Meta-Llama-3-70B-Instruct": { p: "azure_ai", i: 0.0000011, o: 3.7e-7 }, + "azure_ai/Meta-Llama-3.1-405B-Instruct": { p: "azure_ai", i: 0.00000533, o: 0.000016 }, + "azure_ai/Meta-Llama-3.1-70B-Instruct": { p: "azure_ai", i: 0.00000268, o: 0.00000354 }, + "azure_ai/Meta-Llama-3.1-8B-Instruct": { p: "azure_ai", i: 3e-7, o: 6.1e-7 }, + "azure_ai/Phi-3-medium-128k-instruct": { p: "azure_ai", i: 1.7e-7, o: 6.8e-7 }, + "azure_ai/Phi-3-medium-4k-instruct": { p: "azure_ai", i: 1.7e-7, o: 6.8e-7 }, + "azure_ai/Phi-3-mini-128k-instruct": { p: "azure_ai", i: 1.3e-7, o: 5.2e-7 }, + "azure_ai/Phi-3-mini-4k-instruct": { p: "azure_ai", i: 1.3e-7, o: 5.2e-7 }, + "azure_ai/Phi-3-small-128k-instruct": { p: "azure_ai", i: 1.5e-7, o: 6e-7 }, + "azure_ai/Phi-3-small-8k-instruct": { p: "azure_ai", i: 1.5e-7, o: 6e-7 }, + "azure_ai/Phi-3.5-MoE-instruct": { p: "azure_ai", i: 1.6e-7, o: 6.4e-7 }, + "azure_ai/Phi-3.5-mini-instruct": { p: "azure_ai", i: 1.3e-7, o: 5.2e-7 }, + "azure_ai/Phi-3.5-vision-instruct": { p: "azure_ai", i: 1.3e-7, o: 5.2e-7 }, + "azure_ai/Phi-4": { p: "azure_ai", i: 1.25e-7, o: 5e-7 }, + "azure_ai/Phi-4-mini-instruct": { p: "azure_ai", i: 7.5e-8, o: 3e-7 }, + "azure_ai/Phi-4-mini-reasoning": { p: "azure_ai", i: 8e-8, o: 3.2e-7 }, + "azure_ai/Phi-4-multimodal-instruct": { p: "azure_ai", i: 8e-8, o: 3.2e-7 }, + "azure_ai/Phi-4-reasoning": { p: "azure_ai", i: 1.25e-7, o: 5e-7 }, + "azure_ai/claude-fable-5": { p: "azure_ai", i: 0.00001, o: 0.00005, cr: 0.000001, cw: 0.0000125 }, + "azure_ai/claude-haiku-4-5": { + p: "azure_ai", + i: 0.000001, + o: 0.000005, + cr: 1e-7, + cw: 0.00000125, + }, + "azure_ai/claude-opus-4-1": { + p: "azure_ai", + i: 0.000015, + o: 0.000075, + cr: 0.0000015, + cw: 0.00001875, + }, + "azure_ai/claude-opus-4-5": { p: "azure_ai", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "azure_ai/claude-opus-4-6": { p: "azure_ai", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "azure_ai/claude-opus-4-7": { p: "azure_ai", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "azure_ai/claude-opus-4-8": { p: "azure_ai", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "azure_ai/claude-opus-5": { p: "azure_ai", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "azure_ai/claude-sonnet-4-5": { + p: "azure_ai", + i: 0.000003, + o: 0.000015, + cr: 3e-7, + cw: 0.00000375, + }, + "azure_ai/claude-sonnet-4-6": { + p: "azure_ai", + i: 0.000003, + o: 0.000015, + cr: 3e-7, + cw: 0.00000375, + }, + "azure_ai/claude-sonnet-5": { p: "azure_ai", i: 0.000002, o: 0.00001, cr: 2e-7, cw: 0.0000025 }, + "azure_ai/deepseek-r1": { p: "azure_ai", i: 0.00000135, o: 0.0000054 }, + "azure_ai/deepseek-v3": { p: "azure_ai", i: 0.00000114, o: 0.00000456 }, + "azure_ai/deepseek-v3-0324": { p: "azure_ai", i: 0.00000114, o: 0.00000456 }, + "azure_ai/deepseek-v3.1": { p: "azure_ai", i: 0.00000123, o: 0.00000494 }, + "azure_ai/deepseek-v3.2": { p: "azure_ai", i: 5.8e-7, o: 0.00000168 }, + "azure_ai/deepseek-v3.2-speciale": { p: "azure_ai", i: 5.8e-7, o: 0.00000168 }, + "azure_ai/deepseek-v4-flash": { p: "azure_ai", i: 1.9e-7, o: 5.1e-7 }, + "azure_ai/deepseek-v4-pro": { p: "azure_ai", i: 0.00000174, o: 0.00000348 }, + "azure_ai/global/grok-3": { p: "azure_ai", i: 0.000003, o: 0.000015 }, + "azure_ai/global/grok-3-mini": { p: "azure_ai", i: 2.5e-7, o: 0.00000127 }, + "azure_ai/gpt-5.4": { p: "azure_ai", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "azure_ai/gpt-5.4-2026-03-05": { p: "azure_ai", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "azure_ai/gpt-5.4-mini": { p: "azure_ai", i: 7.5e-7, o: 0.0000045, cr: 7.5e-8 }, + "azure_ai/gpt-5.4-mini-2026-03-17": { p: "azure_ai", i: 7.5e-7, o: 0.0000045, cr: 7.5e-8 }, + "azure_ai/gpt-5.4-nano": { p: "azure_ai", i: 2e-7, o: 0.00000125, cr: 2e-8 }, + "azure_ai/gpt-5.4-nano-2026-03-17": { p: "azure_ai", i: 2e-7, o: 0.00000125, cr: 2e-8 }, + "azure_ai/gpt-5.4-pro": { p: "azure_ai", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "azure_ai/gpt-5.4-pro-2026-03-05": { p: "azure_ai", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "azure_ai/gpt-5.5": { p: "azure_ai", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "azure_ai/gpt-5.5-2026-04-23": { p: "azure_ai", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "azure_ai/gpt-oss-120b": { p: "azure_ai", i: 1.5e-7, o: 6e-7 }, + "azure_ai/grok-3": { p: "azure_ai", i: 0.000003, o: 0.000015 }, + "azure_ai/grok-3-mini": { p: "azure_ai", i: 2.5e-7, o: 0.00000127 }, + "azure_ai/grok-4": { p: "azure_ai", i: 0.000003, o: 0.000015 }, + "azure_ai/grok-4-1-fast-non-reasoning": { p: "azure_ai", i: 2e-7, o: 5e-7 }, + "azure_ai/grok-4-1-fast-reasoning": { p: "azure_ai", i: 2e-7, o: 5e-7 }, + "azure_ai/grok-4-fast-non-reasoning": { p: "azure_ai", i: 2e-7, o: 5e-7 }, + "azure_ai/grok-4-fast-reasoning": { p: "azure_ai", i: 2e-7, o: 5e-7 }, + "azure_ai/grok-code-fast-1": { p: "azure_ai", i: 2e-7, o: 0.0000015 }, + "azure_ai/jais-30b-chat": { p: "azure_ai", i: 0.0032, o: 0.00971 }, + "azure_ai/jamba-instruct": { p: "azure_ai", i: 5e-7, o: 7e-7 }, + "azure_ai/kimi-k2.5": { p: "azure_ai", i: 6e-7, o: 0.000003 }, + "azure_ai/kimi-k2.6": { p: "azure_ai", i: 9.5e-7, o: 0.000004 }, + "azure_ai/ministral-3b": { p: "azure_ai", i: 4e-8, o: 4e-8 }, + "azure_ai/mistral-large": { p: "azure_ai", i: 0.000004, o: 0.000012 }, + "azure_ai/mistral-large-2407": { p: "azure_ai", i: 0.000002, o: 0.000006 }, + "azure_ai/mistral-large-3": { p: "azure_ai", i: 5e-7, o: 0.0000015 }, + "azure_ai/mistral-large-latest": { p: "azure_ai", i: 0.000002, o: 0.000006 }, + "azure_ai/mistral-medium-2505": { p: "azure_ai", i: 4e-7, o: 0.000002 }, + "azure_ai/mistral-nemo": { p: "azure_ai", i: 1.5e-7, o: 1.5e-7 }, + "azure_ai/mistral-small": { p: "azure_ai", i: 0.000001, o: 0.000003 }, + "azure_ai/mistral-small-2503": { p: "azure_ai", i: 1e-7, o: 3e-7 }, + "azure_ai/model_router": { p: "azure_ai", i: 1.4e-7, o: 0 }, + "chatgpt-4o-latest": { p: "openai", i: 0.000005, o: 0.000015 }, + "claude-3-7-sonnet-20250219": { + p: "anthropic", + i: 0.000003, + o: 0.000015, + cr: 3e-7, + cw: 0.00000375, + }, + "claude-3-haiku-20240307": { p: "anthropic", i: 2.5e-7, o: 0.00000125, cr: 3e-8, cw: 3e-7 }, + "claude-3-opus-20240229": { + p: "anthropic", + i: 0.000015, + o: 0.000075, + cr: 0.0000015, + cw: 0.00001875, + }, + "claude-4-opus-20250514": { + p: "anthropic", + i: 0.000015, + o: 0.000075, + cr: 0.0000015, + cw: 0.00001875, + }, + "claude-4-sonnet-20250514": { + p: "anthropic", + i: 0.000003, + o: 0.000015, + cr: 3e-7, + cw: 0.00000375, + }, + "claude-fable-5": { p: "anthropic", i: 0.00001, o: 0.00005, cr: 0.000001, cw: 0.0000125 }, + "claude-haiku-4-5": { p: "anthropic", i: 0.000001, o: 0.000005, cr: 1e-7, cw: 0.00000125 }, + "claude-haiku-4-5-20251001": { + p: "anthropic", + i: 0.000001, + o: 0.000005, + cr: 1e-7, + cw: 0.00000125, + }, + "claude-opus-4-1": { p: "anthropic", i: 0.000015, o: 0.000075, cr: 0.0000015, cw: 0.00001875 }, + "claude-opus-4-1-20250805": { + p: "anthropic", + i: 0.000015, + o: 0.000075, + cr: 0.0000015, + cw: 0.00001875, + }, + "claude-opus-4-20250514": { + p: "anthropic", + i: 0.000015, + o: 0.000075, + cr: 0.0000015, + cw: 0.00001875, + }, + "claude-opus-4-5": { p: "anthropic", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "claude-opus-4-5-20251101": { + p: "anthropic", + i: 0.000005, + o: 0.000025, + cr: 5e-7, + cw: 0.00000625, + }, + "claude-opus-4-6": { p: "anthropic", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "claude-opus-4-6-20260205": { + p: "anthropic", + i: 0.000005, + o: 0.000025, + cr: 5e-7, + cw: 0.00000625, + }, + "claude-opus-4-7": { p: "anthropic", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "claude-opus-4-7-20260416": { + p: "anthropic", + i: 0.000005, + o: 0.000025, + cr: 5e-7, + cw: 0.00000625, + }, + "claude-opus-4-8": { p: "anthropic", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "claude-opus-5": { p: "anthropic", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "claude-sonnet-4-20250514": { + p: "anthropic", + i: 0.000003, + o: 0.000015, + cr: 3e-7, + cw: 0.00000375, + }, + "claude-sonnet-4-5": { p: "anthropic", i: 0.000003, o: 0.000015, cr: 3e-7, cw: 0.00000375 }, + "claude-sonnet-4-5-20250929": { + p: "anthropic", + i: 0.000003, + o: 0.000015, + cr: 3e-7, + cw: 0.00000375, + }, + "claude-sonnet-4-6": { p: "anthropic", i: 0.000003, o: 0.000015, cr: 3e-7, cw: 0.00000375 }, + "claude-sonnet-5": { p: "anthropic", i: 0.000002, o: 0.00001, cr: 2e-7, cw: 0.0000025 }, + "codex-mini-latest": { p: "openai", i: 0.0000015, o: 0.000006, cr: 3.75e-7 }, + "computer-use-preview": { p: "azure", i: 0.000003, o: 0.000012 }, + "deepseek-chat": { p: "deepseek", i: 2.8e-7, o: 4.2e-7, cr: 2.8e-8 }, + "deepseek-reasoner": { p: "deepseek", i: 2.8e-7, o: 4.2e-7, cr: 2.8e-8 }, + "deepseek-v4-flash": { p: "deepseek", i: 1.4e-7, o: 2.8e-7, cr: 2.8e-9, cw: 0 }, + "deepseek-v4-pro": { p: "deepseek", i: 4.35e-7, o: 8.7e-7, cr: 3.625e-9, cw: 0 }, + "deepseek/deepseek-chat": { p: "deepseek", i: 2.8e-7, o: 4.2e-7, cr: 2.8e-8, cw: 0 }, + "deepseek/deepseek-coder": { p: "deepseek", i: 1.4e-7, o: 2.8e-7 }, + "deepseek/deepseek-r1": { p: "deepseek", i: 5.5e-7, o: 0.00000219 }, + "deepseek/deepseek-reasoner": { p: "deepseek", i: 2.8e-7, o: 4.2e-7, cr: 2.8e-8 }, + "deepseek/deepseek-v3": { p: "deepseek", i: 2.7e-7, o: 0.0000011, cr: 7e-8, cw: 0 }, + "deepseek/deepseek-v3.2": { p: "deepseek", i: 2.8e-7, o: 4e-7 }, + "deepseek/deepseek-v4-flash": { p: "deepseek", i: 1.4e-7, o: 2.8e-7, cr: 2.8e-9, cw: 0 }, + "deepseek/deepseek-v4-pro": { p: "deepseek", i: 4.35e-7, o: 8.7e-7, cr: 3.625e-9, cw: 0 }, + "ft:gpt-3.5-turbo": { p: "openai", i: 0.000003, o: 0.000006 }, + "ft:gpt-3.5-turbo-0125": { p: "openai", i: 0.000003, o: 0.000006 }, + "ft:gpt-3.5-turbo-0613": { p: "openai", i: 0.000003, o: 0.000006 }, + "ft:gpt-3.5-turbo-1106": { p: "openai", i: 0.000003, o: 0.000006 }, + "ft:gpt-4-0613": { p: "openai", i: 0.00003, o: 0.00006 }, + "ft:gpt-4.1-2025-04-14": { p: "openai", i: 0.000003, o: 0.000012, cr: 7.5e-7 }, + "ft:gpt-4.1-mini-2025-04-14": { p: "openai", i: 8e-7, o: 0.0000032, cr: 2e-7 }, + "ft:gpt-4.1-nano-2025-04-14": { p: "openai", i: 2e-7, o: 8e-7, cr: 5e-8 }, + "ft:gpt-4o-2024-08-06": { p: "openai", i: 0.00000375, o: 0.000015, cr: 0.000001875 }, + "ft:gpt-4o-2024-11-20": { p: "openai", i: 0.00000375, o: 0.000015, cw: 0.000001875 }, + "ft:gpt-4o-mini-2024-07-18": { p: "openai", i: 3e-7, o: 0.0000012, cr: 1.5e-7 }, + "ft:o4-mini-2025-04-16": { p: "openai", i: 0.000004, o: 0.000016, cr: 0.000001 }, + "gemini-2.0-flash": { p: "vertex_ai-language-models", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gemini-2.0-flash-001": { p: "vertex_ai-language-models", i: 1.5e-7, o: 6e-7, cr: 3.75e-8 }, + "gemini-2.0-flash-lite": { p: "vertex_ai-language-models", i: 7.5e-8, o: 3e-7, cr: 1.875e-8 }, + "gemini-2.0-flash-lite-001": { p: "vertex_ai-language-models", i: 7.5e-8, o: 3e-7, cr: 1.875e-8 }, + "gemini-2.5-computer-use-preview-10-2025": { + p: "vertex_ai-language-models", + i: 0.00000125, + o: 0.00001, + }, + "gemini-2.5-flash": { p: "vertex_ai-language-models", i: 3e-7, o: 0.0000025, cr: 3e-8 }, + "gemini-2.5-flash-lite": { p: "vertex_ai-language-models", i: 1e-7, o: 4e-7, cr: 1e-8 }, + "gemini-2.5-flash-lite-preview-06-17": { + p: "vertex_ai-language-models", + i: 1e-7, + o: 4e-7, + cr: 2.5e-8, + }, + "gemini-2.5-flash-lite-preview-09-2025": { + p: "vertex_ai-language-models", + i: 1e-7, + o: 4e-7, + cr: 1e-8, + }, + "gemini-2.5-flash-native-audio-latest": { p: "gemini", i: 3e-7, o: 0.0000025 }, + "gemini-2.5-flash-native-audio-preview-09-2025": { p: "gemini", i: 3e-7, o: 0.0000025 }, + "gemini-2.5-flash-native-audio-preview-12-2025": { p: "gemini", i: 3e-7, o: 0.0000025 }, + "gemini-2.5-flash-preview-09-2025": { + p: "vertex_ai-language-models", + i: 3e-7, + o: 0.0000025, + cr: 7.5e-8, + }, + "gemini-2.5-pro": { p: "vertex_ai-language-models", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gemini-2.5-pro-preview-tts": { + p: "vertex_ai-language-models", + i: 0.00000125, + o: 0.00001, + cr: 1.25e-7, + }, + "gemini-3-flash-preview": { p: "vertex_ai-language-models", i: 5e-7, o: 0.000003, cr: 5e-8 }, + "gemini-3-pro-preview": { p: "vertex_ai-language-models", i: 0.000002, o: 0.000012, cr: 2e-7 }, + "gemini-3.1-flash-lite": { p: "vertex_ai-language-models", i: 2.5e-7, o: 0.0000015, cr: 2.5e-8 }, + "gemini-3.1-flash-lite-preview": { + p: "vertex_ai-language-models", + i: 2.5e-7, + o: 0.0000015, + cr: 2.5e-8, + }, + "gemini-3.1-flash-live-preview": { p: "gemini", i: 7.5e-7, o: 0.0000045 }, + "gemini-3.1-pro-preview": { p: "vertex_ai-language-models", i: 0.000002, o: 0.000012, cr: 2e-7 }, + "gemini-3.1-pro-preview-customtools": { + p: "vertex_ai-language-models", + i: 0.000002, + o: 0.000012, + cr: 2e-7, + }, + "gemini-3.5-flash": { p: "vertex_ai-language-models", i: 0.0000015, o: 0.000009, cr: 1.5e-7 }, + "gemini-3.5-flash-lite": { p: "vertex_ai-language-models", i: 3e-7, o: 0.0000025, cr: 3e-8 }, + "gemini-3.6-flash": { p: "vertex_ai-language-models", i: 0.0000015, o: 0.0000075, cr: 1.5e-7 }, + "gemini-exp-1206": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 3e-8 }, + "gemini-flash-latest": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 3e-8 }, + "gemini-flash-lite-latest": { p: "gemini", i: 1e-7, o: 4e-7, cr: 1e-8 }, + "gemini-omni-flash-preview": { p: "vertex_ai-language-models", i: 0.0000015, o: 0.000009 }, + "gemini-pro-latest": { p: "gemini", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gemini-robotics-er-1.5-preview": { + p: "vertex_ai-language-models", + i: 3e-7, + o: 0.0000025, + cr: 0, + }, + "gemini/gemini-2.0-flash": { p: "gemini", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gemini/gemini-2.0-flash-001": { p: "gemini", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gemini/gemini-2.0-flash-lite": { p: "gemini", i: 7.5e-8, o: 3e-7, cr: 1.875e-8 }, + "gemini/gemini-2.0-flash-lite-001": { p: "gemini", i: 7.5e-8, o: 3e-7, cr: 1.875e-8 }, + "gemini/gemini-2.5-computer-use-preview-10-2025": { p: "gemini", i: 0.00000125, o: 0.00001 }, + "gemini/gemini-2.5-flash": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 3e-8 }, + "gemini/gemini-2.5-flash-lite": { p: "gemini", i: 1e-7, o: 4e-7, cr: 1e-8 }, + "gemini/gemini-2.5-flash-lite-preview-06-17": { p: "gemini", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gemini/gemini-2.5-flash-lite-preview-09-2025": { p: "gemini", i: 1e-7, o: 4e-7, cr: 1e-8 }, + "gemini/gemini-2.5-flash-native-audio-latest": { p: "gemini", i: 3e-7, o: 0.0000025 }, + "gemini/gemini-2.5-flash-native-audio-preview-09-2025": { p: "gemini", i: 3e-7, o: 0.0000025 }, + "gemini/gemini-2.5-flash-native-audio-preview-12-2025": { p: "gemini", i: 3e-7, o: 0.0000025 }, + "gemini/gemini-2.5-flash-preview-09-2025": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 7.5e-8 }, + "gemini/gemini-2.5-pro": { p: "gemini", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gemini/gemini-2.5-pro-preview-tts": { p: "gemini", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gemini/gemini-3-flash-preview": { p: "gemini", i: 5e-7, o: 0.000003, cr: 5e-8 }, + "gemini/gemini-3-pro-preview": { p: "gemini", i: 0.000002, o: 0.000012, cr: 2e-7 }, + "gemini/gemini-3.1-flash-lite": { p: "gemini", i: 2.5e-7, o: 0.0000015, cr: 2.5e-8 }, + "gemini/gemini-3.1-flash-lite-preview": { p: "gemini", i: 2.5e-7, o: 0.0000015, cr: 2.5e-8 }, + "gemini/gemini-3.1-flash-live-preview": { p: "gemini", i: 7.5e-7, o: 0.0000045 }, + "gemini/gemini-3.1-pro-preview": { p: "gemini", i: 0.000002, o: 0.000012, cr: 2e-7 }, + "gemini/gemini-3.1-pro-preview-customtools": { p: "gemini", i: 0.000002, o: 0.000012, cr: 2e-7 }, + "gemini/gemini-3.5-flash": { p: "gemini", i: 0.0000015, o: 0.000009, cr: 1.5e-7 }, + "gemini/gemini-3.5-flash-lite": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 3e-8 }, + "gemini/gemini-3.6-flash": { p: "gemini", i: 0.0000015, o: 0.0000075, cr: 1.5e-7 }, + "gemini/gemini-exp-1114": { p: "gemini", i: 0, o: 0 }, + "gemini/gemini-exp-1206": { p: "gemini", i: 0, o: 0 }, + "gemini/gemini-flash-latest": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 7.5e-8 }, + "gemini/gemini-flash-lite-latest": { p: "gemini", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gemini/gemini-gemma-2-27b-it": { p: "gemini", i: 3.5e-7, o: 0.00000105 }, + "gemini/gemini-gemma-2-9b-it": { p: "gemini", i: 3.5e-7, o: 0.00000105 }, + "gemini/gemini-omni-flash-preview": { p: "gemini", i: 0.0000015, o: 0.000009 }, + "gemini/gemini-pro-latest": { p: "gemini", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gemini/gemini-robotics-er-1.5-preview": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 0 }, + "gemini/gemma-3-27b-it": { p: "gemini", i: 0, o: 0 }, + "gemini/learnlm-1.5-pro-experimental": { p: "gemini", i: 0, o: 0 }, + "gemini/lyria-3-clip-preview": { p: "gemini", i: 0, o: 0 }, + "gemini/lyria-3-pro-preview": { p: "gemini", i: 0, o: 0 }, + "gpt-3.5-turbo": { p: "openai", i: 5e-7, o: 0.0000015 }, + "gpt-3.5-turbo-0125": { p: "openai", i: 5e-7, o: 0.0000015 }, + "gpt-3.5-turbo-1106": { p: "openai", i: 0.000001, o: 0.000002 }, + "gpt-3.5-turbo-16k": { p: "openai", i: 0.000003, o: 0.000004 }, + "gpt-4": { p: "openai", i: 0.00003, o: 0.00006 }, + "gpt-4-0125-preview": { p: "openai", i: 0.00001, o: 0.00003 }, + "gpt-4-0314": { p: "openai", i: 0.00003, o: 0.00006 }, + "gpt-4-0613": { p: "openai", i: 0.00003, o: 0.00006 }, + "gpt-4-1106-preview": { p: "openai", i: 0.00001, o: 0.00003 }, + "gpt-4-turbo": { p: "openai", i: 0.00001, o: 0.00003 }, + "gpt-4-turbo-2024-04-09": { p: "openai", i: 0.00001, o: 0.00003 }, + "gpt-4-turbo-preview": { p: "openai", i: 0.00001, o: 0.00003 }, + "gpt-4.1": { p: "openai", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "gpt-4.1-2025-04-14": { p: "openai", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "gpt-4.1-mini": { p: "openai", i: 4e-7, o: 0.0000016, cr: 1e-7 }, + "gpt-4.1-mini-2025-04-14": { p: "openai", i: 4e-7, o: 0.0000016, cr: 1e-7 }, + "gpt-4.1-nano": { p: "openai", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gpt-4.1-nano-2025-04-14": { p: "openai", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gpt-4o": { p: "openai", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "gpt-4o-2024-05-13": { p: "openai", i: 0.000005, o: 0.000015 }, + "gpt-4o-2024-08-06": { p: "openai", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "gpt-4o-2024-11-20": { p: "openai", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "gpt-4o-audio-preview": { p: "openai", i: 0.0000025, o: 0.00001 }, + "gpt-4o-audio-preview-2024-12-17": { p: "openai", i: 0.0000025, o: 0.00001 }, + "gpt-4o-audio-preview-2025-06-03": { p: "openai", i: 0.0000025, o: 0.00001 }, + "gpt-4o-mini": { p: "openai", i: 1.5e-7, o: 6e-7, cr: 7.5e-8 }, + "gpt-4o-mini-2024-07-18": { p: "openai", i: 1.5e-7, o: 6e-7, cr: 7.5e-8 }, + "gpt-4o-mini-audio-preview": { p: "openai", i: 1.5e-7, o: 6e-7 }, + "gpt-4o-mini-audio-preview-2024-12-17": { p: "openai", i: 1.5e-7, o: 6e-7 }, + "gpt-4o-mini-search-preview": { p: "openai", i: 1.5e-7, o: 6e-7, cr: 7.5e-8 }, + "gpt-4o-mini-search-preview-2025-03-11": { p: "openai", i: 1.5e-7, o: 6e-7, cr: 7.5e-8 }, + "gpt-4o-search-preview": { p: "openai", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "gpt-4o-search-preview-2025-03-11": { p: "openai", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "gpt-5": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5-2025-08-07": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5-chat": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5-chat-latest": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5-codex": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5-mini": { p: "openai", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "gpt-5-mini-2025-08-07": { p: "openai", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "gpt-5-nano": { p: "openai", i: 5e-8, o: 4e-7, cr: 5e-9 }, + "gpt-5-nano-2025-08-07": { p: "openai", i: 5e-8, o: 4e-7, cr: 5e-9 }, + "gpt-5-pro": { p: "openai", i: 0.000015, o: 0.00012 }, + "gpt-5-pro-2025-10-06": { p: "openai", i: 0.000015, o: 0.00012 }, + "gpt-5-search-api": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5-search-api-2025-10-14": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5.1": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5.1-2025-11-13": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5.1-chat-latest": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5.1-codex": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5.1-codex-max": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5.1-codex-mini": { p: "openai", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "gpt-5.2": { p: "openai", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "gpt-5.2-2025-12-11": { p: "openai", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "gpt-5.2-chat-latest": { p: "openai", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "gpt-5.2-codex": { p: "openai", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "gpt-5.2-pro": { p: "openai", i: 0.000021, o: 0.000168 }, + "gpt-5.2-pro-2025-12-11": { p: "openai", i: 0.000021, o: 0.000168 }, + "gpt-5.3-chat-latest": { p: "openai", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "gpt-5.3-codex": { p: "openai", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "gpt-5.4": { p: "openai", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "gpt-5.4-2026-03-05": { p: "openai", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "gpt-5.4-mini": { p: "openai", i: 7.5e-7, o: 0.0000045, cr: 7.5e-8 }, + "gpt-5.4-mini-2026-03-17": { p: "openai", i: 7.5e-7, o: 0.0000045, cr: 7.5e-8 }, + "gpt-5.4-nano": { p: "openai", i: 2e-7, o: 0.00000125, cr: 2e-8 }, + "gpt-5.4-nano-2026-03-17": { p: "openai", i: 2e-7, o: 0.00000125, cr: 2e-8 }, + "gpt-5.4-pro": { p: "openai", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "gpt-5.4-pro-2026-03-05": { p: "openai", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "gpt-5.5": { p: "openai", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "gpt-5.5-2026-04-23": { p: "openai", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "gpt-5.5-pro": { p: "openai", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "gpt-5.5-pro-2026-04-23": { p: "openai", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "gpt-5.6": { p: "openai", i: 0.000005, o: 0.00003, cr: 5e-7, cw: 0.00000625 }, + "gpt-5.6-luna": { p: "openai", i: 2e-7, o: 0.0000012, cr: 2e-8, cw: 2.5e-7 }, + "gpt-5.6-sol": { p: "openai", i: 0.000005, o: 0.00003, cr: 5e-7, cw: 0.00000625 }, + "gpt-5.6-terra": { p: "openai", i: 0.000002, o: 0.000012, cr: 2e-7, cw: 0.0000025 }, + "gpt-audio": { p: "openai", i: 0.0000025, o: 0.00001 }, + "gpt-audio-1.5": { p: "openai", i: 0.0000025, o: 0.00001 }, + "gpt-audio-2025-08-28": { p: "openai", i: 0.0000025, o: 0.00001 }, + "gpt-audio-mini": { p: "openai", i: 6e-7, o: 0.0000024 }, + "gpt-audio-mini-2025-10-06": { p: "openai", i: 6e-7, o: 0.0000024 }, + "gpt-audio-mini-2025-12-15": { p: "openai", i: 6e-7, o: 0.0000024 }, + "groq/gemma-7b-it": { p: "groq", i: 5e-8, o: 8e-8 }, + "groq/llama-3.1-8b-instant": { p: "groq", i: 5e-8, o: 8e-8 }, + "groq/llama-3.3-70b-versatile": { p: "groq", i: 5.9e-7, o: 7.9e-7 }, + "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { p: "groq", i: 2e-7, o: 6e-7 }, + "groq/meta-llama/llama-4-scout-17b-16e-instruct": { p: "groq", i: 1.1e-7, o: 3.4e-7 }, + "groq/meta-llama/llama-guard-4-12b": { p: "groq", i: 2e-7, o: 2e-7 }, + "groq/moonshotai/kimi-k2-instruct-0905": { p: "groq", i: 0.000001, o: 0.000003, cr: 5e-7 }, + "groq/openai/gpt-oss-120b": { p: "groq", i: 1.5e-7, o: 6e-7, cr: 7.5e-8 }, + "groq/openai/gpt-oss-20b": { p: "groq", i: 7.5e-8, o: 3e-7, cr: 3.75e-8 }, + "groq/openai/gpt-oss-safeguard-20b": { p: "groq", i: 7.5e-8, o: 3e-7, cr: 3.7e-8 }, + "groq/qwen/qwen3-32b": { p: "groq", i: 2.9e-7, o: 5.9e-7 }, + "mistral/codestral-2405": { p: "mistral", i: 0.000001, o: 0.000003 }, + "mistral/codestral-2508": { p: "mistral", i: 3e-7, o: 9e-7 }, + "mistral/codestral-latest": { p: "mistral", i: 0.000001, o: 0.000003 }, + "mistral/codestral-mamba-latest": { p: "mistral", i: 2.5e-7, o: 2.5e-7 }, + "mistral/devstral-2512": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/devstral-latest": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/devstral-medium-2507": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/devstral-medium-latest": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/devstral-small-2505": { p: "mistral", i: 1e-7, o: 3e-7 }, + "mistral/devstral-small-2507": { p: "mistral", i: 1e-7, o: 3e-7 }, + "mistral/devstral-small-latest": { p: "mistral", i: 1e-7, o: 3e-7 }, + "mistral/labs-devstral-small-2512": { p: "mistral", i: 1e-7, o: 3e-7 }, + "mistral/magistral-medium-1-2-2509": { p: "mistral", i: 0.000002, o: 0.000005 }, + "mistral/magistral-medium-2506": { p: "mistral", i: 0.000002, o: 0.000005 }, + "mistral/magistral-medium-2509": { p: "mistral", i: 0.000002, o: 0.000005 }, + "mistral/magistral-medium-latest": { p: "mistral", i: 0.000002, o: 0.000005 }, + "mistral/magistral-small-1-2-2509": { p: "mistral", i: 5e-7, o: 0.0000015 }, + "mistral/magistral-small-2506": { p: "mistral", i: 5e-7, o: 0.0000015 }, + "mistral/magistral-small-latest": { p: "mistral", i: 5e-7, o: 0.0000015 }, + "mistral/ministral-3-14b-2512": { p: "mistral", i: 2e-7, o: 2e-7 }, + "mistral/ministral-3-3b-2512": { p: "mistral", i: 1e-7, o: 1e-7 }, + "mistral/ministral-3-8b-2512": { p: "mistral", i: 1.5e-7, o: 1.5e-7 }, + "mistral/ministral-8b-2512": { p: "mistral", i: 1.5e-7, o: 1.5e-7 }, + "mistral/ministral-8b-latest": { p: "mistral", i: 1.5e-7, o: 1.5e-7 }, + "mistral/mistral-large-2402": { p: "mistral", i: 0.000004, o: 0.000012 }, + "mistral/mistral-large-2407": { p: "mistral", i: 0.000003, o: 0.000009 }, + "mistral/mistral-large-2411": { p: "mistral", i: 0.000002, o: 0.000006 }, + "mistral/mistral-large-2512": { p: "mistral", i: 5e-7, o: 0.0000015 }, + "mistral/mistral-large-3": { p: "mistral", i: 5e-7, o: 0.0000015 }, + "mistral/mistral-large-latest": { p: "mistral", i: 5e-7, o: 0.0000015 }, + "mistral/mistral-medium": { p: "mistral", i: 0.0000027, o: 0.0000081 }, + "mistral/mistral-medium-2312": { p: "mistral", i: 0.0000027, o: 0.0000081 }, + "mistral/mistral-medium-2505": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/mistral-medium-2508": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/mistral-medium-2604": { p: "mistral", i: 0.0000015, o: 0.0000075 }, + "mistral/mistral-medium-3-1-2508": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/mistral-medium-3-5": { p: "mistral", i: 0.0000015, o: 0.0000075 }, + "mistral/mistral-medium-latest": { p: "mistral", i: 0.0000015, o: 0.0000075 }, + "mistral/mistral-small": { p: "mistral", i: 1e-7, o: 3e-7 }, + "mistral/mistral-small-3-2-2506": { p: "mistral", i: 6e-8, o: 1.8e-7 }, + "mistral/mistral-small-latest": { p: "mistral", i: 6e-8, o: 1.8e-7 }, + "mistral/mistral-tiny": { p: "mistral", i: 2.5e-7, o: 2.5e-7 }, + "mistral/open-codestral-mamba": { p: "mistral", i: 2.5e-7, o: 2.5e-7 }, + "mistral/open-mistral-7b": { p: "mistral", i: 2.5e-7, o: 2.5e-7 }, + "mistral/open-mistral-nemo": { p: "mistral", i: 3e-7, o: 3e-7 }, + "mistral/open-mistral-nemo-2407": { p: "mistral", i: 3e-7, o: 3e-7 }, + "mistral/open-mixtral-8x22b": { p: "mistral", i: 0.000002, o: 0.000006 }, + "mistral/open-mixtral-8x7b": { p: "mistral", i: 7e-7, o: 7e-7 }, + "mistral/pixtral-12b-2409": { p: "mistral", i: 1.5e-7, o: 1.5e-7 }, + "mistral/pixtral-large-2411": { p: "mistral", i: 0.000002, o: 0.000006 }, + "mistral/pixtral-large-latest": { p: "mistral", i: 0.000002, o: 0.000006 }, + o1: { p: "openai", i: 0.000015, o: 0.00006, cr: 0.0000075 }, + "o1-2024-12-17": { p: "openai", i: 0.000015, o: 0.00006, cr: 0.0000075 }, + "o1-pro": { p: "openai", i: 0.00015, o: 0.0006 }, + "o1-pro-2025-03-19": { p: "openai", i: 0.00015, o: 0.0006 }, + o3: { p: "openai", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "o3-2025-04-16": { p: "openai", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "o3-deep-research": { p: "openai", i: 0.00001, o: 0.00004, cr: 0.0000025 }, + "o3-deep-research-2025-06-26": { p: "openai", i: 0.00001, o: 0.00004, cr: 0.0000025 }, + "o3-mini": { p: "openai", i: 0.0000011, o: 0.0000044, cr: 5.5e-7 }, + "o3-mini-2025-01-31": { p: "openai", i: 0.0000011, o: 0.0000044, cr: 5.5e-7 }, + "o3-pro": { p: "openai", i: 0.00002, o: 0.00008 }, + "o3-pro-2025-06-10": { p: "openai", i: 0.00002, o: 0.00008 }, + "o4-mini": { p: "openai", i: 0.0000011, o: 0.0000044, cr: 2.75e-7 }, + "o4-mini-2025-04-16": { p: "openai", i: 0.0000011, o: 0.0000044, cr: 2.75e-7 }, + "o4-mini-deep-research": { p: "openai", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "o4-mini-deep-research-2025-06-26": { p: "openai", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "vertex_ai/gemini-3.1-flash-lite": { + p: "vertex_ai-language-models", + i: 2.5e-7, + o: 0.0000015, + cr: 2.5e-8, + }, + "vertex_ai/gemini-3.1-flash-lite-preview": { + p: "vertex_ai-language-models", + i: 2.5e-7, + o: 0.0000015, + cr: 2.5e-8, + }, + "vertex_ai/gemini-3.5-flash-lite": { + p: "vertex_ai-language-models", + i: 3e-7, + o: 0.0000025, + cr: 3e-8, + }, + "xai/grok-2": { p: "xai", i: 0.000002, o: 0.00001 }, + "xai/grok-2-1212": { p: "xai", i: 0.000002, o: 0.00001 }, + "xai/grok-2-latest": { p: "xai", i: 0.000002, o: 0.00001 }, + "xai/grok-2-vision": { p: "xai", i: 0.000002, o: 0.00001 }, + "xai/grok-2-vision-1212": { p: "xai", i: 0.000002, o: 0.00001 }, + "xai/grok-2-vision-latest": { p: "xai", i: 0.000002, o: 0.00001 }, + "xai/grok-3": { p: "xai", i: 0.000003, o: 0.000015, cr: 7.5e-7 }, + "xai/grok-3-beta": { p: "xai", i: 0.000003, o: 0.000015, cr: 7.5e-7 }, + "xai/grok-3-fast-beta": { p: "xai", i: 0.000005, o: 0.000025, cr: 0.00000125 }, + "xai/grok-3-fast-latest": { p: "xai", i: 0.000005, o: 0.000025, cr: 0.00000125 }, + "xai/grok-3-latest": { p: "xai", i: 0.000003, o: 0.000015, cr: 7.5e-7 }, + "xai/grok-3-mini": { p: "xai", i: 3e-7, o: 5e-7, cr: 7.5e-8 }, + "xai/grok-3-mini-beta": { p: "xai", i: 3e-7, o: 5e-7, cr: 7.5e-8 }, + "xai/grok-3-mini-fast": { p: "xai", i: 6e-7, o: 0.000004, cr: 1.5e-7 }, + "xai/grok-3-mini-fast-beta": { p: "xai", i: 6e-7, o: 0.000004, cr: 1.5e-7 }, + "xai/grok-3-mini-fast-latest": { p: "xai", i: 6e-7, o: 0.000004, cr: 1.5e-7 }, + "xai/grok-3-mini-latest": { p: "xai", i: 3e-7, o: 5e-7, cr: 7.5e-8 }, + "xai/grok-4": { p: "xai", i: 0.000003, o: 0.000015 }, + "xai/grok-4-0709": { p: "xai", i: 0.000003, o: 0.000015 }, + "xai/grok-4-1-fast": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-1-fast-non-reasoning": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-1-fast-non-reasoning-latest": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-1-fast-reasoning": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-1-fast-reasoning-latest": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-fast-non-reasoning": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-fast-reasoning": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-latest": { p: "xai", i: 0.000003, o: 0.000015 }, + "xai/grok-4.20-0309-reasoning": { p: "xai", i: 0.000002, o: 0.000006, cr: 2e-7 }, + "xai/grok-4.20-beta-0309-non-reasoning": { p: "xai", i: 0.000002, o: 0.000006, cr: 2e-7 }, + "xai/grok-4.20-beta-0309-reasoning": { p: "xai", i: 0.000002, o: 0.000006, cr: 2e-7 }, + "xai/grok-4.20-multi-agent-beta-0309": { p: "xai", i: 0.000002, o: 0.000006, cr: 2e-7 }, + "xai/grok-4.3": { p: "xai", i: 0.00000125, o: 0.0000025, cr: 2e-7 }, + "xai/grok-4.3-latest": { p: "xai", i: 0.00000125, o: 0.0000025, cr: 2e-7 }, + "xai/grok-4.5": { p: "xai", i: 0.000002, o: 0.000006, cr: 5e-7 }, + "xai/grok-4.5-latest": { p: "xai", i: 0.000002, o: 0.000006, cr: 5e-7 }, + "xai/grok-beta": { p: "xai", i: 0.000005, o: 0.000015 }, + "xai/grok-code-fast": { p: "xai", i: 2e-7, o: 0.0000015, cr: 2e-8 }, + "xai/grok-code-fast-1": { p: "xai", i: 2e-7, o: 0.0000015, cr: 2e-8 }, + "xai/grok-code-fast-1-0825": { p: "xai", i: 2e-7, o: 0.0000015, cr: 2e-8 }, + "xai/grok-vision-beta": { p: "xai", i: 0.000005, o: 0.000015 }, +}; diff --git a/core/src/pricing/providerAliases.ts b/core/src/pricing/providerAliases.ts new file mode 100644 index 00000000..cd50dd4f --- /dev/null +++ b/core/src/pricing/providerAliases.ts @@ -0,0 +1,45 @@ +/** + * Maps each opfor provider to the `litellm_provider` values that legitimately + * price it in the upstream price map. + * + * Two names for one thing is the norm, not the exception: opfor's `google` + * appears upstream as both `gemini` (direct API) and `vertex_ai-language-models` + * (the same models via Vertex), and `azure` splits into `azure` / `azure_ai`. + * + * This map is the single source of truth for both sides of the feature: + * `scripts/build-pricing.ts` filters the vendored table by the union of these + * values, and `lookupPrice` uses them to reject a match from the wrong provider. + * Keeping one definition is what stops the table from containing entries the + * lookup can never accept, or vice versa. + * + * `null` means "cannot be verified" — a proxy speaking the OpenAI protocol can + * serve any model from any vendor, so there is no provider to check against. + */ +export const LITELLM_PROVIDER_ALIASES: Record = { + openai: ["openai"], + anthropic: ["anthropic"], + groq: ["groq"], + google: ["gemini", "vertex_ai-language-models"], + deepseek: ["deepseek"], + azure: ["azure", "azure_ai"], + "openai-compatible": null, +}; + +/** + * Upstream providers whose entries are kept in the vendored table. + * + * The union of the aliases above, plus a few vendors that are commonly reached + * *through* an `openai-compatible` proxy and are cheap to include. Deliberately + * excluded: bedrock, fireworks, openrouter and friends — together they are an + * order of magnitude more entries, and a setup routing through them is one where + * the proxy can report exact cost directly, making a list price redundant. + */ +export const VENDORED_LITELLM_PROVIDERS: string[] = [ + ...new Set([ + ...Object.values(LITELLM_PROVIDER_ALIASES) + .filter((v): v is string[] => v !== null) + .flat(), + "mistral", + "xai", + ]), +]; diff --git a/core/src/pricing/types.ts b/core/src/pricing/types.ts new file mode 100644 index 00000000..53fce989 --- /dev/null +++ b/core/src/pricing/types.ts @@ -0,0 +1,60 @@ +/** + * Types for run cost estimation. + * + * Prices are USD per single token (the unit the upstream LiteLLM price map uses), + * not per million — keeping the raw unit avoids a scaling step at every call site + * and a class of off-by-1e6 bugs. Format for display at the edge. + */ + +/** Per-token USD prices for one model. */ +export interface ModelPrice { + /** USD per input token. */ + inputPerToken: number; + /** USD per output token. */ + outputPerToken: number; + /** + * USD per cached (repeat) input token, when the provider publishes one. + * Not applied yet — token counting does not separate repeat tokens today. + */ + cacheReadPerToken?: number; + /** USD per cache-write token, when the provider publishes one. Not applied yet. */ + cacheWritePerToken?: number; +} + +/** Where a price came from. Ordered best-first; today only `table` and `unknown` occur. */ +export type CostSource = "table" | "unknown"; + +/** Cost attributed to one model. */ +export interface ModelCost { + /** `":"` — matches the token breakdown's key. */ + key: string; + provider: string; + model: string; + /** Run phases this model served, e.g. `["attacker"]`. */ + roles: string[]; + inputTokens: number; + outputTokens: number; + totalTokens: number; + /** USD, or undefined when the model could not be priced. */ + usd?: number; + source: CostSource; + /** The price-table key this matched, for auditability. Absent when unpriced. */ + matchedKey?: string; +} + +/** Cost of one run (or one evaluator), split by model. */ +export interface RunCost { + /** Sum of every priced model's cost. Excludes unpriced models by definition. */ + totalUsd: number; + currency: "USD"; + byModel: ModelCost[]; + /** + * Models whose price was not found, as `":"`. Non-empty means + * `totalUsd` is a lower bound — surface this rather than implying $0. + */ + unpricedModels: string[]; + /** True when every model that spent tokens was priced. */ + complete: boolean; + /** Which price snapshot produced this, for reproducibility. */ + priceTableVersion: string; +} diff --git a/core/src/providers/factory.ts b/core/src/providers/factory.ts index 90589dc8..667fc1b1 100644 --- a/core/src/providers/factory.ts +++ b/core/src/providers/factory.ts @@ -7,6 +7,7 @@ import { createAzure } from "@ai-sdk/azure"; import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; import type { LlmConfig } from "../config/types.js"; import { getEnv } from "../lib/env.js"; +import { rememberModelIdentity } from "./modelIdentity.js"; export interface ProviderCapabilities { supportsJsonMode: boolean; @@ -201,5 +202,9 @@ export function createModel(llm: LlmConfig): LanguageModel { if (adapter.capabilities.requiresBaseURL && !llm.baseURL) { throw new Error(adapter.baseUrlError ?? `baseURL is required for provider '${llm.provider}'`); } - return adapter.build({ apiKey, model: llm.model, baseURL: llm.baseURL }); + const built = adapter.build({ apiKey, model: llm.model, baseURL: llm.baseURL }); + // The built model no longer carries the opfor provider name (openai-compatible + // becomes "custom.chat"), so record it here for token-usage attribution. + rememberModelIdentity(built, llm); + return built; } diff --git a/core/src/providers/modelIdentity.ts b/core/src/providers/modelIdentity.ts new file mode 100644 index 00000000..994b08ec --- /dev/null +++ b/core/src/providers/modelIdentity.ts @@ -0,0 +1,88 @@ +/** + * Model identity registry — maps a built AI SDK model back to the opfor + * provider/model pair it was created from. + * + * Why this exists: {@link createModel} returns an opaque AI SDK object whose own + * `provider` field is SDK-flavored and lossy — `openai-compatible` surfaces as + * `"custom.chat"`, `anthropic` as `"anthropic.messages"`. Token-usage + * attribution (and, later, cost lookup) needs the provider name the *user* + * configured, because that is what the price tables are keyed against. + * + * Registering identity at creation time means every downstream recording site + * can attribute usage from the model object it already holds, instead of + * threading `LlmConfig` through a dozen call signatures. + */ + +import type { LanguageModel } from "ai"; +import type { LlmConfig } from "../config/types.js"; + +/** The opfor-side identity of one model: provider name + model string, as configured. */ +export interface ModelIdentity { + /** opfor provider name, e.g. `"anthropic"` or `"openai-compatible"`. */ + provider: string; + /** Model string exactly as the user configured it, e.g. `"deepseek/deepseek-v4-pro"`. */ + model: string; +} + +/** Bucket key for usage that could not be attributed to any model. */ +export const UNKNOWN_MODEL_KEY = "unknown"; + +/** Stable map/display key for one identity. */ +export function modelKey(identity: ModelIdentity): string { + return `${identity.provider}:${identity.model}`; +} + +// WeakMap so registering a model never keeps it alive. Keyed by the model object +// itself; `createModel` is the single factory for every path (CLI, SDK, MCP +// runner, browser extension), so one registration point covers all of them. +const registry = new WeakMap(); + +/** Record which provider/model a built AI SDK model came from. Called by `createModel`. */ +export function rememberModelIdentity(model: LanguageModel, llm: LlmConfig): void { + if (model && typeof model === "object") { + registry.set(model, { provider: llm.provider, model: llm.model }); + } +} + +/** Anything a call site might have on hand when recording token usage. */ +export type ModelRef = LanguageModel | LlmConfig | ModelIdentity; + +/** Shape test for the `{ provider, model }` pair shared by LlmConfig and ModelIdentity. */ +function isIdentityLike(value: object): value is ModelIdentity { + const v = value as Partial; + return typeof v.provider === "string" && typeof v.model === "string"; +} + +/** Shape test for a built AI SDK language model. */ +function isLanguageModelObject(value: object): value is { modelId: string; provider?: string } { + return typeof (value as { modelId?: unknown }).modelId === "string"; +} + +/** + * Resolve any model reference to an identity. + * + * Prefers the registry (exact configured provider + model). Falls back to the AI + * SDK object's own fields, narrowing `"anthropic.messages"` to `"anthropic"` — + * lossy for `openai-compatible` (reports `"custom"`), which is why the registry + * is consulted first. Returns undefined when nothing identifying is available, + * so callers can bucket the usage as unattributed rather than guess. + */ +export function resolveModelIdentity(ref?: ModelRef): ModelIdentity | undefined { + if (!ref) return undefined; + + // `ai` allows a bare model string; there is no provider to recover from it. + if (typeof ref === "string") return { provider: "unknown", model: ref }; + if (typeof ref !== "object") return undefined; + + const registered = registry.get(ref); + if (registered) return registered; + + if (isLanguageModelObject(ref)) { + const sdkProvider = typeof ref.provider === "string" ? ref.provider.split(".")[0] : "unknown"; + return { provider: sdkProvider || "unknown", model: ref.modelId }; + } + + if (isIdentityLike(ref)) return { provider: ref.provider, model: ref.model }; + + return undefined; +} diff --git a/core/src/report/buildReport.ts b/core/src/report/buildReport.ts index bd5f1e16..74081aae 100644 --- a/core/src/report/buildReport.ts +++ b/core/src/report/buildReport.ts @@ -90,6 +90,8 @@ function toEvaluatorViewModel(ev: EvaluatorResult): EvaluatorViewModel { passRate: ev.passRate, results: ev.attacks.map(toResultViewModel), tokenUsage: ev.tokenUsage, + tokenUsageByModel: ev.tokenUsageByModel, + cost: ev.cost, }; } diff --git a/core/src/report/render.ts b/core/src/report/render.ts index 2227c9db..96eef2a2 100644 --- a/core/src/report/render.ts +++ b/core/src/report/render.ts @@ -22,6 +22,35 @@ import { roleLabel, SEV_HEX, } from "./format.js"; +import type { RunCost } from "../pricing/types.js"; +import { formatUsd } from "../pricing/estimateCost.js"; + +/** + * Sub-label for the cost card: the attacker/judge split when both are present + * (the number people act on — "the judge is most of the spend"), otherwise a + * coverage note. Always states when models went unpriced, so a partial total is + * never mistaken for a complete one. + */ +function costSubLabel(cost: RunCost): string { + if (cost.unpricedModels.length > 0) { + const priced = cost.byModel.length - cost.unpricedModels.length; + return `${priced} of ${cost.byModel.length} models priced — lower bound`; + } + const byRole = new Map(); + for (const m of cost.byModel) { + // One role → label it. Several → "mixed". None → "unknown": a bucket with no + // recorded phase is unattributed, which is not the same as serving several. + const role = m.roles.length === 1 ? m.roles[0] : m.roles.length > 1 ? "mixed" : "unknown"; + byRole.set(role, (byRole.get(role) ?? 0) + (m.usd ?? 0)); + } + if (byRole.size > 1) { + return [...byRole.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([role, usd]) => `${role} ${formatUsd(usd)}`) + .join(" · "); + } + return `${cost.byModel.length} model${cost.byModel.length === 1 ? "" : "s"} · attacker + judge only`; +} // ── Mode-specific labels ───────────────────────────────────────── @@ -131,7 +160,7 @@ export function renderReport(model: ReportViewModel): string {
- ${e.tokenUsage ? `${formatTokenCount(e.tokenUsage.totalTokens)} tokens|` : ""} + ${e.tokenUsage ? `${formatTokenCount(e.tokenUsage.totalTokens)} tokens${e.cost ? ` · ≈${formatUsd(e.cost.totalUsd)}` : ""}|` : ""} ${evalVerdict === "PASS" ? "Pass" : evalVerdict === "ERROR" ? "Error" : "Fail"} Safety score: ${avgScore ?? "—"}/10
@@ -436,6 +465,15 @@ export function renderReport(model: ReportViewModel): string { ` : "" } + ${ + summary.cost + ? `
+
Testing Cost
+
≈${formatUsd(summary.cost.totalUsd)}
+
${esc(costSubLabel(summary.cost))}
+
` + : "" + } ${ summary.durationMs !== undefined ? `
diff --git a/core/src/report/types.ts b/core/src/report/types.ts index 468b9a63..f86d1e25 100644 --- a/core/src/report/types.ts +++ b/core/src/report/types.ts @@ -4,6 +4,8 @@ */ import type { JudgeResult } from "../lib/judgeTypes.js"; +import type { ModelTokenUsage } from "../execute/tokenTracker.js"; +import type { RunCost } from "../pricing/types.js"; /** @deprecated Use JudgeResult from @keyvaluesystems/agent-opfor-core/lib/judgeTypes.js directly. */ export type ReportJudge = JudgeResult; @@ -45,6 +47,10 @@ export interface EvaluatorViewModel { passRate: number; results: ResultViewModel[]; tokenUsage?: { inputTokens: number; outputTokens: number; totalTokens: number }; + /** Same tokens as `tokenUsage`, split by the model that spent them. */ + tokenUsageByModel?: ModelTokenUsage[]; + /** Estimated USD cost of this evaluator's attacker + judge calls. */ + cost?: RunCost; } export interface ReportViewModel { @@ -74,6 +80,13 @@ export interface ReportViewModel { safetyScore: number; attackSuccessRate: number; tokenUsage?: { inputTokens: number; outputTokens: number; totalTokens: number }; + /** Same tokens as `tokenUsage`, split by the model that spent them. */ + tokenUsageByModel?: ModelTokenUsage[]; + /** + * Estimated USD cost of the attacker + judge LLM calls. Excludes the + * target's own inference cost, which opfor cannot observe. + */ + cost?: RunCost; /** Wall-clock time for the whole run, from the first evaluator to the last. */ durationMs?: number; }; diff --git a/core/src/run/judge.ts b/core/src/run/judge.ts index 3e33a8bb..7e3f264b 100644 --- a/core/src/run/judge.ts +++ b/core/src/run/judge.ts @@ -194,6 +194,7 @@ export async function judgeToolResponse( // Safe as a predicate because parseJson never throws — it degrades to ERROR. isAcceptableJson: (json) => verdictParser.parseJson(json).verdict !== "ERROR", tokenTracker: args.tokenTracker, + role: "judge", }); return verdictParser.parseJson(raw); diff --git a/core/tests/modelIdentity.test.ts b/core/tests/modelIdentity.test.ts new file mode 100644 index 00000000..7ddab8f2 --- /dev/null +++ b/core/tests/modelIdentity.test.ts @@ -0,0 +1,93 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import type { LanguageModel } from "ai"; +import { + modelKey, + rememberModelIdentity, + resolveModelIdentity, + UNKNOWN_MODEL_KEY, +} from "../src/providers/modelIdentity.js"; +import type { LlmConfig } from "../src/config/types.js"; + +/** Stand-in for a built AI SDK model — only the fields the resolver reads. */ +function fakeSdkModel(modelId: string, provider: string): LanguageModel { + return { modelId, provider } as unknown as LanguageModel; +} + +test("modelKey joins provider and model", () => { + assert.equal( + modelKey({ provider: "anthropic", model: "claude-opus-5" }), + "anthropic:claude-opus-5" + ); +}); + +test("UNKNOWN_MODEL_KEY is distinct from any real key", () => { + assert.notEqual(UNKNOWN_MODEL_KEY, modelKey({ provider: "openai", model: "gpt-4o-mini" })); +}); + +test("resolveModelIdentity returns undefined for no reference", () => { + assert.equal(resolveModelIdentity(undefined), undefined); +}); + +test("resolveModelIdentity handles a bare model string with no recoverable provider", () => { + assert.deepStrictEqual(resolveModelIdentity("gpt-4o-mini"), { + provider: "unknown", + model: "gpt-4o-mini", + }); +}); + +test("resolveModelIdentity accepts an LlmConfig directly", () => { + const llm: LlmConfig = { + provider: "openai-compatible", + model: "deepseek/deepseek-v4-pro", + apiKeyEnv: "OPFOR_API_KEY", + baseURL: "https://llm.keyvalue.systems/v1", + }; + assert.deepStrictEqual(resolveModelIdentity(llm), { + provider: "openai-compatible", + model: "deepseek/deepseek-v4-pro", + }); +}); + +test("resolveModelIdentity falls back to SDK fields, narrowing the dotted provider", () => { + const m = fakeSdkModel("claude-opus-5", "anthropic.messages"); + assert.deepStrictEqual(resolveModelIdentity(m), { + provider: "anthropic", + model: "claude-opus-5", + }); +}); + +test("SDK fallback yields the lossy provider that makes the registry necessary", () => { + // An openai-compatible model reports "custom.chat" — the configured provider + // name is unrecoverable from the object alone. This is why createModel registers. + const m = fakeSdkModel("deepseek/deepseek-v4-pro", "custom.chat"); + assert.equal(resolveModelIdentity(m)?.provider, "custom"); +}); + +test("a registered identity wins over the SDK's own fields", () => { + const m = fakeSdkModel("deepseek/deepseek-v4-pro", "custom.chat"); + rememberModelIdentity(m, { + provider: "openai-compatible", + model: "deepseek/deepseek-v4-pro", + apiKeyEnv: "OPFOR_API_KEY", + baseURL: "https://llm.keyvalue.systems/v1", + }); + assert.deepStrictEqual(resolveModelIdentity(m), { + provider: "openai-compatible", + model: "deepseek/deepseek-v4-pro", + }); +}); + +test("rememberModelIdentity is a no-op for a non-object model reference", () => { + const llm: LlmConfig = { provider: "openai", model: "gpt-4o-mini", apiKeyEnv: "OPENAI_API_KEY" }; + // Must not throw — `ai` permits a bare string model. + rememberModelIdentity("gpt-4o-mini" as unknown as LanguageModel, llm); + assert.deepStrictEqual(resolveModelIdentity("gpt-4o-mini"), { + provider: "unknown", + model: "gpt-4o-mini", + }); +}); + +test("resolveModelIdentity returns undefined for an object with nothing identifying", () => { + assert.equal(resolveModelIdentity({} as unknown as LanguageModel), undefined); +}); diff --git a/core/tests/pricing.test.ts b/core/tests/pricing.test.ts new file mode 100644 index 00000000..3efb001a --- /dev/null +++ b/core/tests/pricing.test.ts @@ -0,0 +1,231 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { lookupPrice, priceCandidates, PRICE_TABLE_VERSION } from "../src/pricing/lookupPrice.js"; +import { estimateRunCost, formatUsd } from "../src/pricing/estimateCost.js"; +import { PRICE_TABLE } from "../src/pricing/priceTable.generated.js"; +import { LITELLM_PROVIDER_ALIASES } from "../src/pricing/providerAliases.js"; +import type { ModelTokenUsage } from "../src/execute/tokenTracker.js"; + +/** Build a token-breakdown row without repeating the boilerplate. */ +function usage( + provider: string, + model: string, + inputTokens: number, + outputTokens: number, + roles: string[] = ["attacker"] +): ModelTokenUsage { + return { + key: `${provider}:${model}`, + provider, + model, + roles, + calls: 1, + inputTokens, + outputTokens, + totalTokens: inputTokens + outputTokens, + }; +} + +// --------------------------------------------------------------------------- +// Vendored table sanity +// --------------------------------------------------------------------------- + +test("the vendored price table is populated and versioned", () => { + assert.ok(Object.keys(PRICE_TABLE).length > 100); + assert.match(PRICE_TABLE_VERSION, /^litellm-[0-9a-f]{12}$/); +}); + +test("every table row carries a provider the alias map can accept", () => { + const acceptable = new Set( + Object.values(LITELLM_PROVIDER_ALIASES) + .filter((v): v is string[] => v !== null) + .flat() + ); + // Extra vendors are vendored deliberately for proxy setups; the invariant is + // that no row is unreachable *because its provider was never in the aliases* + // for a provider that claims to support it. + const rows = Object.values(PRICE_TABLE); + assert.ok(rows.every((r) => typeof r.p === "string" && r.p.length > 0)); + assert.ok(rows.some((r) => acceptable.has(r.p))); +}); + +test("every table row has a usable input price", () => { + for (const [key, row] of Object.entries(PRICE_TABLE)) { + assert.equal(typeof row.i, "number", `${key} has no numeric input price`); + assert.ok(row.i >= 0, `${key} has a negative input price`); + } +}); + +// --------------------------------------------------------------------------- +// Candidate ladder +// --------------------------------------------------------------------------- + +test("candidates try the provider-scoped form before the bare name", () => { + const c = priceCandidates("azure", "gpt-4o-mini"); + assert.equal(c[0], "azure/gpt-4o-mini"); + assert.ok(c.indexOf("gpt-4o-mini") > 0); +}); + +test("candidates include the prefix-stripped form for vendor-prefixed names", () => { + assert.ok(priceCandidates("anthropic", "anthropic/claude-opus-5").includes("claude-opus-5")); +}); + +test("candidates are deduped and empty models yield none", () => { + const c = priceCandidates("deepseek", "deepseek-chat"); + assert.equal(new Set(c).size, c.length); + assert.deepStrictEqual(priceCandidates("openai", " "), []); +}); + +// --------------------------------------------------------------------------- +// Lookup +// --------------------------------------------------------------------------- + +test("prices a bare direct-API model", () => { + const r = lookupPrice("openai", "gpt-4o-mini"); + assert.ok(r); + assert.ok(r.price.inputPerToken > 0); + assert.equal(r.matchedKey, "gpt-4o-mini"); +}); + +test("prices a vendor-prefixed name by stripping the prefix", () => { + const r = lookupPrice("anthropic", "anthropic/claude-opus-5"); + assert.ok(r); + assert.equal(r.matchedKey, "claude-opus-5"); +}); + +test("prices a provider-scoped row when one exists", () => { + const r = lookupPrice("groq", "llama-3.3-70b-versatile"); + assert.ok(r); + assert.equal(r.matchedKey, "groq/llama-3.3-70b-versatile"); +}); + +test("azure resells at its own rate, not OpenAI's", () => { + const azure = lookupPrice("azure", "gpt-4o-mini"); + const openai = lookupPrice("openai", "gpt-4o-mini"); + assert.ok(azure && openai); + assert.equal(azure.matchedKey, "azure/gpt-4o-mini"); + assert.notEqual(azure.price.inputPerToken, openai.price.inputPerToken); +}); + +test("google accepts both the direct-API and Vertex naming of a model", () => { + const r = lookupPrice("google", "gemini-2.0-flash"); + assert.ok(r, "gemini-2.0-flash should price under the google provider"); + assert.ok(r.price.inputPerToken > 0); +}); + +test("a model from the wrong provider is refused rather than mispriced", () => { + // gpt-4o exists, but not on Groq. Matching it would bill Groq usage at + // OpenAI's rates — a confidently wrong number. + assert.equal(lookupPrice("groq", "gpt-4o"), undefined); +}); + +test("an unknown model is unpriced, not free", () => { + assert.equal(lookupPrice("openai", "totally-made-up-model-v9"), undefined); +}); + +test("openai-compatible skips the provider guard so proxied models still price", () => { + const r = lookupPrice("openai-compatible", "deepseek/deepseek-v4-pro"); + assert.ok(r); + assert.ok(r.price.inputPerToken > 0); +}); + +test("openai-compatible resolves a vendor-prefixed Anthropic model", () => { + const r = lookupPrice("openai-compatible", "anthropic/claude-opus-5"); + assert.ok(r); + assert.equal(r.matchedKey, "claude-opus-5"); +}); + +test("the unattributed bucket does not accidentally match a row", () => { + assert.equal(lookupPrice("unknown", "unknown"), undefined); +}); + +// --------------------------------------------------------------------------- +// Cost estimation +// --------------------------------------------------------------------------- + +test("estimateRunCost returns undefined when nothing was spent", () => { + assert.equal(estimateRunCost(undefined), undefined); + assert.equal(estimateRunCost([]), undefined); + assert.equal(estimateRunCost([usage("openai", "gpt-4o-mini", 0, 0)]), undefined); +}); + +test("cost is tokens times the published per-token rate", () => { + const price = lookupPrice("openai", "gpt-4o-mini"); + assert.ok(price); + const cost = estimateRunCost([usage("openai", "gpt-4o-mini", 1_000_000, 500_000)]); + assert.ok(cost); + const expected = 1_000_000 * price.price.inputPerToken + 500_000 * price.price.outputPerToken; + assert.ok(Math.abs(cost.totalUsd - expected) < 1e-9); + assert.equal(cost.byModel[0].source, "table"); + assert.equal(cost.complete, true); + assert.deepStrictEqual(cost.unpricedModels, []); +}); + +test("attacker and judge are priced separately at their own rates", () => { + const cost = estimateRunCost([ + usage("openai-compatible", "deepseek/deepseek-v4-pro", 80_000, 12_000, ["attacker"]), + usage("anthropic", "claude-opus-5", 20_000, 3_000, ["judge"]), + ]); + assert.ok(cost); + assert.equal(cost.byModel.length, 2); + const judge = cost.byModel.find((m) => m.roles.includes("judge")); + const attacker = cost.byModel.find((m) => m.roles.includes("attacker")); + assert.ok(judge?.usd && attacker?.usd); + // Opus is far pricier per token, so despite ~4x fewer tokens it dominates. + assert.ok(judge.usd > attacker.usd); + assert.ok(Math.abs(cost.totalUsd - (judge.usd + attacker.usd)) < 1e-9); +}); + +test("an unpriced model is reported, not silently counted as free", () => { + const cost = estimateRunCost([ + usage("openai", "gpt-4o-mini", 1000, 100), + usage("openai", "totally-made-up-model-v9", 999_999, 999_999), + ]); + assert.ok(cost); + assert.equal(cost.complete, false); + assert.deepStrictEqual(cost.unpricedModels, ["openai:totally-made-up-model-v9"]); + const unpriced = cost.byModel.find((m) => m.model === "totally-made-up-model-v9"); + assert.equal(unpriced?.usd, undefined); + assert.equal(unpriced?.source, "unknown"); + // The total is a lower bound: it reflects only the model we could price. + const priced = cost.byModel.find((m) => m.model === "gpt-4o-mini"); + assert.equal(cost.totalUsd, priced?.usd); +}); + +test("cost records which price snapshot produced it", () => { + const cost = estimateRunCost([usage("openai", "gpt-4o-mini", 10, 10)]); + assert.equal(cost?.priceTableVersion, PRICE_TABLE_VERSION); +}); + +test("the matched table key is retained for auditability", () => { + const cost = estimateRunCost([usage("groq", "llama-3.3-70b-versatile", 10, 10)]); + assert.equal(cost?.byModel[0].matchedKey, "groq/llama-3.3-70b-versatile"); +}); + +// --------------------------------------------------------------------------- +// Display +// --------------------------------------------------------------------------- + +test("sub-cent runs keep enough precision not to read as free", () => { + assert.equal(formatUsd(0), "$0.00"); + // A real per-model cost from a smoke run. Fixed decimals would show "$0.0000". + assert.equal(formatUsd(0.0000305), "$0.00003"); + assert.equal(formatUsd(0.0000355), "$0.000036"); + assert.equal(formatUsd(0.000001), "$0.000001"); + assert.equal(formatUsd(0.0034), "$0.0034"); + assert.equal(formatUsd(0.0099), "$0.0099"); + assert.equal(formatUsd(0.25), "$0.250"); + assert.equal(formatUsd(12.3456), "$12.35"); +}); + +test("amounts too small to show are labelled, never rounded to zero", () => { + assert.equal(formatUsd(0.0000000001), "<$0.000001"); +}); + +test("no displayed amount collapses to a bare zero unless it is truly zero", () => { + for (const v of [1e-9, 1e-7, 1e-6, 3.05e-5, 0.0001, 0.009, 0.5, 3.2]) { + const s = formatUsd(v); + assert.notEqual(s, "$0.00", `${v} rendered as $0.00`); + assert.ok(!/^\$0\.0+$/.test(s), `${v} rendered as all-zero string ${s}`); + } +}); diff --git a/core/tests/tokenTracker.test.ts b/core/tests/tokenTracker.test.ts index 82957ddb..9ed369a9 100644 --- a/core/tests/tokenTracker.test.ts +++ b/core/tests/tokenTracker.test.ts @@ -126,3 +126,109 @@ test("parseUsage strips unknown provider metadata and returns normalized usage", }); assert.deepStrictEqual(result, { inputTokens: 100, outputTokens: 20, totalTokens: 150 }); }); + +// --------------------------------------------------------------------------- +// Per-model attribution +// --------------------------------------------------------------------------- + +const ATTACKER = { provider: "openai-compatible", model: "deepseek/deepseek-v4-pro" }; +const JUDGE = { provider: "anthropic", model: "claude-opus-5" }; + +test("fresh tracker has an empty breakdown", () => { + const t = new TokenTracker(); + assert.deepStrictEqual(t.breakdown, []); +}); + +test("usage recorded without attribution lands in the unknown bucket, not dropped", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 100, outputTokens: 20 }); + assert.equal(t.breakdown.length, 1); + assert.equal(t.breakdown[0].key, "unknown"); + assert.equal(t.breakdown[0].totalTokens, 120); + assert.deepStrictEqual(t.totals, { inputTokens: 100, outputTokens: 20, totalTokens: 120 }); +}); + +test("attributed usage is keyed by provider and model", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 100, outputTokens: 20 }, { model: ATTACKER, role: "attacker" }); + const [b] = t.breakdown; + assert.equal(b.key, "openai-compatible:deepseek/deepseek-v4-pro"); + assert.equal(b.provider, "openai-compatible"); + assert.equal(b.model, "deepseek/deepseek-v4-pro"); + assert.deepStrictEqual(b.roles, ["attacker"]); + assert.equal(b.calls, 1); +}); + +test("two models are tracked separately and still sum to run totals", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 80_000, outputTokens: 12_000 }, { model: ATTACKER, role: "attacker" }); + t.record({ inputTokens: 20_000, outputTokens: 3_000 }, { model: JUDGE, role: "judge" }); + + assert.equal(t.breakdown.length, 2); + const sum = t.breakdown.reduce((n, b) => n + b.totalTokens, 0); + assert.equal(sum, t.totals.totalTokens); + assert.equal(t.totals.totalTokens, 115_000); +}); + +test("breakdown is ordered heaviest first", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 10, outputTokens: 1 }, { model: JUDGE }); + t.record({ inputTokens: 500, outputTokens: 50 }, { model: ATTACKER }); + assert.equal(t.breakdown[0].model, "deepseek/deepseek-v4-pro"); +}); + +test("one model used for both phases accumulates both roles, deduped and sorted", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 10, outputTokens: 2 }, { model: JUDGE, role: "judge" }); + t.record({ inputTokens: 10, outputTokens: 2 }, { model: JUDGE, role: "attacker" }); + t.record({ inputTokens: 10, outputTokens: 2 }, { model: JUDGE, role: "judge" }); + + assert.equal(t.breakdown.length, 1); + assert.deepStrictEqual(t.breakdown[0].roles, ["attacker", "judge"]); + assert.equal(t.breakdown[0].calls, 3); +}); + +test("roles are normalized to lowercase so 'Judge' and 'judge' do not split", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 10, outputTokens: 2 }, { model: JUDGE, role: "Judge" }); + t.record({ inputTokens: 10, outputTokens: 2 }, { model: JUDGE, role: "judge" }); + assert.deepStrictEqual(t.breakdown[0].roles, ["judge"]); +}); + +test("attributed and unattributed usage coexist without losing tokens", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 100, outputTokens: 20 }, { model: ATTACKER, role: "attacker" }); + t.record({ inputTokens: 7, outputTokens: 3 }); + const sum = t.breakdown.reduce((n, b) => n + b.totalTokens, 0); + assert.equal(sum, t.totals.totalTokens); + assert.ok(t.breakdown.some((b) => b.key === "unknown")); +}); + +test("child tracker propagates attribution to its parent, not just totals", () => { + const parent = new TokenTracker(); + const child = parent.child(); + child.record({ inputTokens: 100, outputTokens: 20 }, { model: JUDGE, role: "judge" }); + + assert.equal(child.breakdown[0].key, "anthropic:claude-opus-5"); + assert.equal(parent.breakdown[0].key, "anthropic:claude-opus-5"); + assert.deepStrictEqual(parent.breakdown[0].roles, ["judge"]); +}); + +test("sibling children merge into one parent bucket per model", () => { + const parent = new TokenTracker(); + const a = parent.child(); + const b = parent.child(); + a.record({ inputTokens: 100, outputTokens: 20 }, { model: ATTACKER, role: "attacker" }); + b.record({ inputTokens: 200, outputTokens: 40 }, { model: ATTACKER, role: "attacker" }); + + assert.equal(parent.breakdown.length, 1); + assert.equal(parent.breakdown[0].calls, 2); + assert.equal(parent.breakdown[0].totalTokens, 360); +}); + +test("explicit totalTokens is preserved in the per-model bucket too", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 100, outputTokens: 20, totalTokens: 150 }, { model: JUDGE }); + assert.equal(t.breakdown[0].totalTokens, 150); + assert.equal(t.totals.totalTokens, 150); +}); diff --git a/package.json b/package.json index 882c8801..4f346950 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "format:check": "prettier --check .", "validate:skills": "tsx scripts/validate-skills.ts", "build:catalog": "tsx scripts/build-catalog.ts", + "build:pricing": "tsx scripts/build-pricing.ts", "build:catalog:check": "tsx scripts/build-catalog.ts --check", "test:catalog-generators": "node --import tsx/esm --test scripts/tests/*.test.mjs scripts/tests/*.test.ts runners/extension/tests/*.test.mjs", "smoke:pack": "tsx scripts/smoke-pack.ts", diff --git a/runners/cli/src/commands/run.ts b/runners/cli/src/commands/run.ts index 96edf824..64eecc03 100644 --- a/runners/cli/src/commands/run.ts +++ b/runners/cli/src/commands/run.ts @@ -7,6 +7,7 @@ import { writeReport } from "@keyvaluesystems/agent-opfor-core/report/buildRepor import type { RunConfig } from "@keyvaluesystems/agent-opfor-core/execute/types.js"; import { parseRunConfig } from "@keyvaluesystems/agent-opfor-core/config/schema.js"; import { normalizeEffort } from "@keyvaluesystems/agent-opfor-core/execute/effortCompat.js"; +import { formatUsd } from "@keyvaluesystems/agent-opfor-core/pricing/estimateCost.js"; import { runSetupAndWrite } from "./setup.js"; import { ensureOpforDirs, OPFOR_DIR, OPFOR_REPORTS_DIR } from "../lib/artifacts.js"; import { ConsoleProgressListener } from "../lib/consoleProgressListener.js"; @@ -260,6 +261,19 @@ export function registerRunCommand(program: Command): void { `Token usage: ${inputTokens.toLocaleString()} input / ${outputTokens.toLocaleString()} output (${totalTokens.toLocaleString()} total)` ); } + if (summary.cost) { + // "Testing cost", not "total cost" — the target's own inference spend is + // invisible to opfor, and an unpriced model makes this a lower bound. + const note = summary.cost.complete + ? "" + : ` (lower bound — unpriced: ${summary.cost.unpricedModels.join(", ")})`; + log.info(`Testing cost: ≈${formatUsd(summary.cost.totalUsd)}${note}`); + for (const m of summary.cost.byModel) { + const amount = m.usd === undefined ? "not priced" : `≈${formatUsd(m.usd)}`; + const roles = m.roles.length ? ` [${m.roles.join(", ")}]` : ""; + log.info(` ${m.model}${roles}: ${amount}`); + } + } log.success(`\nReport: ${html}`); log.info(` JSON: ${json}`); diff --git a/scripts/build-pricing.ts b/scripts/build-pricing.ts new file mode 100644 index 00000000..fb4dac26 --- /dev/null +++ b/scripts/build-pricing.ts @@ -0,0 +1,211 @@ +/** + * Generate the vendored model price table. + * + * Downloads LiteLLM's community price map, prunes it to the providers opfor can + * actually reach, and writes core/src/pricing/priceTable.generated.ts. + * + * Why vendored rather than fetched at runtime: opfor runs in air-gapped CI and + * locked-down corporate networks, and a security report should produce the same + * numbers when re-run months later. Pruned, the table is ~60 KB — small enough + * to inline, which also lets the browser extension use it (esbuild bundles the + * generated module; nothing here touches node:fs at runtime). + * + * Usage: + * npm run build:pricing # refresh the table + * npm run build:pricing -- --check # exit 1 if the table differs from upstream + * + * NOTE: --check is intentionally NOT a blocking CI gate. Unlike the evaluator + * catalog (which goes stale only when this repo changes), this table goes stale + * when a third party reprices a model — gating merges on that would let outside + * events break the build. Run it on a schedule and open a PR instead. + */ + +import { createHash } from "node:crypto"; +import { readFile, writeFile, mkdir } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { VENDORED_LITELLM_PROVIDERS } from "../core/src/pricing/providerAliases.js"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, ".."); +const OUT_FILE = path.join(REPO_ROOT, "core/src/pricing/priceTable.generated.ts"); +const CHECK_ONLY = process.argv.includes("--check"); + +const SOURCE_URL = + "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"; + +/** Upstream entry shape — only the fields we consume; everything else is ignored. */ +interface UpstreamEntry { + litellm_provider?: string; + mode?: string; + input_cost_per_token?: number; + output_cost_per_token?: number; + cache_read_input_token_cost?: number; + cache_creation_input_token_cost?: number; +} + +/** Compact on-disk shape. Short keys because this file is generated, not read. */ +interface CompactPrice { + /** litellm_provider */ + p: string; + /** input cost per token */ + i: number; + /** output cost per token */ + o: number; + /** cache-read cost per token */ + cr?: number; + /** cache-write cost per token */ + cw?: number; +} + +/** Only chat-shaped models can be an attacker or a judge; skip image/audio/embedding rows. */ +const CHAT_MODES = new Set(["chat", "responses"]); + +function prune(raw: Record): { + table: Record; + droppedProviders: Map; +} { + const keep = new Set(VENDORED_LITELLM_PROVIDERS); + const table: Record = {}; + const droppedProviders = new Map(); + + for (const [key, value] of Object.entries(raw)) { + if (!value || typeof value !== "object") continue; + const e = value as UpstreamEntry; + const provider = e.litellm_provider; + if (!provider) continue; + + if (!keep.has(provider)) { + droppedProviders.set(provider, (droppedProviders.get(provider) ?? 0) + 1); + continue; + } + if (!e.mode || !CHAT_MODES.has(e.mode)) continue; + // A row without an input price cannot produce a cost; drop it rather than + // let it match and silently price at zero. + if (typeof e.input_cost_per_token !== "number") continue; + + const entry: CompactPrice = { + p: provider, + i: e.input_cost_per_token, + o: typeof e.output_cost_per_token === "number" ? e.output_cost_per_token : 0, + }; + if (typeof e.cache_read_input_token_cost === "number") { + entry.cr = e.cache_read_input_token_cost; + } + if (typeof e.cache_creation_input_token_cost === "number") { + entry.cw = e.cache_creation_input_token_cost; + } + table[key] = entry; + } + + return { table, droppedProviders }; +} + +/** Stable stringify: sorted keys so an unchanged upstream yields an identical file. */ +function serializeTable(table: Record): string { + const keys = Object.keys(table).sort(); + const lines = keys.map((k) => ` ${JSON.stringify(k)}: ${JSON.stringify(table[k])},`); + return lines.join("\n"); +} + +function renderModule(body: string, version: string, entryCount: number): string { + return `/** + * GENERATED FILE — do not edit by hand. + * + * Model prices in USD per token, pruned from LiteLLM's community price map: + * ${SOURCE_URL} + * + * Regenerate with: npm run build:pricing + * + * Entries: ${entryCount} + * + * Contains no Node imports so the browser extension can bundle it. + */ + +/** Compact price row. Short keys keep the generated table small. */ +export interface CompactPrice { + /** Upstream \`litellm_provider\` — used to reject a match from the wrong vendor. */ + p: string; + /** USD per input token. */ + i: number; + /** USD per output token. */ + o: number; + /** USD per cached input token, when published. */ + cr?: number; + /** USD per cache-write token, when published. */ + cw?: number; +} + +/** Identifies this snapshot; recorded in reports so a cost figure is reproducible. */ +export const PRICE_TABLE_VERSION = ${JSON.stringify(version)}; + +export const PRICE_TABLE: Record = { +${body} +}; +`; +} + +async function main(): Promise { + process.stdout.write(`[build-pricing] fetching ${SOURCE_URL}\n`); + const res = await fetch(SOURCE_URL); + if (!res.ok) { + throw new Error( + `Failed to download the price map: HTTP ${res.status}. ` + + `Check network access, or retry later — the vendored table is still usable meanwhile.` + ); + } + const rawText = await res.text(); + const raw = JSON.parse(rawText) as Record; + + const { table, droppedProviders } = prune(raw); + const entryCount = Object.keys(table).length; + if (entryCount === 0) { + throw new Error( + "Pruning produced an empty table — upstream may have changed shape. " + + "Refusing to overwrite the vendored table with nothing." + ); + } + + const body = serializeTable(table); + // Version off the pruned content, not the fetch time: regenerating against an + // unchanged upstream must be a no-op diff. + const version = `litellm-${createHash("sha256").update(body).digest("hex").slice(0, 12)}`; + const contents = renderModule(body, version, entryCount); + + const existing = await readFile(OUT_FILE, "utf8").catch(() => null); + + process.stdout.write( + `[build-pricing] upstream ${Object.keys(raw).length} entries -> kept ${entryCount} ` + + `across ${VENDORED_LITELLM_PROVIDERS.length} providers (${(body.length / 1024).toFixed(1)} KB)\n` + ); + // Say what was left out — a silent prune reads as full coverage when it isn't. + const topDropped = [...droppedProviders.entries()].sort((a, b) => b[1] - a[1]).slice(0, 5); + process.stdout.write( + `[build-pricing] dropped providers (top 5): ${topDropped.map(([p, n]) => `${p}=${n}`).join(", ")}\n` + ); + + if (CHECK_ONLY) { + if (existing !== contents) { + process.stderr.write( + `[build-pricing] STALE — ${OUT_FILE} differs from upstream. Run: npm run build:pricing\n` + ); + process.exit(1); + } + process.stdout.write(`[build-pricing] up to date (${version})\n`); + return; + } + + if (existing === contents) { + process.stdout.write(`[build-pricing] unchanged (${version})\n`); + return; + } + + await mkdir(path.dirname(OUT_FILE), { recursive: true }); + await writeFile(OUT_FILE, contents, "utf8"); + process.stdout.write(`[build-pricing] wrote ${OUT_FILE} (${version})\n`); +} + +main().catch((err: unknown) => { + process.stderr.write(`[build-pricing] ${err instanceof Error ? err.message : String(err)}\n`); + process.exit(1); +}); From 35a35211f20070b94ea4f8de40d8da55ca326be8 Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Tue, 4 Aug 2026 16:30:59 +0530 Subject: [PATCH 2/6] fix(pricing): address review findings on cost estimation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the review on #231. build:pricing --check was permanently broken. The generator emitted compact JSON.stringify output while the pre-commit Prettier hook reformatted the committed file, so the two never matched: --check reported "stale" even straight after regenerating, and the "unchanged -> no-op" fast path never fired, turning every run into a 557-line diff. The script now formats its output with the repo's Prettier config before comparing or writing. MCP baseline scans judged every tool description and resource without a token tracker, so their spend was invisible. That understated cost on every MCP run while `complete: true` claimed full coverage — the tokens never arrived, so there was no bucket to flag as unpriced. runAll now threads the run tracker into runBaselineScans. Also: - RunCost.complete documents that it means "nothing we saw was unpriced", not "nothing was missed" — a call site that reports no usage produces no bucket to flag. - costSubLabel labels a role-less bucket "unknown" rather than "mixed", which implied several phases where there were none. - Drop the PRICE_TABLE_VERSION re-export from lookupPrice; importers now take it from the module that owns it (AGENTS.md: no barrel re-exports). - Cover roleFromContext against the literal context labels used at call sites, so renaming a log string can't silently drop the role split. - Add the missing end-to-end test: createModel -> withRetry -> correct bucket, exercising the WeakMap identity registry that every other test bypasses with hand-built identities. Co-Authored-By: Claude Opus 5 --- core/src/browser.ts | 3 +- core/src/execute/baselineScanner.ts | 9 ++++ core/src/execute/runAll.ts | 1 + core/src/pricing/estimateCost.ts | 3 +- core/src/pricing/lookupPrice.ts | 2 - core/src/pricing/types.ts | 10 ++++- core/tests/modelIdentity.test.ts | 70 +++++++++++++++++++++++++++++ core/tests/pricing.test.ts | 4 +- scripts/build-pricing.ts | 10 ++++- 9 files changed, 104 insertions(+), 8 deletions(-) diff --git a/core/src/browser.ts b/core/src/browser.ts index 8281a17e..0809310e 100644 --- a/core/src/browser.ts +++ b/core/src/browser.ts @@ -72,7 +72,8 @@ export type { LlmConfig, ProviderName } from "./config/types.js"; // Cost estimation. The vendored price table is a plain inlined module with no // Node imports, so esbuild bundles it for the extension like any other source. export { estimateRunCost, formatUsd } from "./pricing/estimateCost.js"; -export { lookupPrice, PRICE_TABLE_VERSION } from "./pricing/lookupPrice.js"; +export { lookupPrice } from "./pricing/lookupPrice.js"; +export { PRICE_TABLE_VERSION } from "./pricing/priceTable.generated.js"; export type { RunCost, ModelCost, ModelPrice, CostSource } from "./pricing/types.js"; export type { ModelTokenUsage, TokenUsage } from "./execute/tokenTracker.js"; diff --git a/core/src/execute/baselineScanner.ts b/core/src/execute/baselineScanner.ts index 6707a2d3..91f3e5ce 100644 --- a/core/src/execute/baselineScanner.ts +++ b/core/src/execute/baselineScanner.ts @@ -8,6 +8,7 @@ import { createHash } from "node:crypto"; import { z } from "zod"; import { randomUUID } from "../lib/random.js"; import { judgeToolResponse } from "../run/judge.js"; +import type { TokenTracker } from "./tokenTracker.js"; import { errorJudge as mcpErrorJudge } from "../lib/judgeTypes.js"; import { toEvaluatorResult } from "./aggregate.js"; import { log } from "../lib/logger.js"; @@ -24,6 +25,12 @@ export interface BaselineScanContext { config: RunConfig; outputDir?: string; notify: (event: ProgressEvent) => void; + /** + * Run-level token accumulator. Baseline scans judge every tool description and + * resource before the evaluator loop starts, so without this their spend is + * invisible to the run's token count and cost estimate. + */ + tokenTracker?: TokenTracker; } /** @@ -132,6 +139,7 @@ async function scanResources(ctx: BaselineScanContext): Promise try { judgeResult = await judgeToolResponse({ model: judgeModelConfig, + tokenTracker: ctx.tokenTracker, evaluator: { id: evalId, name: "MCP Resource Exposure", @@ -186,6 +194,7 @@ async function scanToolDescriptions(ctx: BaselineScanContext): Promise test("resolveModelIdentity returns undefined for an object with nothing identifying", () => { assert.equal(resolveModelIdentity({} as unknown as LanguageModel), undefined); }); + +// --------------------------------------------------------------------------- +// Role derivation + end-to-end attribution +// +// roleFromContext reads the free-text label withRetry already takes for logging. +// That coupling is load-bearing but invisible: renaming a log string would +// silently drop the attacker/judge split from every report. These pin it. +// --------------------------------------------------------------------------- + +test("roleFromContext maps the labels actually used at call sites", () => { + // These four strings are the literal `context:` values passed by + // generateAttacks.ts, judge.ts and withRetry's default. Changing one without + // updating roleFromContext silently removes roles from the cost breakdown. + assert.equal(roleFromContext("Attacker"), "attacker"); + assert.equal(roleFromContext("Attacker (MCP)"), "attacker"); + assert.equal(roleFromContext("Judge"), "judge"); + assert.equal(roleFromContext("LLM"), undefined); +}); + +test("roleFromContext is case- and whitespace-insensitive, and safe on empty", () => { + assert.equal(roleFromContext(" jUdGe "), "judge"); + assert.equal(roleFromContext(undefined), undefined); + assert.equal(roleFromContext(" "), undefined); +}); + +test("withRetry attributes usage to the model built by createModel", async () => { + // The load-bearing path: createModel registers identity in a WeakMap, and + // withRetry resolves it from the model object alone. Every unit test above + // uses hand-built identities, so this is the only check that the real + // factory -> tracker chain works. + process.env.MODEL_IDENTITY_TEST_KEY = "dummy-key-not-used"; + const llm: LlmConfig = { + provider: "openai-compatible", + model: "deepseek/deepseek-v4-pro", + apiKeyEnv: "MODEL_IDENTITY_TEST_KEY", + baseURL: "https://example.invalid/v1", + }; + const model = createModel(llm); + + const tracker = new TokenTracker(); + // withRetry auto-records from a successful result's `.usage`; no network needed. + await withRetry(async () => ({ usage: { inputTokens: 120, outputTokens: 30 } }), { + context: "Judge", + tokenTracker: tracker, + model, + }); + + assert.equal(tracker.breakdown.length, 1); + const [bucket] = tracker.breakdown; + // Provider survives as the CONFIGURED name, not the SDK's "custom" — this is + // the whole reason the identity registry exists. + assert.equal(bucket.key, "openai-compatible:deepseek/deepseek-v4-pro"); + assert.equal(bucket.provider, "openai-compatible"); + assert.deepStrictEqual(bucket.roles, ["judge"]); + assert.equal(bucket.totalTokens, 150); +}); + +test("withRetry without a model still counts the tokens, unattributed", async () => { + const tracker = new TokenTracker(); + await withRetry(async () => ({ usage: { inputTokens: 10, outputTokens: 5 } }), { + context: "Attacker", + tokenTracker: tracker, + }); + assert.equal(tracker.totals.totalTokens, 15); + assert.equal(tracker.breakdown[0].key, "unknown"); + assert.deepStrictEqual(tracker.breakdown[0].roles, ["attacker"]); +}); diff --git a/core/tests/pricing.test.ts b/core/tests/pricing.test.ts index 3efb001a..a30f9ff7 100644 --- a/core/tests/pricing.test.ts +++ b/core/tests/pricing.test.ts @@ -1,8 +1,8 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { lookupPrice, priceCandidates, PRICE_TABLE_VERSION } from "../src/pricing/lookupPrice.js"; +import { lookupPrice, priceCandidates } from "../src/pricing/lookupPrice.js"; import { estimateRunCost, formatUsd } from "../src/pricing/estimateCost.js"; -import { PRICE_TABLE } from "../src/pricing/priceTable.generated.js"; +import { PRICE_TABLE, PRICE_TABLE_VERSION } from "../src/pricing/priceTable.generated.js"; import { LITELLM_PROVIDER_ALIASES } from "../src/pricing/providerAliases.js"; import type { ModelTokenUsage } from "../src/execute/tokenTracker.js"; diff --git a/scripts/build-pricing.ts b/scripts/build-pricing.ts index fb4dac26..aeeeec8c 100644 --- a/scripts/build-pricing.ts +++ b/scripts/build-pricing.ts @@ -24,6 +24,7 @@ import { createHash } from "node:crypto"; import { readFile, writeFile, mkdir } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { format, resolveConfig } from "prettier"; import { VENDORED_LITELLM_PROVIDERS } from "../core/src/pricing/providerAliases.js"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -170,7 +171,14 @@ async function main(): Promise { // Version off the pruned content, not the fetch time: regenerating against an // unchanged upstream must be a no-op diff. const version = `litellm-${createHash("sha256").update(body).digest("hex").slice(0, 12)}`; - const contents = renderModule(body, version, entryCount); + // Run the rendered module through the repo's Prettier config before comparing + // or writing. Without this the committed file (reformatted by the pre-commit + // hook) never equals this script's raw output, which would make `--check` + // report "stale" forever and turn every regeneration into a full-file diff. + const contents = await format(renderModule(body, version, entryCount), { + ...(await resolveConfig(OUT_FILE)), + parser: "typescript", + }); const existing = await readFile(OUT_FILE, "utf8").catch(() => null); From fde4dcc905a7a17a0ac2dab45d718b616ca521d3 Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Tue, 4 Aug 2026 16:37:43 +0530 Subject: [PATCH 3/6] fix(pricing): never render an unpriced run as $0.00 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses CodeRabbit's review on #231. totalUsd sums only the models that could be priced, so a run where none of them could legitimately totals 0 — and formatUsd(0) is "$0.00". Both cost displays called it unconditionally, so a run with entirely unknown cost rendered identically to one that genuinely cost nothing. That is the exact failure the pricing module documents itself as preventing; the sub-label said "0 of 1 models priced" while the headline number said free. formatCostDisplay now distinguishes all three cases: "≈" for a complete estimate, "≥" for a partial total that is a real floor, and "unpriced" when nothing could be priced at all. Applied to the executive-summary cost card and the per-evaluator span, with regression tests covering each case — including that a genuinely free run may still show $0.00. Also validate the upstream price map with Zod instead of casting JSON.parse output (AGENTS.md: Zod for all external input). Rows are parsed individually so one malformed entry cannot fail the build, and skipped rows are reported rather than folded into the drop count; a wholesale format change now fails loudly instead of silently producing an empty table. Output is byte-identical — same 557 entries, same version hash. Co-Authored-By: Claude Opus 5 --- core/src/report/render.ts | 18 ++++++++++-- core/tests/pricing.test.ts | 41 ++++++++++++++++++++++++++ scripts/build-pricing.ts | 60 +++++++++++++++++++++++++++++--------- 3 files changed, 104 insertions(+), 15 deletions(-) diff --git a/core/src/report/render.ts b/core/src/report/render.ts index 96eef2a2..1f67eecb 100644 --- a/core/src/report/render.ts +++ b/core/src/report/render.ts @@ -25,6 +25,20 @@ import { import type { RunCost } from "../pricing/types.js"; import { formatUsd } from "../pricing/estimateCost.js"; +/** + * Cost for display, carrying its own confidence. + * + * `totalUsd` sums only the models that could be priced, so when none of them + * could it is legitimately `0` — and `formatUsd(0)` is `"$0.00"`, which reads as + * "this run was free" rather than "we don't know". Distinguish the three cases + * at the point of display: `≈` for a complete estimate, `≥` for a partial one + * that is genuinely a floor, and no number at all when nothing was priced. + */ +export function formatCostDisplay(cost: RunCost): string { + if (cost.complete) return `≈${formatUsd(cost.totalUsd)}`; + return cost.totalUsd > 0 ? `≥${formatUsd(cost.totalUsd)}` : "unpriced"; +} + /** * Sub-label for the cost card: the attacker/judge split when both are present * (the number people act on — "the judge is most of the spend"), otherwise a @@ -160,7 +174,7 @@ export function renderReport(model: ReportViewModel): string {
- ${e.tokenUsage ? `${formatTokenCount(e.tokenUsage.totalTokens)} tokens${e.cost ? ` · ≈${formatUsd(e.cost.totalUsd)}` : ""}|` : ""} + ${e.tokenUsage ? `${formatTokenCount(e.tokenUsage.totalTokens)} tokens${e.cost ? ` · ${formatCostDisplay(e.cost)}` : ""}|` : ""} ${evalVerdict === "PASS" ? "Pass" : evalVerdict === "ERROR" ? "Error" : "Fail"} Safety score: ${avgScore ?? "—"}/10
@@ -469,7 +483,7 @@ export function renderReport(model: ReportViewModel): string { summary.cost ? `
Testing Cost
-
≈${formatUsd(summary.cost.totalUsd)}
+
${formatCostDisplay(summary.cost)}
${esc(costSubLabel(summary.cost))}
` : "" diff --git a/core/tests/pricing.test.ts b/core/tests/pricing.test.ts index a30f9ff7..e6355d85 100644 --- a/core/tests/pricing.test.ts +++ b/core/tests/pricing.test.ts @@ -5,6 +5,8 @@ import { estimateRunCost, formatUsd } from "../src/pricing/estimateCost.js"; import { PRICE_TABLE, PRICE_TABLE_VERSION } from "../src/pricing/priceTable.generated.js"; import { LITELLM_PROVIDER_ALIASES } from "../src/pricing/providerAliases.js"; import type { ModelTokenUsage } from "../src/execute/tokenTracker.js"; +import type { RunCost } from "../src/pricing/types.js"; +import { formatCostDisplay } from "../src/report/render.js"; /** Build a token-breakdown row without repeating the boilerplate. */ function usage( @@ -229,3 +231,42 @@ test("no displayed amount collapses to a bare zero unless it is truly zero", () assert.ok(!/^\$0\.0+$/.test(s), `${v} rendered as all-zero string ${s}`); } }); + +// --------------------------------------------------------------------------- +// Cost display confidence +// +// Regression: totalUsd sums only *priced* models, so a run where nothing could +// be priced legitimately totals 0 — and rendering that as "$0.00" makes an +// unknown cost look free, which is the exact failure the design forbids. +// --------------------------------------------------------------------------- + +/** Minimal RunCost for display tests. */ +function runCost(totalUsd: number, unpriced: string[]): RunCost { + return { + totalUsd, + currency: "USD", + byModel: [], + unpricedModels: unpriced, + complete: unpriced.length === 0, + priceTableVersion: PRICE_TABLE_VERSION, + }; +} + +test("a fully-priced run reads as an estimate", () => { + assert.equal(formatCostDisplay(runCost(1.25, [])), "≈$1.25"); +}); + +test("a partially-priced run reads as a floor, not an estimate", () => { + assert.equal(formatCostDisplay(runCost(1.25, ["openai:mystery"])), "≥$1.25"); +}); + +test("a run where nothing could be priced never renders as $0.00", () => { + const display = formatCostDisplay(runCost(0, ["openai:mystery"])); + assert.equal(display, "unpriced"); + assert.ok(!display.includes("0.00"), "unknown cost must not look free"); +}); + +test("a genuinely free run is still allowed to show zero", () => { + // Everything priced, everything free (e.g. a self-hosted model at $0). + assert.equal(formatCostDisplay(runCost(0, [])), "≈$0.00"); +}); diff --git a/scripts/build-pricing.ts b/scripts/build-pricing.ts index aeeeec8c..aee11a8b 100644 --- a/scripts/build-pricing.ts +++ b/scripts/build-pricing.ts @@ -25,6 +25,7 @@ import { readFile, writeFile, mkdir } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { format, resolveConfig } from "prettier"; +import { z } from "zod"; import { VENDORED_LITELLM_PROVIDERS } from "../core/src/pricing/providerAliases.js"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -35,15 +36,26 @@ const CHECK_ONLY = process.argv.includes("--check"); const SOURCE_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"; -/** Upstream entry shape — only the fields we consume; everything else is ignored. */ -interface UpstreamEntry { - litellm_provider?: string; - mode?: string; - input_cost_per_token?: number; - output_cost_per_token?: number; - cache_read_input_token_cost?: number; - cache_creation_input_token_cost?: number; -} +/** + * Upstream entry shape — only the fields we consume; everything else is ignored. + * + * The map is third-party data fetched over the network, so it is validated + * rather than cast (AGENTS.md: Zod for all external input). Rows are parsed + * individually and a row that fails is skipped and counted, so one malformed + * entry upstream cannot fail the whole build — but a wholesale format change + * shows up as a collapsed row count and trips the empty-table guard below. + */ +const UpstreamEntrySchema = z.object({ + litellm_provider: z.string().min(1).optional(), + mode: z.string().optional(), + input_cost_per_token: z.number().nonnegative().optional(), + output_cost_per_token: z.number().nonnegative().optional(), + cache_read_input_token_cost: z.number().nonnegative().optional(), + cache_creation_input_token_cost: z.number().nonnegative().optional(), +}); + +/** The document itself must be an object of named entries. */ +const UpstreamMapSchema = z.record(z.string(), z.unknown()); /** Compact on-disk shape. Short keys because this file is generated, not read. */ interface CompactPrice { @@ -65,14 +77,21 @@ const CHAT_MODES = new Set(["chat", "responses"]); function prune(raw: Record): { table: Record; droppedProviders: Map; + malformed: string[]; } { const keep = new Set(VENDORED_LITELLM_PROVIDERS); const table: Record = {}; const droppedProviders = new Map(); + const malformed: string[] = []; for (const [key, value] of Object.entries(raw)) { if (!value || typeof value !== "object") continue; - const e = value as UpstreamEntry; + const parsed = UpstreamEntrySchema.safeParse(value); + if (!parsed.success) { + malformed.push(key); + continue; + } + const e = parsed.data; const provider = e.litellm_provider; if (!provider) continue; @@ -99,7 +118,7 @@ function prune(raw: Record): { table[key] = entry; } - return { table, droppedProviders }; + return { table, droppedProviders, malformed }; } /** Stable stringify: sorted keys so an unchanged upstream yields an identical file. */ @@ -156,9 +175,16 @@ async function main(): Promise { ); } const rawText = await res.text(); - const raw = JSON.parse(rawText) as Record; + const parsedDoc = UpstreamMapSchema.safeParse(JSON.parse(rawText)); + if (!parsedDoc.success) { + throw new Error( + `The downloaded price map is not an object of named entries — upstream may have ` + + `changed format. Refusing to regenerate from it. (${parsedDoc.error.issues[0]?.message})` + ); + } + const raw = parsedDoc.data; - const { table, droppedProviders } = prune(raw); + const { table, droppedProviders, malformed } = prune(raw); const entryCount = Object.keys(table).length; if (entryCount === 0) { throw new Error( @@ -191,6 +217,14 @@ async function main(): Promise { process.stdout.write( `[build-pricing] dropped providers (top 5): ${topDropped.map(([p, n]) => `${p}=${n}`).join(", ")}\n` ); + // Surface schema rejections rather than folding them into the drop count — + // a sudden spike means upstream changed a field's type on us. + if (malformed.length > 0) { + process.stdout.write( + `[build-pricing] ${malformed.length} row(s) failed validation and were skipped: ` + + `${malformed.slice(0, 5).join(", ")}${malformed.length > 5 ? ", …" : ""}\n` + ); + } if (CHECK_ONLY) { if (existing !== contents) { From 7457ccdefcdbe5012cd607c2f8466f3fe9106fff Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Tue, 4 Aug 2026 17:02:20 +0530 Subject: [PATCH 4/6] docs(pricing): document testing cost and scale display precision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cost precision now scales in three bands rather than two. Between a cent and a dime the third decimal is what distinguishes one evaluator from another — $0.037 vs $0.049 is a third more expensive, but both round to the same two-decimal figure — so that band keeps it. From a dime up the precision is noise, so a run total reads as money ($0.18, not $0.177). Sub-cent amounts keep two significant figures as before, since a fixed decimal count renders them "$0.0000" and they read as free. docs/cli.md said "No cost estimation is performed", which this feature made false. That section is now "Token usage and testing cost" and covers the per-model split, where prices come from, and the accuracy caveats — chiefly that multi-turn runs read high, because providers discount repeated context and opfor prices every input token at full rate. Users budgeting off this number should know it is conservative. Also documents that "testing cost" is opfor's own spend and excludes the target's inference cost, and that an unpriced model is never counted as free. Same summary added to the README and referenced from the browser extension guide. Note: the SDK's report type exposes neither tokenUsage nor cost. That gap predates this work and is left alone here. Co-Authored-By: Claude Opus 5 --- README.md | 20 ++++++++++++++++++++ core/src/pricing/estimateCost.ts | 18 +++++++++++------- core/tests/pricing.test.ts | 16 +++++++++++++++- docs/browser-extension.md | 2 +- docs/cli.md | 27 ++++++++++++++++++++++----- 5 files changed, 69 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3a504769..ff909f05 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,26 @@ When you run a scan, opfor: Each run lands in its own subfolder under `.opfor/reports/run-report---/` containing `-report.html` and `-report.json`. Autonomous `opfor hunt` runs use the same layout under `hunt-report---/`. +### Testing cost + +Every run reports what it cost to run, broken down by model: + +``` +Token usage: 51,323 input / 6,057 output (57,380 total) +Testing cost: ≈$0.18 + deepseek/deepseek-v4-pro [attacker]: ≈$0.037 + anthropic/claude-opus-5 [judge]: ≈$0.14 +``` + +This is **opfor's own spend** — the attacker and judge LLMs. It excludes your target's inference cost, which opfor cannot see from the outside. The per-model split is the useful part: the judge is often the bigger share, and pointing it at a cheaper model is usually the easiest saving. + +Prices come from a snapshot of LiteLLM's public price map that ships with the package, so runs work offline and a report re-rendered later produces the same figure. Two caveats worth knowing: + +- **Multi-turn runs read high.** Providers discount repeated context, and a multi-turn attack re-sends the conversation each turn — opfor prices every input token at full rate, so the real bill is usually lower. +- **Unknown models are never counted as free.** A model missing from the price table is reported as unpriced and the total is marked a lower bound, rather than silently reading as $0. + +→ [Token usage and testing cost](docs/cli.md#token-usage-and-testing-cost) + ## Evaluator coverage Opfor ships with curated suites that map to industry standards. Pick a suite or run individual evaluators. diff --git a/core/src/pricing/estimateCost.ts b/core/src/pricing/estimateCost.ts index 00ccb80e..4452a0a3 100644 --- a/core/src/pricing/estimateCost.ts +++ b/core/src/pricing/estimateCost.ts @@ -68,13 +68,17 @@ export function estimateRunCost(breakdown?: ModelTokenUsage[]): RunCost | undefi } /** - * Format a USD amount for display. + * Format a USD amount for display, with precision that scales to the amount. * - * Fractions of a cent are the norm here — a smoke suite on a cheap model lands - * around $0.00003 — and a fixed number of decimals renders those as "$0.0000", - * which reads as free. So small amounts are shown to two significant figures - * instead, and anything below a millionth of a dollar is labelled as such rather - * than rounded away. + * Three bands, each solving a different problem: + * - Below a cent, a fixed decimal count renders "$0.0000" and reads as free, + * so these show two significant figures instead ("$0.0034"). + * - Between a cent and a dime, the third decimal is what distinguishes one + * evaluator from another — "$0.037" vs "$0.049" is a third more expensive, + * but both round to the same two-decimal figure. + * - From a dime up, that precision is noise; show it as money ("$0.18"). + * + * Anything below a millionth of a dollar is labelled rather than rounded away. */ export function formatUsd(usd: number): string { if (usd === 0) return "$0.00"; @@ -82,6 +86,6 @@ export function formatUsd(usd: number): string { // Number() strips the trailing zeros toPrecision leaves behind (0.0034 stays // "0.0034" rather than becoming "0.0034000"). if (usd < 0.01) return `$${Number(usd.toPrecision(2))}`; - if (usd < 1) return `$${usd.toFixed(3)}`; + if (usd < 0.1) return `$${usd.toFixed(3)}`; return `$${usd.toFixed(2)}`; } diff --git a/core/tests/pricing.test.ts b/core/tests/pricing.test.ts index e6355d85..6bcde18d 100644 --- a/core/tests/pricing.test.ts +++ b/core/tests/pricing.test.ts @@ -216,10 +216,24 @@ test("sub-cent runs keep enough precision not to read as free", () => { assert.equal(formatUsd(0.000001), "$0.000001"); assert.equal(formatUsd(0.0034), "$0.0034"); assert.equal(formatUsd(0.0099), "$0.0099"); - assert.equal(formatUsd(0.25), "$0.250"); assert.equal(formatUsd(12.3456), "$12.35"); }); +test("cent-to-dime amounts keep the digit that separates one evaluator from another", () => { + // Real per-evaluator figures from a run. At two decimals these collapse to + // $0.04 / $0.05 / $0.06 and you can no longer see which evaluator is dearest. + assert.equal(formatUsd(0.037), "$0.037"); + assert.equal(formatUsd(0.049), "$0.049"); + assert.equal(formatUsd(0.055), "$0.055"); +}); + +test("a dime and up reads as plain money, not false precision", () => { + assert.equal(formatUsd(0.1), "$0.10"); + assert.equal(formatUsd(0.177), "$0.18"); + assert.equal(formatUsd(0.25), "$0.25"); + assert.equal(formatUsd(0.999), "$1.00"); +}); + test("amounts too small to show are labelled, never rounded to zero", () => { assert.equal(formatUsd(0.0000000001), "<$0.000001"); }); diff --git a/docs/browser-extension.md b/docs/browser-extension.md index a7bf35e7..8204bad1 100644 --- a/docs/browser-extension.md +++ b/docs/browser-extension.md @@ -88,7 +88,7 @@ The extension uses a **single LLM configuration** for all operations (attack gen The extension runs up to **20 turns per evaluator** (default 10). It stops a given evaluator early when the judge returns a definitive verdict. -**Token usage** is tracked per evaluator and shown on the Done screen and in the downloadable HTML report. +**Token usage and testing cost** are tracked per evaluator and shown on the Done screen and in the downloadable HTML report. Cost covers the attacker and judge LLMs you configured in Options — the chat UI you are testing is driven through the browser, so there is nothing to bill for the target itself. See [Token usage and testing cost](cli.md#token-usage-and-testing-cost) for how the figure is derived and its caveats. --- diff --git a/docs/cli.md b/docs/cli.md index 60f982ad..eb6cbbc5 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -215,21 +215,38 @@ Partial reports include all completed evaluator results and are marked with `sto --- -## Token usage tracking +## Token usage and testing cost -Every LLM call (attacker generation, adaptive follow-ups, judge) is metered. After the run completes, the CLI prints a summary line: +Every LLM call (attacker generation, adaptive follow-ups, judge) is metered, and each is attributed to the model that made it. After the run completes, the CLI prints: ``` Results: 5 passed, 2 failed, 0 errors Safety score: 71% Token usage: 51,323 input / 6,057 output (57,380 total) +Testing cost: ≈$0.18 + deepseek/deepseek-v4-pro [attacker]: ≈$0.037 + anthropic/claude-opus-5 [judge]: ≈$0.14 ``` -Token usage is also included in the JSON report (`summary.tokenUsage` and per-evaluator `tokenUsage` fields) and in the HTML report's executive summary card. When using `--events`, the `run_finish` event includes token counts in its `summary` payload. +The per-model split is the actionable part — the judge is frequently the larger share, and switching it to a cheaper model is usually the easiest saving. -The browser extension shows a `Tokens` stat on its Done screen. +Both are included in the JSON report (`summary.tokenUsage`, `summary.tokenUsageByModel`, `summary.cost`, plus the same fields per evaluator) and in the HTML report's executive summary. When using `--events`, the `run_finish` event includes them in its `summary` payload. -> Token counts reflect raw model usage (input + output tokens). No cost estimation is performed — provider pricing varies and changes frequently. +The browser extension shows the same figures on its Done screen. + +### What the cost figure covers + +**"Testing cost" is opfor's own spend — the attacker and judge LLM calls.** It excludes your target's inference cost, which opfor cannot observe from the outside. + +Prices come from a snapshot of LiteLLM's public price map, vendored into the package. Nothing is downloaded at runtime, so runs work offline and a report re-rendered months later produces the same figure. The snapshot version is recorded in the JSON as `summary.cost.priceTableVersion`. + +Maintainers refresh it with `npm run build:pricing` (`-- --check` reports whether it has drifted from upstream). + +### Accuracy caveats + +- **Multi-turn runs are over-estimated.** Providers discount repeated context — and multi-turn attacks re-send the whole conversation each turn — but opfor prices every input token at the full rate. The more turns, the more conservative the figure. +- **List prices only.** Negotiated rates, credits, and proxy markup are not reflected. +- **Unknown models are never counted as free.** If a model isn't in the price table, the report says so and marks the total a lower bound (`≥` instead of `≈`, or `unpriced` when nothing could be priced). Treat `totalUsd` as a floor whenever `summary.cost.complete` is `false`. --- From 0c9a749901383400e6e8eedeb0ad6537046ae14a Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Tue, 4 Aug 2026 17:06:31 +0530 Subject: [PATCH 5/6] fix(report): show a complete cost total without the approx marker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decorating every figure with ≈ added noise to the common case. That a total is a list-price estimate is already stated in the cost card's tooltip and in the docs, so a complete total now renders plain ($0.18). The other two markers stay, because there the number alone would mislead: ≥ says the real total is higher than shown, and an unpriced run has no number worth printing at all. Applied to the HTML report and the CLI summary; docs and README samples updated to match. Co-Authored-By: Claude Opus 5 --- README.md | 6 +++--- core/src/report/render.ts | 11 +++++++---- core/tests/pricing.test.ts | 4 ++-- docs/cli.md | 8 ++++---- runners/cli/src/commands/run.ts | 4 ++-- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index ff909f05..d1f5a3fe 100644 --- a/README.md +++ b/README.md @@ -117,9 +117,9 @@ Every run reports what it cost to run, broken down by model: ``` Token usage: 51,323 input / 6,057 output (57,380 total) -Testing cost: ≈$0.18 - deepseek/deepseek-v4-pro [attacker]: ≈$0.037 - anthropic/claude-opus-5 [judge]: ≈$0.14 +Testing cost: $0.18 + deepseek/deepseek-v4-pro [attacker]: $0.037 + anthropic/claude-opus-5 [judge]: $0.14 ``` This is **opfor's own spend** — the attacker and judge LLMs. It excludes your target's inference cost, which opfor cannot see from the outside. The per-model split is the useful part: the judge is often the bigger share, and pointing it at a cheaper model is usually the easiest saving. diff --git a/core/src/report/render.ts b/core/src/report/render.ts index 1f67eecb..14d33528 100644 --- a/core/src/report/render.ts +++ b/core/src/report/render.ts @@ -30,12 +30,15 @@ import { formatUsd } from "../pricing/estimateCost.js"; * * `totalUsd` sums only the models that could be priced, so when none of them * could it is legitimately `0` — and `formatUsd(0)` is `"$0.00"`, which reads as - * "this run was free" rather than "we don't know". Distinguish the three cases - * at the point of display: `≈` for a complete estimate, `≥` for a partial one - * that is genuinely a floor, and no number at all when nothing was priced. + * "this run was free" rather than "we don't know". + * + * A complete total is shown plain; that it is a list-price estimate is stated in + * the card's tooltip and the docs rather than decorating every figure. The other + * two cases keep their marker because the number alone would mislead: `≥` says + * the real total is higher, and an unpriced run has no number worth printing. */ export function formatCostDisplay(cost: RunCost): string { - if (cost.complete) return `≈${formatUsd(cost.totalUsd)}`; + if (cost.complete) return formatUsd(cost.totalUsd); return cost.totalUsd > 0 ? `≥${formatUsd(cost.totalUsd)}` : "unpriced"; } diff --git a/core/tests/pricing.test.ts b/core/tests/pricing.test.ts index 6bcde18d..7e710a59 100644 --- a/core/tests/pricing.test.ts +++ b/core/tests/pricing.test.ts @@ -267,7 +267,7 @@ function runCost(totalUsd: number, unpriced: string[]): RunCost { } test("a fully-priced run reads as an estimate", () => { - assert.equal(formatCostDisplay(runCost(1.25, [])), "≈$1.25"); + assert.equal(formatCostDisplay(runCost(1.25, [])), "$1.25"); }); test("a partially-priced run reads as a floor, not an estimate", () => { @@ -282,5 +282,5 @@ test("a run where nothing could be priced never renders as $0.00", () => { test("a genuinely free run is still allowed to show zero", () => { // Everything priced, everything free (e.g. a self-hosted model at $0). - assert.equal(formatCostDisplay(runCost(0, [])), "≈$0.00"); + assert.equal(formatCostDisplay(runCost(0, [])), "$0.00"); }); diff --git a/docs/cli.md b/docs/cli.md index eb6cbbc5..2dbb6f09 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -223,9 +223,9 @@ Every LLM call (attacker generation, adaptive follow-ups, judge) is metered, and Results: 5 passed, 2 failed, 0 errors Safety score: 71% Token usage: 51,323 input / 6,057 output (57,380 total) -Testing cost: ≈$0.18 - deepseek/deepseek-v4-pro [attacker]: ≈$0.037 - anthropic/claude-opus-5 [judge]: ≈$0.14 +Testing cost: $0.18 + deepseek/deepseek-v4-pro [attacker]: $0.037 + anthropic/claude-opus-5 [judge]: $0.14 ``` The per-model split is the actionable part — the judge is frequently the larger share, and switching it to a cheaper model is usually the easiest saving. @@ -246,7 +246,7 @@ Maintainers refresh it with `npm run build:pricing` (`-- --check` reports whethe - **Multi-turn runs are over-estimated.** Providers discount repeated context — and multi-turn attacks re-send the whole conversation each turn — but opfor prices every input token at the full rate. The more turns, the more conservative the figure. - **List prices only.** Negotiated rates, credits, and proxy markup are not reflected. -- **Unknown models are never counted as free.** If a model isn't in the price table, the report says so and marks the total a lower bound (`≥` instead of `≈`, or `unpriced` when nothing could be priced). Treat `totalUsd` as a floor whenever `summary.cost.complete` is `false`. +- **Unknown models are never counted as free.** If a model isn't in the price table, the report says so and marks the total a lower bound (`≥` prefix, or `unpriced` when nothing could be priced). Treat `totalUsd` as a floor whenever `summary.cost.complete` is `false`. --- diff --git a/runners/cli/src/commands/run.ts b/runners/cli/src/commands/run.ts index 64eecc03..a139c5b5 100644 --- a/runners/cli/src/commands/run.ts +++ b/runners/cli/src/commands/run.ts @@ -267,9 +267,9 @@ export function registerRunCommand(program: Command): void { const note = summary.cost.complete ? "" : ` (lower bound — unpriced: ${summary.cost.unpricedModels.join(", ")})`; - log.info(`Testing cost: ≈${formatUsd(summary.cost.totalUsd)}${note}`); + log.info(`Testing cost: ${formatUsd(summary.cost.totalUsd)}${note}`); for (const m of summary.cost.byModel) { - const amount = m.usd === undefined ? "not priced" : `≈${formatUsd(m.usd)}`; + const amount = m.usd === undefined ? "not priced" : formatUsd(m.usd); const roles = m.roles.length ? ` [${m.roles.join(", ")}]` : ""; log.info(` ${m.model}${roles}: ${amount}`); } From 5ecf1cb0cfb041fe6885f55a398a6f6e24be4c9a Mon Sep 17 00:00:00 2001 From: Arun Sunny Date: Tue, 4 Aug 2026 17:09:34 +0530 Subject: [PATCH 6/6] docs(pricing): correct two overclaims in the cost documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both were introduced by the previous commit and flagged in review. "Every run reports what it cost to run" is not true: trace curation, session summarisation and generateJsonObject record no token usage, so their spend never reaches the total. The wording now says *instrumented* LLM calls, and unrecorded calls are listed alongside unpriced models as a reason the figure is a floor. docs/cli.md also spells out that summary.cost.complete reports only that every model opfor saw was priced — it cannot vouch for calls that reported nothing. The extension guide said there was "nothing to bill for the target itself". Driving a chat UI through the browser means opfor is not billed for it, but somebody is — that cost is simply invisible to us. Reworded to say target-side inference is excluded and unobservable. Also tags both sample-output fences as `text` for markdownlint. Co-Authored-By: Claude Opus 5 --- README.md | 5 +++-- docs/browser-extension.md | 2 +- docs/cli.md | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d1f5a3fe..d6867fc9 100644 --- a/README.md +++ b/README.md @@ -113,9 +113,9 @@ Each run lands in its own subfolder under `.opfor/reports/run-report- ### Testing cost -Every run reports what it cost to run, broken down by model: +Every run reports what its instrumented LLM calls cost, broken down by model: -``` +```text Token usage: 51,323 input / 6,057 output (57,380 total) Testing cost: $0.18 deepseek/deepseek-v4-pro [attacker]: $0.037 @@ -128,6 +128,7 @@ Prices come from a snapshot of LiteLLM's public price map that ships with the pa - **Multi-turn runs read high.** Providers discount repeated context, and a multi-turn attack re-sends the conversation each turn — opfor prices every input token at full rate, so the real bill is usually lower. - **Unknown models are never counted as free.** A model missing from the price table is reported as unpriced and the total is marked a lower bound, rather than silently reading as $0. +- **A few helper calls aren't metered yet.** Trace curation, session summarisation and one JSON helper don't report token usage, so their spend is missing from the total. Treat the figure as a floor. → [Token usage and testing cost](docs/cli.md#token-usage-and-testing-cost) diff --git a/docs/browser-extension.md b/docs/browser-extension.md index 8204bad1..ad4d8578 100644 --- a/docs/browser-extension.md +++ b/docs/browser-extension.md @@ -88,7 +88,7 @@ The extension uses a **single LLM configuration** for all operations (attack gen The extension runs up to **20 turns per evaluator** (default 10). It stops a given evaluator early when the judge returns a definitive verdict. -**Token usage and testing cost** are tracked per evaluator and shown on the Done screen and in the downloadable HTML report. Cost covers the attacker and judge LLMs you configured in Options — the chat UI you are testing is driven through the browser, so there is nothing to bill for the target itself. See [Token usage and testing cost](cli.md#token-usage-and-testing-cost) for how the figure is derived and its caveats. +**Token usage and testing cost** are tracked per evaluator and shown on the Done screen and in the downloadable HTML report. Cost covers the attacker and judge LLMs you configured in Options. Whatever the target chat spends on its own inference is excluded — opfor drives it through the browser and cannot observe or meter that spend. See [Token usage and testing cost](cli.md#token-usage-and-testing-cost) for how the figure is derived and its caveats. --- diff --git a/docs/cli.md b/docs/cli.md index 2dbb6f09..cd2e5e0f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -217,9 +217,9 @@ Partial reports include all completed evaluator results and are marked with `sto ## Token usage and testing cost -Every LLM call (attacker generation, adaptive follow-ups, judge) is metered, and each is attributed to the model that made it. After the run completes, the CLI prints: +The instrumented LLM calls (attacker generation, adaptive follow-ups, judge, MCP baseline scans) are metered, and each is attributed to the model that made it. After the run completes, the CLI prints: -``` +```text Results: 5 passed, 2 failed, 0 errors Safety score: 71% Token usage: 51,323 input / 6,057 output (57,380 total) @@ -246,7 +246,8 @@ Maintainers refresh it with `npm run build:pricing` (`-- --check` reports whethe - **Multi-turn runs are over-estimated.** Providers discount repeated context — and multi-turn attacks re-send the whole conversation each turn — but opfor prices every input token at the full rate. The more turns, the more conservative the figure. - **List prices only.** Negotiated rates, credits, and proxy markup are not reflected. -- **Unknown models are never counted as free.** If a model isn't in the price table, the report says so and marks the total a lower bound (`≥` prefix, or `unpriced` when nothing could be priced). Treat `totalUsd` as a floor whenever `summary.cost.complete` is `false`. +- **Unknown models are never counted as free.** If a model isn't in the price table, the report says so and marks the total a lower bound (`≥` prefix, or `unpriced` when nothing could be priced). +- **Not every call is instrumented yet.** Trace curation, session summarisation, and `generateJsonObject` record no token usage, so their spend never reaches the total. `summary.cost.complete` reports only that every model opfor _saw_ was priced — it cannot vouch for calls that reported nothing. Treat `totalUsd` as a floor in all cases. ---