diff --git a/README.md b/README.md index 3a504769..d6867fc9 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,27 @@ When you run a scan, opfor: Each run lands in its own subfolder under `.opfor/reports/run-report---/` containing `-report.html` and `-report.json`. Autonomous `opfor hunt` runs use the same layout under `hunt-report---/`. +### Testing cost + +Every run reports what its instrumented LLM calls cost, broken down by model: + +```text +Token usage: 51,323 input / 6,057 output (57,380 total) +Testing cost: $0.18 + deepseek/deepseek-v4-pro [attacker]: $0.037 + anthropic/claude-opus-5 [judge]: $0.14 +``` + +This is **opfor's own spend** — the attacker and judge LLMs. It excludes your target's inference cost, which opfor cannot see from the outside. The per-model split is the useful part: the judge is often the bigger share, and pointing it at a cheaper model is usually the easiest saving. + +Prices come from a snapshot of LiteLLM's public price map that ships with the package, so runs work offline and a report re-rendered later produces the same figure. Two caveats worth knowing: + +- **Multi-turn runs read high.** Providers discount repeated context, and a multi-turn attack re-sends the conversation each turn — opfor prices every input token at full rate, so the real bill is usually lower. +- **Unknown models are never counted as free.** A model missing from the price table is reported as unpriced and the total is marked a lower bound, rather than silently reading as $0. +- **A few helper calls aren't metered yet.** Trace curation, session summarisation and one JSON helper don't report token usage, so their spend is missing from the total. Treat the figure as a floor. + +→ [Token usage and testing cost](docs/cli.md#token-usage-and-testing-cost) + ## Evaluator coverage Opfor ships with curated suites that map to industry standards. Pick a suite or run individual evaluators. diff --git a/core/package.json b/core/package.json index a370ed84..367942c8 100644 --- a/core/package.json +++ b/core/package.json @@ -43,6 +43,10 @@ "types": "./dist/providers/*.d.ts", "default": "./dist/providers/*.js" }, + "./pricing/*.js": { + "types": "./dist/pricing/*.d.ts", + "default": "./dist/pricing/*.js" + }, "./evaluators/*.js": { "types": "./dist/evaluators/*.d.ts", "default": "./dist/evaluators/*.js" diff --git a/core/src/browser.ts b/core/src/browser.ts index 61fa9d15..0809310e 100644 --- a/core/src/browser.ts +++ b/core/src/browser.ts @@ -69,6 +69,14 @@ export { } from "./providers/factory.js"; export type { LlmConfig, ProviderName } from "./config/types.js"; +// Cost estimation. The vendored price table is a plain inlined module with no +// Node imports, so esbuild bundles it for the extension like any other source. +export { estimateRunCost, formatUsd } from "./pricing/estimateCost.js"; +export { lookupPrice } from "./pricing/lookupPrice.js"; +export { PRICE_TABLE_VERSION } from "./pricing/priceTable.generated.js"; +export type { RunCost, ModelCost, ModelPrice, CostSource } from "./pricing/types.js"; +export type { ModelTokenUsage, TokenUsage } from "./execute/tokenTracker.js"; + export { getAdapter } from "./telemetry/adapter.js"; export { renderReport } from "./report/render.js"; diff --git a/core/src/evaluators/judge.ts b/core/src/evaluators/judge.ts index c55a5ffe..e1a42217 100644 --- a/core/src/evaluators/judge.ts +++ b/core/src/evaluators/judge.ts @@ -206,7 +206,7 @@ export async function judgeResponse( try { const result = await withRetry( () => generateText({ model, system: JUDGE_SYSTEM, prompt: judgePrompt }), - { context: "Judge", maxRetries: 3, tokenTracker } + { context: "Judge", maxRetries: 3, tokenTracker, model } ); return parseJudgeOutput(result.text); } catch (err) { diff --git a/core/src/execute/baselineScanner.ts b/core/src/execute/baselineScanner.ts index 6707a2d3..91f3e5ce 100644 --- a/core/src/execute/baselineScanner.ts +++ b/core/src/execute/baselineScanner.ts @@ -8,6 +8,7 @@ import { createHash } from "node:crypto"; import { z } from "zod"; import { randomUUID } from "../lib/random.js"; import { judgeToolResponse } from "../run/judge.js"; +import type { TokenTracker } from "./tokenTracker.js"; import { errorJudge as mcpErrorJudge } from "../lib/judgeTypes.js"; import { toEvaluatorResult } from "./aggregate.js"; import { log } from "../lib/logger.js"; @@ -24,6 +25,12 @@ export interface BaselineScanContext { config: RunConfig; outputDir?: string; notify: (event: ProgressEvent) => void; + /** + * Run-level token accumulator. Baseline scans judge every tool description and + * resource before the evaluator loop starts, so without this their spend is + * invisible to the run's token count and cost estimate. + */ + tokenTracker?: TokenTracker; } /** @@ -132,6 +139,7 @@ async function scanResources(ctx: BaselineScanContext): Promise try { judgeResult = await judgeToolResponse({ model: judgeModelConfig, + tokenTracker: ctx.tokenTracker, evaluator: { id: evalId, name: "MCP Resource Exposure", @@ -186,6 +194,7 @@ async function scanToolDescriptions(ctx: BaselineScanContext): Promise 0) { report.summary.tokenUsage = usage; + report.summary.tokenUsageByModel = tokenTracker.breakdown; + report.summary.cost = estimateRunCost(report.summary.tokenUsageByModel); } report.summary.durationMs = Date.now() - runStartedAt; if (stopReason) { diff --git a/core/src/execute/runAllBrowser.ts b/core/src/execute/runAllBrowser.ts index 763d8376..bad15ce0 100644 --- a/core/src/execute/runAllBrowser.ts +++ b/core/src/execute/runAllBrowser.ts @@ -20,6 +20,7 @@ import { modelLabel, } from "./aggregate.js"; import { TokenTracker } from "./tokenTracker.js"; +import { estimateRunCost } from "../pricing/estimateCost.js"; import type { AgentAttackSpec, AttackResult, @@ -189,6 +190,8 @@ export async function runAllBrowser( attackResults ); partialResult.tokenUsage = evalTracker.totals; + partialResult.tokenUsageByModel = evalTracker.breakdown; + partialResult.cost = estimateRunCost(partialResult.tokenUsageByModel); evaluatorResults.push(partialResult); }; @@ -230,6 +233,8 @@ export async function runAllBrowser( attackResults ); evResult.tokenUsage = evalTracker.totals; + evResult.tokenUsageByModel = evalTracker.breakdown; + evResult.cost = estimateRunCost(evResult.tokenUsageByModel); evaluatorResults.push(evResult); } @@ -237,6 +242,8 @@ export async function runAllBrowser( const usage = tokenTracker.totals; if (usage.totalTokens > 0) { report.summary.tokenUsage = usage; + report.summary.tokenUsageByModel = tokenTracker.breakdown; + report.summary.cost = estimateRunCost(report.summary.tokenUsageByModel); } report.summary.durationMs = Date.now() - runStartedAt; return report; diff --git a/core/src/execute/tokenTracker.ts b/core/src/execute/tokenTracker.ts index 2bf2d01c..ed7ba6a7 100644 --- a/core/src/execute/tokenTracker.ts +++ b/core/src/execute/tokenTracker.ts @@ -5,9 +5,20 @@ * attack drivers. Each `generateText` / `generateObject` call site records its * usage after the call completes (including retries). Aggregated totals are * surfaced in the CLI summary, HTML/JSON report, and extension popup. + * + * Usage is recorded twice: once into flat run totals, and once into a per-model + * bucket. The per-model breakdown exists because a run can mix models — the + * judge may be a different (and far more expensive) model than the attacker — + * so a single combined total cannot be priced. See {@link ModelTokenUsage}. */ import { z } from "zod"; +import { + modelKey, + resolveModelIdentity, + UNKNOWN_MODEL_KEY, + type ModelRef, +} from "../providers/modelIdentity.js"; /** Aggregated input/output/total token counts from LLM calls. */ export interface TokenUsage { @@ -16,6 +27,26 @@ export interface TokenUsage { totalTokens: number; } +/** Token usage attributed to one provider/model pair. */ +export interface ModelTokenUsage extends TokenUsage { + /** `":"`, or `"unknown"` for usage that could not be attributed. */ + key: string; + provider: string; + model: string; + /** Which phases used this model — `"attacker"`, `"judge"`. Sorted, deduped. */ + roles: string[]; + /** Number of LLM calls recorded against this model. */ + calls: number; +} + +/** Optional provenance supplied alongside a usage recording. */ +export interface RecordAttribution { + /** The model the call was made against — an AI SDK model, an LlmConfig, or an identity. */ + model?: ModelRef; + /** Run phase, e.g. `"attacker"` or `"judge"`. Case-insensitive. */ + role?: string; +} + /** Shared zero-value constant to avoid re-allocating empty usage objects. */ export const ZERO_USAGE: TokenUsage = Object.freeze({ inputTokens: 0, @@ -53,6 +84,18 @@ export function parseUsage(raw: unknown): TokenUsage | undefined { return result.success ? result.data : undefined; } +/** Mutable per-model accumulator; projected to {@link ModelTokenUsage} on read. */ +interface ModelBucket { + key: string; + provider: string; + model: string; + roles: Set; + calls: number; + input: number; + output: number; + total: number; +} + /** * Accumulator for LLM token usage. * @@ -64,20 +107,65 @@ export class TokenTracker { private input = 0; private output = 0; private total = 0; + private readonly buckets = new Map(); /** * Record usage from a single LLM call. Safe to call with undefined/partial * usage. When `totalTokens` is supplied it is preserved; otherwise it falls * back to `inputTokens + outputTokens`. + * + * `attribution` is optional — usage recorded without it lands in the + * `"unknown"` bucket and still counts toward run totals, so an un-migrated + * call site degrades to today's behavior rather than losing tokens. */ - record(usage?: { inputTokens?: number; outputTokens?: number; totalTokens?: number }): void { + record( + usage?: { inputTokens?: number; outputTokens?: number; totalTokens?: number }, + attribution?: RecordAttribution + ): void { if (!usage) return; const inp = usage.inputTokens ?? 0; const out = usage.outputTokens ?? 0; const tot = usage.totalTokens ?? 0; + const resolvedTotal = tot > 0 ? tot : inp + out; + this.input += inp; this.output += out; - this.total += tot > 0 ? tot : inp + out; + this.total += resolvedTotal; + + this.recordToBucket(inp, out, resolvedTotal, attribution); + } + + /** Fold one call's usage into its per-model bucket, creating the bucket on first sight. */ + private recordToBucket( + inp: number, + out: number, + total: number, + attribution?: RecordAttribution + ): void { + const identity = resolveModelIdentity(attribution?.model); + const key = identity ? modelKey(identity) : UNKNOWN_MODEL_KEY; + + let bucket = this.buckets.get(key); + if (!bucket) { + bucket = { + key, + provider: identity?.provider ?? "unknown", + model: identity?.model ?? "unknown", + roles: new Set(), + calls: 0, + input: 0, + output: 0, + total: 0, + }; + this.buckets.set(key, bucket); + } + + const role = attribution?.role?.trim().toLowerCase(); + if (role) bucket.roles.add(role); + bucket.calls += 1; + bucket.input += inp; + bucket.output += out; + bucket.total += total; } /** Current accumulated totals. Uses the provider-supplied total when available. */ @@ -89,6 +177,26 @@ export class TokenTracker { }; } + /** + * Per-model usage, heaviest first. Always sums to {@link totals}; models that + * could not be attributed appear under the `"unknown"` key rather than being + * dropped. + */ + get breakdown(): ModelTokenUsage[] { + return [...this.buckets.values()] + .map((b) => ({ + key: b.key, + provider: b.provider, + model: b.model, + roles: [...b.roles].sort(), + calls: b.calls, + inputTokens: b.input, + outputTokens: b.output, + totalTokens: b.total, + })) + .sort((a, b) => b.totalTokens - a.totalTokens || a.key.localeCompare(b.key)); + } + /** * Create a child tracker whose recordings propagate to this parent. * Used per-evaluator so individual usage is readable while the parent @@ -109,12 +217,15 @@ class ChildTracker extends TokenTracker { super(); } - override record(usage?: { - inputTokens?: number; - outputTokens?: number; - totalTokens?: number; - }): void { - super.record(usage); - this.parent.record(usage); + override record( + usage?: { + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + }, + attribution?: RecordAttribution + ): void { + super.record(usage, attribution); + this.parent.record(usage, attribution); } } diff --git a/core/src/execute/types.ts b/core/src/execute/types.ts index 1a84ea62..e9980373 100644 --- a/core/src/execute/types.ts +++ b/core/src/execute/types.ts @@ -1,5 +1,7 @@ import type { LlmConfig, TelemetryConfig } from "../config/types.js"; import type { JudgeResult } from "../run/types.js"; +import type { ModelTokenUsage } from "./tokenTracker.js"; +import type { RunCost } from "../pricing/types.js"; export type Effort = "adaptive" | "comprehensive"; @@ -249,6 +251,10 @@ export interface EvaluatorResult { passRate: number; attacks: AttackResult[]; tokenUsage?: { inputTokens: number; outputTokens: number; totalTokens: number }; + /** Same tokens as `tokenUsage`, split by the model that spent them. */ + tokenUsageByModel?: ModelTokenUsage[]; + /** Estimated USD cost of this evaluator's attacker + judge calls. */ + cost?: RunCost; } export interface UnifiedRunReport { @@ -273,6 +279,13 @@ export interface UnifiedRunReport { safetyScore: number; attackSuccessRate: number; tokenUsage?: { inputTokens: number; outputTokens: number; totalTokens: number }; + /** Same tokens as `tokenUsage`, split by the model that spent them. */ + tokenUsageByModel?: ModelTokenUsage[]; + /** + * Estimated USD cost of the attacker + judge LLM calls. Excludes the + * target's own inference cost, which opfor cannot observe. + */ + cost?: RunCost; /** Wall-clock time for the whole run, from the first evaluator to the last. */ durationMs?: number; }; diff --git a/core/src/generate/generateAttacks.ts b/core/src/generate/generateAttacks.ts index d7b5976f..d4768670 100644 --- a/core/src/generate/generateAttacks.ts +++ b/core/src/generate/generateAttacks.ts @@ -183,6 +183,7 @@ async function generatePatternAgentAttack( context: "Attacker", maxRetries: 3, tokenTracker, + model, }); return result.text.trim(); } catch (err) { @@ -387,6 +388,7 @@ async function generateSingleMcpAttack( context: "Attacker (MCP)", maxRetries: 3, tokenTracker, + model, }); return parseMcpAttackJson(result.text, tools[0].name); } catch (err) { @@ -437,6 +439,7 @@ async function generatePatternMcpAttack( context: "Attacker (MCP)", maxRetries: 3, tokenTracker, + model, }); return parseMcpAttackJson(result.text, tools[0].name); } catch (err) { diff --git a/core/src/generate/generateNextTurn.ts b/core/src/generate/generateNextTurn.ts index df8a0f73..e11c1c19 100644 --- a/core/src/generate/generateNextTurn.ts +++ b/core/src/generate/generateNextTurn.ts @@ -194,7 +194,7 @@ export async function generateNextAdaptiveTurn(params: { const result = await generateText({ model, system, prompt: userBlock }); const usage1 = parseUsage(result.usage); - if (usage1) params.tokenTracker?.record(usage1); + if (usage1) params.tokenTracker?.record(usage1, { model, role: "attacker" }); const parsed = parseAttackerOutput(result.text); if (!parsed.message) throw new Error("generateNextAdaptiveTurn: empty model response"); const message = @@ -343,7 +343,7 @@ export async function generateNextMcpTurn( const result = await generateText({ model, system, prompt: user }); const usage2 = parseUsage(result.usage); - if (usage2) tokenTracker?.record(usage2); + if (usage2) tokenTracker?.record(usage2, { model, role: "attacker" }); try { const cleaned = result.text diff --git a/core/src/lib/llmRetry.ts b/core/src/lib/llmRetry.ts index 680ac7ae..90430477 100644 --- a/core/src/lib/llmRetry.ts +++ b/core/src/lib/llmRetry.ts @@ -6,6 +6,7 @@ import { log } from "./logger.js"; import type { TokenTracker } from "../execute/tokenTracker.js"; import { parseUsage } from "../execute/tokenTracker.js"; +import type { ModelRef } from "../providers/modelIdentity.js"; export interface LlmError { isRetryable: boolean; @@ -127,6 +128,25 @@ export interface RetryOptions { context?: string; // e.g., "attacker", "judge" for logging /** When set, usage from a successful result's `.usage` field is auto-recorded. */ tokenTracker?: TokenTracker; + /** + * The model the wrapped call runs against. Used only to attribute recorded + * token usage — without it the usage still counts, but lands in the + * unattributed bucket and cannot be priced. + */ + model?: ModelRef; +} + +/** + * Derive a run phase from the free-text `context` label already passed for + * logging (`"Attacker"`, `"Attacker (MCP)"`, `"Judge"`), so call sites don't + * have to repeat themselves. Unrecognized labels yield no role. + */ +export function roleFromContext(context?: string): string | undefined { + const c = context?.trim().toLowerCase(); + if (!c) return undefined; + if (c.startsWith("attacker")) return "attacker"; + if (c.startsWith("judge")) return "judge"; + return undefined; } /** @@ -140,6 +160,7 @@ export async function withRetry(fn: () => Promise, options: RetryOptions = maxDelayMs = 30000, context = "LLM", tokenTracker, + model, } = options; let lastError: LlmError | null = null; @@ -149,7 +170,7 @@ export async function withRetry(fn: () => Promise, options: RetryOptions = const result = await fn(); if (tokenTracker && result && typeof result === "object" && "usage" in result) { const validated = parseUsage((result as { usage?: unknown }).usage); - if (validated) tokenTracker.record(validated); + if (validated) tokenTracker.record(validated, { model, role: roleFromContext(context) }); } return result; } catch (err) { diff --git a/core/src/llm/openaiCompatible.ts b/core/src/llm/openaiCompatible.ts index 484c099e..32f178ee 100644 --- a/core/src/llm/openaiCompatible.ts +++ b/core/src/llm/openaiCompatible.ts @@ -108,6 +108,8 @@ export async function chatCompletionJsonContent(args: { user: string; isAcceptableJson?: (json: string) => boolean; tokenTracker?: TokenTracker; + /** Run phase this call belongs to, used to attribute recorded token usage. */ + role?: string; }): Promise { const apiKey = resolveApiKey(args.model); if (!apiKey) { @@ -210,7 +212,7 @@ export async function chatCompletionJsonContent(args: { outputTokens: data.usage.completion_tokens ?? 0, totalTokens: data.usage.total_tokens ?? 0, }); - if (validated) args.tokenTracker.record(validated); + if (validated) args.tokenTracker.record(validated, { model: args.model, role: args.role }); } const content = data.choices?.[0]?.message?.content; if (typeof content !== "string" || !content.trim()) { diff --git a/core/src/pricing/estimateCost.ts b/core/src/pricing/estimateCost.ts new file mode 100644 index 00000000..4452a0a3 --- /dev/null +++ b/core/src/pricing/estimateCost.ts @@ -0,0 +1,91 @@ +/** + * Turn a per-model token breakdown into a run cost. + * + * Deliberately conservative: a model whose price cannot be found is reported as + * unpriced rather than counted as free. `totalUsd` is therefore a lower bound + * whenever `complete` is false, and the UI must say so — a report that quietly + * treats an unknown model as $0 understates spend while looking authoritative. + * + * Repeat ("cached") input tokens are not yet separated by the token counter, so + * all input is priced at the full input rate. For multi-turn runs against + * providers that discount repeated context, that makes this an over-estimate. + */ + +import type { ModelTokenUsage } from "../execute/tokenTracker.js"; +import { lookupPrice } from "./lookupPrice.js"; +import { PRICE_TABLE_VERSION } from "./priceTable.generated.js"; +import type { ModelCost, RunCost } from "./types.js"; + +/** Price one model's usage. */ +function costOne(usage: ModelTokenUsage): ModelCost { + const found = lookupPrice(usage.provider, usage.model); + + const base: ModelCost = { + key: usage.key, + provider: usage.provider, + model: usage.model, + roles: usage.roles, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + totalTokens: usage.totalTokens, + source: "unknown", + }; + + if (!found) return base; + + return { + ...base, + usd: + usage.inputTokens * found.price.inputPerToken + + usage.outputTokens * found.price.outputPerToken, + source: "table", + matchedKey: found.matchedKey, + }; +} + +/** + * Estimate the cost of a run from its per-model token breakdown. + * + * Returns undefined when there is nothing to price, so callers can omit the + * field entirely rather than render a meaningless $0.00. + */ +export function estimateRunCost(breakdown?: ModelTokenUsage[]): RunCost | undefined { + const spending = (breakdown ?? []).filter((b) => b.totalTokens > 0); + if (spending.length === 0) return undefined; + + const byModel = spending.map(costOne); + const unpricedModels = byModel.filter((c) => c.usd === undefined).map((c) => c.key); + const totalUsd = byModel.reduce((sum, c) => sum + (c.usd ?? 0), 0); + + return { + totalUsd, + currency: "USD", + byModel, + unpricedModels, + complete: unpricedModels.length === 0, + priceTableVersion: PRICE_TABLE_VERSION, + }; +} + +/** + * Format a USD amount for display, with precision that scales to the amount. + * + * Three bands, each solving a different problem: + * - Below a cent, a fixed decimal count renders "$0.0000" and reads as free, + * so these show two significant figures instead ("$0.0034"). + * - Between a cent and a dime, the third decimal is what distinguishes one + * evaluator from another — "$0.037" vs "$0.049" is a third more expensive, + * but both round to the same two-decimal figure. + * - From a dime up, that precision is noise; show it as money ("$0.18"). + * + * Anything below a millionth of a dollar is labelled rather than rounded away. + */ +export function formatUsd(usd: number): string { + if (usd === 0) return "$0.00"; + if (usd < 0.000001) return "<$0.000001"; + // Number() strips the trailing zeros toPrecision leaves behind (0.0034 stays + // "0.0034" rather than becoming "0.0034000"). + if (usd < 0.01) return `$${Number(usd.toPrecision(2))}`; + if (usd < 0.1) return `$${usd.toFixed(3)}`; + return `$${usd.toFixed(2)}`; +} diff --git a/core/src/pricing/lookupPrice.ts b/core/src/pricing/lookupPrice.ts new file mode 100644 index 00000000..3ad284c0 --- /dev/null +++ b/core/src/pricing/lookupPrice.ts @@ -0,0 +1,83 @@ +/** + * Resolve a configured provider/model pair to per-token prices. + * + * The same model is listed upstream under several names depending on how it is + * reached (`claude-opus-5`, `vertex_ai/claude-opus-5`, `anthropic/claude-opus-5`), + * while users type whatever their setup expects. So rather than one lookup, we + * try a short ladder of name forms and take the first that matches. + * + * Every match is then checked against the configured provider. Without that + * guard, someone running a mistyped model name on Groq would silently match + * OpenAI's row and be priced at OpenAI's rates — a wrong number presented with + * full confidence, which is worse than reporting nothing. + */ + +import { PRICE_TABLE, type CompactPrice } from "./priceTable.generated.js"; +import { LITELLM_PROVIDER_ALIASES } from "./providerAliases.js"; +import type { ModelPrice } from "./types.js"; + +/** A resolved price plus the table key it came from, for auditability. */ +export interface PriceLookupResult { + price: ModelPrice; + matchedKey: string; +} + +/** + * Name forms to try, in order. + * + * The provider-scoped form goes first because some vendors resell the same model + * at a different rate — `azure/gpt-4o-mini` costs more than OpenAI's own + * `gpt-4o-mini`. Trying it first picks the right row outright instead of relying + * on the guard below to reject the wrong one. Most providers have no such row, + * in which case the lookup simply falls through to the bare name. + */ +export function priceCandidates(provider: string, model: string): string[] { + const m = model.trim(); + if (!m) return []; + const lower = m.toLowerCase(); + + const forms = [`${provider}/${m}`, m]; + if (m.includes("/")) { + // "anthropic/claude-opus-5" -> "claude-opus-5"; upstream lists direct-API + // models bare, so a prefixed name only matches once the prefix is dropped. + forms.push(m.slice(m.indexOf("/") + 1)); + forms.push(m.slice(m.lastIndexOf("/") + 1)); + } + if (lower !== m) forms.push(lower); + + return [...new Set(forms.filter(Boolean))]; +} + +/** Whether a table row may legitimately price this provider. */ +function providerAccepts(provider: string, entry: CompactPrice): boolean { + const accepted = LITELLM_PROVIDER_ALIASES[provider]; + // Unknown to the alias map, or explicitly unverifiable (an OpenAI-compatible + // proxy can serve any vendor) — nothing to check against, so accept the match. + if (accepted === undefined || accepted === null) return true; + return accepted.includes(entry.p); +} + +function toModelPrice(entry: CompactPrice): ModelPrice { + return { + inputPerToken: entry.i, + outputPerToken: entry.o, + ...(entry.cr !== undefined ? { cacheReadPerToken: entry.cr } : {}), + ...(entry.cw !== undefined ? { cacheWritePerToken: entry.cw } : {}), + }; +} + +/** + * Look up prices for one provider/model pair. + * + * Returns undefined when no name form matches, or when every match belongs to a + * different vendor. Callers must treat that as "unpriced", never as free. + */ +export function lookupPrice(provider: string, model: string): PriceLookupResult | undefined { + for (const key of priceCandidates(provider, model)) { + const entry = PRICE_TABLE[key]; + if (!entry) continue; + if (!providerAccepts(provider, entry)) continue; + return { price: toModelPrice(entry), matchedKey: key }; + } + return undefined; +} diff --git a/core/src/pricing/priceTable.generated.ts b/core/src/pricing/priceTable.generated.ts new file mode 100644 index 00000000..d195c0e4 --- /dev/null +++ b/core/src/pricing/priceTable.generated.ts @@ -0,0 +1,749 @@ +/** + * GENERATED FILE — do not edit by hand. + * + * Model prices in USD per token, pruned from LiteLLM's community price map: + * https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json + * + * Regenerate with: npm run build:pricing + * + * Entries: 557 + * + * Contains no Node imports so the browser extension can bundle it. + */ + +/** Compact price row. Short keys keep the generated table small. */ +export interface CompactPrice { + /** Upstream `litellm_provider` — used to reject a match from the wrong vendor. */ + p: string; + /** USD per input token. */ + i: number; + /** USD per output token. */ + o: number; + /** USD per cached input token, when published. */ + cr?: number; + /** USD per cache-write token, when published. */ + cw?: number; +} + +/** Identifies this snapshot; recorded in reports so a cost figure is reproducible. */ +export const PRICE_TABLE_VERSION = "litellm-0933d4a0129c"; + +export const PRICE_TABLE: Record = { + "azure/codex-mini": { p: "azure", i: 0.0000015, o: 0.000006, cr: 3.75e-7 }, + "azure/command-r-plus": { p: "azure", i: 0.000003, o: 0.000015 }, + "azure/computer-use-preview": { p: "azure", i: 0.000003, o: 0.000012 }, + "azure/eu/gpt-4o-2024-08-06": { p: "azure", i: 0.00000275, o: 0.000011, cr: 0.000001375 }, + "azure/eu/gpt-4o-2024-11-20": { p: "azure", i: 0.00000275, o: 0.000011, cw: 0.00000138 }, + "azure/eu/gpt-4o-mini-2024-07-18": { p: "azure", i: 1.65e-7, o: 6.6e-7, cr: 8.3e-8 }, + "azure/eu/gpt-5-2025-08-07": { p: "azure", i: 0.000001375, o: 0.000011, cr: 1.375e-7 }, + "azure/eu/gpt-5-mini-2025-08-07": { p: "azure", i: 2.75e-7, o: 0.0000022, cr: 2.75e-8 }, + "azure/eu/gpt-5-nano-2025-08-07": { p: "azure", i: 5.5e-8, o: 4.4e-7, cr: 5.5e-9 }, + "azure/eu/gpt-5.1": { p: "azure", i: 0.00000138, o: 0.000011, cr: 1.4e-7 }, + "azure/eu/gpt-5.1-chat": { p: "azure", i: 0.00000138, o: 0.000011, cr: 1.4e-7 }, + "azure/eu/gpt-5.1-codex": { p: "azure", i: 0.00000138, o: 0.000011, cr: 1.4e-7 }, + "azure/eu/gpt-5.1-codex-mini": { p: "azure", i: 2.75e-7, o: 0.0000022, cr: 2.8e-8 }, + "azure/eu/gpt-5.4": { p: "azure", i: 0.00000275, o: 0.0000165, cr: 2.8e-7 }, + "azure/eu/gpt-5.4-2026-03-05": { p: "azure", i: 0.00000275, o: 0.0000165, cr: 2.8e-7 }, + "azure/eu/gpt-5.5": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/eu/gpt-5.5-2026-04-23": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/eu/gpt-5.6": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/eu/gpt-5.6-luna": { p: "azure", i: 0.0000011, o: 0.0000066, cr: 1.1e-7 }, + "azure/eu/gpt-5.6-sol": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/eu/gpt-5.6-terra": { p: "azure", i: 0.00000275, o: 0.0000165, cr: 2.75e-7 }, + "azure/eu/o1-2024-12-17": { p: "azure", i: 0.0000165, o: 0.000066, cr: 0.00000825 }, + "azure/eu/o1-mini-2024-09-12": { p: "azure", i: 0.00000121, o: 0.00000484, cr: 6.05e-7 }, + "azure/eu/o1-preview-2024-09-12": { p: "azure", i: 0.0000165, o: 0.000066, cr: 0.00000825 }, + "azure/eu/o3-mini-2025-01-31": { p: "azure", i: 0.00000121, o: 0.00000484, cr: 6.05e-7 }, + "azure/global-standard/gpt-4o-2024-08-06": { + p: "azure", + i: 0.0000025, + o: 0.00001, + cr: 0.00000125, + }, + "azure/global-standard/gpt-4o-2024-11-20": { + p: "azure", + i: 0.0000025, + o: 0.00001, + cr: 0.00000125, + }, + "azure/global-standard/gpt-4o-mini": { p: "azure", i: 1.5e-7, o: 6e-7 }, + "azure/global/gpt-4o-2024-08-06": { p: "azure", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "azure/global/gpt-4o-2024-11-20": { p: "azure", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "azure/global/gpt-5.1": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/global/gpt-5.1-chat": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/global/gpt-5.1-codex": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/global/gpt-5.1-codex-mini": { p: "azure", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "azure/gpt-3.5-turbo": { p: "azure", i: 5e-7, o: 0.0000015 }, + "azure/gpt-3.5-turbo-0125": { p: "azure", i: 5e-7, o: 0.0000015 }, + "azure/gpt-35-turbo": { p: "azure", i: 5e-7, o: 0.0000015 }, + "azure/gpt-35-turbo-0125": { p: "azure", i: 5e-7, o: 0.0000015 }, + "azure/gpt-35-turbo-1106": { p: "azure", i: 0.000001, o: 0.000002 }, + "azure/gpt-35-turbo-16k": { p: "azure", i: 0.000003, o: 0.000004 }, + "azure/gpt-35-turbo-16k-0613": { p: "azure", i: 0.000003, o: 0.000004 }, + "azure/gpt-4": { p: "azure", i: 0.00003, o: 0.00006 }, + "azure/gpt-4-0125-preview": { p: "azure", i: 0.00001, o: 0.00003 }, + "azure/gpt-4-0613": { p: "azure", i: 0.00003, o: 0.00006 }, + "azure/gpt-4-1106-preview": { p: "azure", i: 0.00001, o: 0.00003 }, + "azure/gpt-4-32k": { p: "azure", i: 0.00006, o: 0.00012 }, + "azure/gpt-4-32k-0613": { p: "azure", i: 0.00006, o: 0.00012 }, + "azure/gpt-4-turbo": { p: "azure", i: 0.00001, o: 0.00003 }, + "azure/gpt-4-turbo-2024-04-09": { p: "azure", i: 0.00001, o: 0.00003 }, + "azure/gpt-4-turbo-vision-preview": { p: "azure", i: 0.00001, o: 0.00003 }, + "azure/gpt-4.1": { p: "azure", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "azure/gpt-4.1-2025-04-14": { p: "azure", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "azure/gpt-4.1-mini": { p: "azure", i: 4e-7, o: 0.0000016, cr: 1e-7 }, + "azure/gpt-4.1-mini-2025-04-14": { p: "azure", i: 4e-7, o: 0.0000016, cr: 1e-7 }, + "azure/gpt-4.1-nano": { p: "azure", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "azure/gpt-4.1-nano-2025-04-14": { p: "azure", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "azure/gpt-4.5-preview": { p: "azure", i: 0.000075, o: 0.00015, cr: 0.0000375 }, + "azure/gpt-4o": { p: "azure", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "azure/gpt-4o-2024-05-13": { p: "azure", i: 0.000005, o: 0.000015 }, + "azure/gpt-4o-2024-08-06": { p: "azure", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "azure/gpt-4o-2024-11-20": { p: "azure", i: 0.00000275, o: 0.000011, cr: 0.00000125 }, + "azure/gpt-4o-audio-preview-2024-12-17": { p: "azure", i: 0.0000025, o: 0.00001 }, + "azure/gpt-4o-mini": { p: "azure", i: 1.65e-7, o: 6.6e-7, cr: 7.5e-8 }, + "azure/gpt-4o-mini-2024-07-18": { p: "azure", i: 1.65e-7, o: 6.6e-7, cr: 7.5e-8 }, + "azure/gpt-4o-mini-audio-preview-2024-12-17": { p: "azure", i: 0.0000025, o: 0.00001 }, + "azure/gpt-5": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5-2025-08-07": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5-chat": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5-chat-latest": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5-codex": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5-mini": { p: "azure", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "azure/gpt-5-mini-2025-08-07": { p: "azure", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "azure/gpt-5-nano": { p: "azure", i: 5e-8, o: 4e-7, cr: 5e-9 }, + "azure/gpt-5-nano-2025-08-07": { p: "azure", i: 5e-8, o: 4e-7, cr: 5e-9 }, + "azure/gpt-5-pro": { p: "azure", i: 0.000015, o: 0.00012 }, + "azure/gpt-5.1": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-2025-11-13": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-chat": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-chat-2025-11-13": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-codex": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-codex-2025-11-13": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-codex-max": { p: "azure", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "azure/gpt-5.1-codex-mini": { p: "azure", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "azure/gpt-5.1-codex-mini-2025-11-13": { p: "azure", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "azure/gpt-5.2": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.2-2025-12-11": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.2-chat": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.2-chat-2025-12-11": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.2-codex": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.2-pro": { p: "azure", i: 0.000021, o: 0.000168 }, + "azure/gpt-5.2-pro-2025-12-11": { p: "azure", i: 0.000021, o: 0.000168 }, + "azure/gpt-5.3-chat": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.3-codex": { p: "azure", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "azure/gpt-5.4": { p: "azure", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "azure/gpt-5.4-2026-03-05": { p: "azure", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "azure/gpt-5.4-mini": { p: "azure", i: 7.5e-7, o: 0.0000045, cr: 7.5e-8 }, + "azure/gpt-5.4-mini-2026-03-17": { p: "azure", i: 7.5e-7, o: 0.0000045, cr: 7.5e-8 }, + "azure/gpt-5.4-nano": { p: "azure", i: 2e-7, o: 0.00000125, cr: 2e-8 }, + "azure/gpt-5.4-nano-2026-03-17": { p: "azure", i: 2e-7, o: 0.00000125, cr: 2e-8 }, + "azure/gpt-5.4-pro": { p: "azure", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "azure/gpt-5.4-pro-2026-03-05": { p: "azure", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "azure/gpt-5.5": { p: "azure", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "azure/gpt-5.5-2026-04-23": { p: "azure", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "azure/gpt-5.5-pro": { p: "azure", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "azure/gpt-5.5-pro-2026-04-23": { p: "azure", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "azure/gpt-5.6": { p: "azure", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "azure/gpt-5.6-luna": { p: "azure", i: 0.000001, o: 0.000006, cr: 1e-7 }, + "azure/gpt-5.6-sol": { p: "azure", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "azure/gpt-5.6-terra": { p: "azure", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "azure/gpt-audio-1.5-2026-02-23": { p: "azure", i: 0.0000025, o: 0.00001 }, + "azure/gpt-audio-2025-08-28": { p: "azure", i: 0.0000025, o: 0.00001 }, + "azure/gpt-audio-mini-2025-10-06": { p: "azure", i: 6e-7, o: 0.0000024 }, + "azure/mistral-large-2402": { p: "azure", i: 0.000008, o: 0.000024 }, + "azure/mistral-large-latest": { p: "azure", i: 0.000008, o: 0.000024 }, + "azure/o1": { p: "azure", i: 0.000015, o: 0.00006, cr: 0.0000075 }, + "azure/o1-2024-12-17": { p: "azure", i: 0.000015, o: 0.00006, cr: 0.0000075 }, + "azure/o1-mini": { p: "azure", i: 0.00000121, o: 0.00000484, cr: 6.05e-7 }, + "azure/o1-mini-2024-09-12": { p: "azure", i: 0.0000011, o: 0.0000044, cr: 5.5e-7 }, + "azure/o1-preview": { p: "azure", i: 0.000015, o: 0.00006, cr: 0.0000075 }, + "azure/o1-preview-2024-09-12": { p: "azure", i: 0.000015, o: 0.00006, cr: 0.0000075 }, + "azure/o3": { p: "azure", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "azure/o3-2025-04-16": { p: "azure", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "azure/o3-deep-research": { p: "azure", i: 0.00001, o: 0.00004, cr: 0.0000025 }, + "azure/o3-mini": { p: "azure", i: 0.0000011, o: 0.0000044, cr: 5.5e-7 }, + "azure/o3-mini-2025-01-31": { p: "azure", i: 0.0000011, o: 0.0000044, cr: 5.5e-7 }, + "azure/o3-pro": { p: "azure", i: 0.00002, o: 0.00008 }, + "azure/o3-pro-2025-06-10": { p: "azure", i: 0.00002, o: 0.00008 }, + "azure/o4-mini": { p: "azure", i: 0.0000011, o: 0.0000044, cr: 2.75e-7 }, + "azure/o4-mini-2025-04-16": { p: "azure", i: 0.0000011, o: 0.0000044, cr: 2.75e-7 }, + "azure/us/gpt-4.1-2025-04-14": { p: "azure", i: 0.0000022, o: 0.0000088, cr: 5.5e-7 }, + "azure/us/gpt-4.1-mini-2025-04-14": { p: "azure", i: 4.4e-7, o: 0.00000176, cr: 1.1e-7 }, + "azure/us/gpt-4.1-nano-2025-04-14": { p: "azure", i: 1.1e-7, o: 4.4e-7, cr: 2.5e-8 }, + "azure/us/gpt-4o-2024-08-06": { p: "azure", i: 0.00000275, o: 0.000011, cr: 0.000001375 }, + "azure/us/gpt-4o-2024-11-20": { p: "azure", i: 0.00000275, o: 0.000011, cw: 0.00000138 }, + "azure/us/gpt-4o-mini-2024-07-18": { p: "azure", i: 1.65e-7, o: 6.6e-7, cr: 8.3e-8 }, + "azure/us/gpt-5-2025-08-07": { p: "azure", i: 0.000001375, o: 0.000011, cr: 1.375e-7 }, + "azure/us/gpt-5-mini-2025-08-07": { p: "azure", i: 2.75e-7, o: 0.0000022, cr: 2.75e-8 }, + "azure/us/gpt-5-nano-2025-08-07": { p: "azure", i: 5.5e-8, o: 4.4e-7, cr: 5.5e-9 }, + "azure/us/gpt-5.1": { p: "azure", i: 0.00000138, o: 0.000011, cr: 1.4e-7 }, + "azure/us/gpt-5.1-chat": { p: "azure", i: 0.00000138, o: 0.000011, cr: 1.4e-7 }, + "azure/us/gpt-5.1-codex": { p: "azure", i: 0.00000138, o: 0.000011, cr: 1.4e-7 }, + "azure/us/gpt-5.1-codex-mini": { p: "azure", i: 2.75e-7, o: 0.0000022, cr: 2.8e-8 }, + "azure/us/gpt-5.4": { p: "azure", i: 0.00000275, o: 0.0000165, cr: 2.8e-7 }, + "azure/us/gpt-5.4-2026-03-05": { p: "azure", i: 0.00000275, o: 0.0000165, cr: 2.8e-7 }, + "azure/us/gpt-5.5": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/us/gpt-5.5-2026-04-23": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/us/gpt-5.6": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/us/gpt-5.6-luna": { p: "azure", i: 0.0000011, o: 0.0000066, cr: 1.1e-7 }, + "azure/us/gpt-5.6-sol": { p: "azure", i: 0.0000055, o: 0.000033, cr: 5.5e-7 }, + "azure/us/gpt-5.6-terra": { p: "azure", i: 0.00000275, o: 0.0000165, cr: 2.75e-7 }, + "azure/us/o1-2024-12-17": { p: "azure", i: 0.0000165, o: 0.000066, cr: 0.00000825 }, + "azure/us/o1-mini-2024-09-12": { p: "azure", i: 0.00000121, o: 0.00000484, cr: 6.05e-7 }, + "azure/us/o1-preview-2024-09-12": { p: "azure", i: 0.0000165, o: 0.000066, cr: 0.00000825 }, + "azure/us/o3-2025-04-16": { p: "azure", i: 0.0000022, o: 0.0000088, cr: 5.5e-7 }, + "azure/us/o3-mini-2025-01-31": { p: "azure", i: 0.00000121, o: 0.00000484, cr: 6.05e-7 }, + "azure/us/o4-mini-2025-04-16": { p: "azure", i: 0.00000121, o: 0.00000484, cr: 3.1e-7 }, + "azure_ai/Llama-3.2-11B-Vision-Instruct": { p: "azure_ai", i: 3.7e-7, o: 3.7e-7 }, + "azure_ai/Llama-3.2-90B-Vision-Instruct": { p: "azure_ai", i: 0.00000204, o: 0.00000204 }, + "azure_ai/Llama-3.3-70B-Instruct": { p: "azure_ai", i: 7.1e-7, o: 7.1e-7 }, + "azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8": { p: "azure_ai", i: 0.00000141, o: 3.5e-7 }, + "azure_ai/Llama-4-Scout-17B-16E-Instruct": { p: "azure_ai", i: 2e-7, o: 7.8e-7 }, + "azure_ai/MAI-DS-R1": { p: "azure_ai", i: 0.00000135, o: 0.0000054 }, + "azure_ai/Meta-Llama-3-70B-Instruct": { p: "azure_ai", i: 0.0000011, o: 3.7e-7 }, + "azure_ai/Meta-Llama-3.1-405B-Instruct": { p: "azure_ai", i: 0.00000533, o: 0.000016 }, + "azure_ai/Meta-Llama-3.1-70B-Instruct": { p: "azure_ai", i: 0.00000268, o: 0.00000354 }, + "azure_ai/Meta-Llama-3.1-8B-Instruct": { p: "azure_ai", i: 3e-7, o: 6.1e-7 }, + "azure_ai/Phi-3-medium-128k-instruct": { p: "azure_ai", i: 1.7e-7, o: 6.8e-7 }, + "azure_ai/Phi-3-medium-4k-instruct": { p: "azure_ai", i: 1.7e-7, o: 6.8e-7 }, + "azure_ai/Phi-3-mini-128k-instruct": { p: "azure_ai", i: 1.3e-7, o: 5.2e-7 }, + "azure_ai/Phi-3-mini-4k-instruct": { p: "azure_ai", i: 1.3e-7, o: 5.2e-7 }, + "azure_ai/Phi-3-small-128k-instruct": { p: "azure_ai", i: 1.5e-7, o: 6e-7 }, + "azure_ai/Phi-3-small-8k-instruct": { p: "azure_ai", i: 1.5e-7, o: 6e-7 }, + "azure_ai/Phi-3.5-MoE-instruct": { p: "azure_ai", i: 1.6e-7, o: 6.4e-7 }, + "azure_ai/Phi-3.5-mini-instruct": { p: "azure_ai", i: 1.3e-7, o: 5.2e-7 }, + "azure_ai/Phi-3.5-vision-instruct": { p: "azure_ai", i: 1.3e-7, o: 5.2e-7 }, + "azure_ai/Phi-4": { p: "azure_ai", i: 1.25e-7, o: 5e-7 }, + "azure_ai/Phi-4-mini-instruct": { p: "azure_ai", i: 7.5e-8, o: 3e-7 }, + "azure_ai/Phi-4-mini-reasoning": { p: "azure_ai", i: 8e-8, o: 3.2e-7 }, + "azure_ai/Phi-4-multimodal-instruct": { p: "azure_ai", i: 8e-8, o: 3.2e-7 }, + "azure_ai/Phi-4-reasoning": { p: "azure_ai", i: 1.25e-7, o: 5e-7 }, + "azure_ai/claude-fable-5": { p: "azure_ai", i: 0.00001, o: 0.00005, cr: 0.000001, cw: 0.0000125 }, + "azure_ai/claude-haiku-4-5": { + p: "azure_ai", + i: 0.000001, + o: 0.000005, + cr: 1e-7, + cw: 0.00000125, + }, + "azure_ai/claude-opus-4-1": { + p: "azure_ai", + i: 0.000015, + o: 0.000075, + cr: 0.0000015, + cw: 0.00001875, + }, + "azure_ai/claude-opus-4-5": { p: "azure_ai", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "azure_ai/claude-opus-4-6": { p: "azure_ai", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "azure_ai/claude-opus-4-7": { p: "azure_ai", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "azure_ai/claude-opus-4-8": { p: "azure_ai", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "azure_ai/claude-opus-5": { p: "azure_ai", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "azure_ai/claude-sonnet-4-5": { + p: "azure_ai", + i: 0.000003, + o: 0.000015, + cr: 3e-7, + cw: 0.00000375, + }, + "azure_ai/claude-sonnet-4-6": { + p: "azure_ai", + i: 0.000003, + o: 0.000015, + cr: 3e-7, + cw: 0.00000375, + }, + "azure_ai/claude-sonnet-5": { p: "azure_ai", i: 0.000002, o: 0.00001, cr: 2e-7, cw: 0.0000025 }, + "azure_ai/deepseek-r1": { p: "azure_ai", i: 0.00000135, o: 0.0000054 }, + "azure_ai/deepseek-v3": { p: "azure_ai", i: 0.00000114, o: 0.00000456 }, + "azure_ai/deepseek-v3-0324": { p: "azure_ai", i: 0.00000114, o: 0.00000456 }, + "azure_ai/deepseek-v3.1": { p: "azure_ai", i: 0.00000123, o: 0.00000494 }, + "azure_ai/deepseek-v3.2": { p: "azure_ai", i: 5.8e-7, o: 0.00000168 }, + "azure_ai/deepseek-v3.2-speciale": { p: "azure_ai", i: 5.8e-7, o: 0.00000168 }, + "azure_ai/deepseek-v4-flash": { p: "azure_ai", i: 1.9e-7, o: 5.1e-7 }, + "azure_ai/deepseek-v4-pro": { p: "azure_ai", i: 0.00000174, o: 0.00000348 }, + "azure_ai/global/grok-3": { p: "azure_ai", i: 0.000003, o: 0.000015 }, + "azure_ai/global/grok-3-mini": { p: "azure_ai", i: 2.5e-7, o: 0.00000127 }, + "azure_ai/gpt-5.4": { p: "azure_ai", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "azure_ai/gpt-5.4-2026-03-05": { p: "azure_ai", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "azure_ai/gpt-5.4-mini": { p: "azure_ai", i: 7.5e-7, o: 0.0000045, cr: 7.5e-8 }, + "azure_ai/gpt-5.4-mini-2026-03-17": { p: "azure_ai", i: 7.5e-7, o: 0.0000045, cr: 7.5e-8 }, + "azure_ai/gpt-5.4-nano": { p: "azure_ai", i: 2e-7, o: 0.00000125, cr: 2e-8 }, + "azure_ai/gpt-5.4-nano-2026-03-17": { p: "azure_ai", i: 2e-7, o: 0.00000125, cr: 2e-8 }, + "azure_ai/gpt-5.4-pro": { p: "azure_ai", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "azure_ai/gpt-5.4-pro-2026-03-05": { p: "azure_ai", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "azure_ai/gpt-5.5": { p: "azure_ai", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "azure_ai/gpt-5.5-2026-04-23": { p: "azure_ai", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "azure_ai/gpt-oss-120b": { p: "azure_ai", i: 1.5e-7, o: 6e-7 }, + "azure_ai/grok-3": { p: "azure_ai", i: 0.000003, o: 0.000015 }, + "azure_ai/grok-3-mini": { p: "azure_ai", i: 2.5e-7, o: 0.00000127 }, + "azure_ai/grok-4": { p: "azure_ai", i: 0.000003, o: 0.000015 }, + "azure_ai/grok-4-1-fast-non-reasoning": { p: "azure_ai", i: 2e-7, o: 5e-7 }, + "azure_ai/grok-4-1-fast-reasoning": { p: "azure_ai", i: 2e-7, o: 5e-7 }, + "azure_ai/grok-4-fast-non-reasoning": { p: "azure_ai", i: 2e-7, o: 5e-7 }, + "azure_ai/grok-4-fast-reasoning": { p: "azure_ai", i: 2e-7, o: 5e-7 }, + "azure_ai/grok-code-fast-1": { p: "azure_ai", i: 2e-7, o: 0.0000015 }, + "azure_ai/jais-30b-chat": { p: "azure_ai", i: 0.0032, o: 0.00971 }, + "azure_ai/jamba-instruct": { p: "azure_ai", i: 5e-7, o: 7e-7 }, + "azure_ai/kimi-k2.5": { p: "azure_ai", i: 6e-7, o: 0.000003 }, + "azure_ai/kimi-k2.6": { p: "azure_ai", i: 9.5e-7, o: 0.000004 }, + "azure_ai/ministral-3b": { p: "azure_ai", i: 4e-8, o: 4e-8 }, + "azure_ai/mistral-large": { p: "azure_ai", i: 0.000004, o: 0.000012 }, + "azure_ai/mistral-large-2407": { p: "azure_ai", i: 0.000002, o: 0.000006 }, + "azure_ai/mistral-large-3": { p: "azure_ai", i: 5e-7, o: 0.0000015 }, + "azure_ai/mistral-large-latest": { p: "azure_ai", i: 0.000002, o: 0.000006 }, + "azure_ai/mistral-medium-2505": { p: "azure_ai", i: 4e-7, o: 0.000002 }, + "azure_ai/mistral-nemo": { p: "azure_ai", i: 1.5e-7, o: 1.5e-7 }, + "azure_ai/mistral-small": { p: "azure_ai", i: 0.000001, o: 0.000003 }, + "azure_ai/mistral-small-2503": { p: "azure_ai", i: 1e-7, o: 3e-7 }, + "azure_ai/model_router": { p: "azure_ai", i: 1.4e-7, o: 0 }, + "chatgpt-4o-latest": { p: "openai", i: 0.000005, o: 0.000015 }, + "claude-3-7-sonnet-20250219": { + p: "anthropic", + i: 0.000003, + o: 0.000015, + cr: 3e-7, + cw: 0.00000375, + }, + "claude-3-haiku-20240307": { p: "anthropic", i: 2.5e-7, o: 0.00000125, cr: 3e-8, cw: 3e-7 }, + "claude-3-opus-20240229": { + p: "anthropic", + i: 0.000015, + o: 0.000075, + cr: 0.0000015, + cw: 0.00001875, + }, + "claude-4-opus-20250514": { + p: "anthropic", + i: 0.000015, + o: 0.000075, + cr: 0.0000015, + cw: 0.00001875, + }, + "claude-4-sonnet-20250514": { + p: "anthropic", + i: 0.000003, + o: 0.000015, + cr: 3e-7, + cw: 0.00000375, + }, + "claude-fable-5": { p: "anthropic", i: 0.00001, o: 0.00005, cr: 0.000001, cw: 0.0000125 }, + "claude-haiku-4-5": { p: "anthropic", i: 0.000001, o: 0.000005, cr: 1e-7, cw: 0.00000125 }, + "claude-haiku-4-5-20251001": { + p: "anthropic", + i: 0.000001, + o: 0.000005, + cr: 1e-7, + cw: 0.00000125, + }, + "claude-opus-4-1": { p: "anthropic", i: 0.000015, o: 0.000075, cr: 0.0000015, cw: 0.00001875 }, + "claude-opus-4-1-20250805": { + p: "anthropic", + i: 0.000015, + o: 0.000075, + cr: 0.0000015, + cw: 0.00001875, + }, + "claude-opus-4-20250514": { + p: "anthropic", + i: 0.000015, + o: 0.000075, + cr: 0.0000015, + cw: 0.00001875, + }, + "claude-opus-4-5": { p: "anthropic", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "claude-opus-4-5-20251101": { + p: "anthropic", + i: 0.000005, + o: 0.000025, + cr: 5e-7, + cw: 0.00000625, + }, + "claude-opus-4-6": { p: "anthropic", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "claude-opus-4-6-20260205": { + p: "anthropic", + i: 0.000005, + o: 0.000025, + cr: 5e-7, + cw: 0.00000625, + }, + "claude-opus-4-7": { p: "anthropic", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "claude-opus-4-7-20260416": { + p: "anthropic", + i: 0.000005, + o: 0.000025, + cr: 5e-7, + cw: 0.00000625, + }, + "claude-opus-4-8": { p: "anthropic", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "claude-opus-5": { p: "anthropic", i: 0.000005, o: 0.000025, cr: 5e-7, cw: 0.00000625 }, + "claude-sonnet-4-20250514": { + p: "anthropic", + i: 0.000003, + o: 0.000015, + cr: 3e-7, + cw: 0.00000375, + }, + "claude-sonnet-4-5": { p: "anthropic", i: 0.000003, o: 0.000015, cr: 3e-7, cw: 0.00000375 }, + "claude-sonnet-4-5-20250929": { + p: "anthropic", + i: 0.000003, + o: 0.000015, + cr: 3e-7, + cw: 0.00000375, + }, + "claude-sonnet-4-6": { p: "anthropic", i: 0.000003, o: 0.000015, cr: 3e-7, cw: 0.00000375 }, + "claude-sonnet-5": { p: "anthropic", i: 0.000002, o: 0.00001, cr: 2e-7, cw: 0.0000025 }, + "codex-mini-latest": { p: "openai", i: 0.0000015, o: 0.000006, cr: 3.75e-7 }, + "computer-use-preview": { p: "azure", i: 0.000003, o: 0.000012 }, + "deepseek-chat": { p: "deepseek", i: 2.8e-7, o: 4.2e-7, cr: 2.8e-8 }, + "deepseek-reasoner": { p: "deepseek", i: 2.8e-7, o: 4.2e-7, cr: 2.8e-8 }, + "deepseek-v4-flash": { p: "deepseek", i: 1.4e-7, o: 2.8e-7, cr: 2.8e-9, cw: 0 }, + "deepseek-v4-pro": { p: "deepseek", i: 4.35e-7, o: 8.7e-7, cr: 3.625e-9, cw: 0 }, + "deepseek/deepseek-chat": { p: "deepseek", i: 2.8e-7, o: 4.2e-7, cr: 2.8e-8, cw: 0 }, + "deepseek/deepseek-coder": { p: "deepseek", i: 1.4e-7, o: 2.8e-7 }, + "deepseek/deepseek-r1": { p: "deepseek", i: 5.5e-7, o: 0.00000219 }, + "deepseek/deepseek-reasoner": { p: "deepseek", i: 2.8e-7, o: 4.2e-7, cr: 2.8e-8 }, + "deepseek/deepseek-v3": { p: "deepseek", i: 2.7e-7, o: 0.0000011, cr: 7e-8, cw: 0 }, + "deepseek/deepseek-v3.2": { p: "deepseek", i: 2.8e-7, o: 4e-7 }, + "deepseek/deepseek-v4-flash": { p: "deepseek", i: 1.4e-7, o: 2.8e-7, cr: 2.8e-9, cw: 0 }, + "deepseek/deepseek-v4-pro": { p: "deepseek", i: 4.35e-7, o: 8.7e-7, cr: 3.625e-9, cw: 0 }, + "ft:gpt-3.5-turbo": { p: "openai", i: 0.000003, o: 0.000006 }, + "ft:gpt-3.5-turbo-0125": { p: "openai", i: 0.000003, o: 0.000006 }, + "ft:gpt-3.5-turbo-0613": { p: "openai", i: 0.000003, o: 0.000006 }, + "ft:gpt-3.5-turbo-1106": { p: "openai", i: 0.000003, o: 0.000006 }, + "ft:gpt-4-0613": { p: "openai", i: 0.00003, o: 0.00006 }, + "ft:gpt-4.1-2025-04-14": { p: "openai", i: 0.000003, o: 0.000012, cr: 7.5e-7 }, + "ft:gpt-4.1-mini-2025-04-14": { p: "openai", i: 8e-7, o: 0.0000032, cr: 2e-7 }, + "ft:gpt-4.1-nano-2025-04-14": { p: "openai", i: 2e-7, o: 8e-7, cr: 5e-8 }, + "ft:gpt-4o-2024-08-06": { p: "openai", i: 0.00000375, o: 0.000015, cr: 0.000001875 }, + "ft:gpt-4o-2024-11-20": { p: "openai", i: 0.00000375, o: 0.000015, cw: 0.000001875 }, + "ft:gpt-4o-mini-2024-07-18": { p: "openai", i: 3e-7, o: 0.0000012, cr: 1.5e-7 }, + "ft:o4-mini-2025-04-16": { p: "openai", i: 0.000004, o: 0.000016, cr: 0.000001 }, + "gemini-2.0-flash": { p: "vertex_ai-language-models", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gemini-2.0-flash-001": { p: "vertex_ai-language-models", i: 1.5e-7, o: 6e-7, cr: 3.75e-8 }, + "gemini-2.0-flash-lite": { p: "vertex_ai-language-models", i: 7.5e-8, o: 3e-7, cr: 1.875e-8 }, + "gemini-2.0-flash-lite-001": { p: "vertex_ai-language-models", i: 7.5e-8, o: 3e-7, cr: 1.875e-8 }, + "gemini-2.5-computer-use-preview-10-2025": { + p: "vertex_ai-language-models", + i: 0.00000125, + o: 0.00001, + }, + "gemini-2.5-flash": { p: "vertex_ai-language-models", i: 3e-7, o: 0.0000025, cr: 3e-8 }, + "gemini-2.5-flash-lite": { p: "vertex_ai-language-models", i: 1e-7, o: 4e-7, cr: 1e-8 }, + "gemini-2.5-flash-lite-preview-06-17": { + p: "vertex_ai-language-models", + i: 1e-7, + o: 4e-7, + cr: 2.5e-8, + }, + "gemini-2.5-flash-lite-preview-09-2025": { + p: "vertex_ai-language-models", + i: 1e-7, + o: 4e-7, + cr: 1e-8, + }, + "gemini-2.5-flash-native-audio-latest": { p: "gemini", i: 3e-7, o: 0.0000025 }, + "gemini-2.5-flash-native-audio-preview-09-2025": { p: "gemini", i: 3e-7, o: 0.0000025 }, + "gemini-2.5-flash-native-audio-preview-12-2025": { p: "gemini", i: 3e-7, o: 0.0000025 }, + "gemini-2.5-flash-preview-09-2025": { + p: "vertex_ai-language-models", + i: 3e-7, + o: 0.0000025, + cr: 7.5e-8, + }, + "gemini-2.5-pro": { p: "vertex_ai-language-models", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gemini-2.5-pro-preview-tts": { + p: "vertex_ai-language-models", + i: 0.00000125, + o: 0.00001, + cr: 1.25e-7, + }, + "gemini-3-flash-preview": { p: "vertex_ai-language-models", i: 5e-7, o: 0.000003, cr: 5e-8 }, + "gemini-3-pro-preview": { p: "vertex_ai-language-models", i: 0.000002, o: 0.000012, cr: 2e-7 }, + "gemini-3.1-flash-lite": { p: "vertex_ai-language-models", i: 2.5e-7, o: 0.0000015, cr: 2.5e-8 }, + "gemini-3.1-flash-lite-preview": { + p: "vertex_ai-language-models", + i: 2.5e-7, + o: 0.0000015, + cr: 2.5e-8, + }, + "gemini-3.1-flash-live-preview": { p: "gemini", i: 7.5e-7, o: 0.0000045 }, + "gemini-3.1-pro-preview": { p: "vertex_ai-language-models", i: 0.000002, o: 0.000012, cr: 2e-7 }, + "gemini-3.1-pro-preview-customtools": { + p: "vertex_ai-language-models", + i: 0.000002, + o: 0.000012, + cr: 2e-7, + }, + "gemini-3.5-flash": { p: "vertex_ai-language-models", i: 0.0000015, o: 0.000009, cr: 1.5e-7 }, + "gemini-3.5-flash-lite": { p: "vertex_ai-language-models", i: 3e-7, o: 0.0000025, cr: 3e-8 }, + "gemini-3.6-flash": { p: "vertex_ai-language-models", i: 0.0000015, o: 0.0000075, cr: 1.5e-7 }, + "gemini-exp-1206": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 3e-8 }, + "gemini-flash-latest": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 3e-8 }, + "gemini-flash-lite-latest": { p: "gemini", i: 1e-7, o: 4e-7, cr: 1e-8 }, + "gemini-omni-flash-preview": { p: "vertex_ai-language-models", i: 0.0000015, o: 0.000009 }, + "gemini-pro-latest": { p: "gemini", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gemini-robotics-er-1.5-preview": { + p: "vertex_ai-language-models", + i: 3e-7, + o: 0.0000025, + cr: 0, + }, + "gemini/gemini-2.0-flash": { p: "gemini", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gemini/gemini-2.0-flash-001": { p: "gemini", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gemini/gemini-2.0-flash-lite": { p: "gemini", i: 7.5e-8, o: 3e-7, cr: 1.875e-8 }, + "gemini/gemini-2.0-flash-lite-001": { p: "gemini", i: 7.5e-8, o: 3e-7, cr: 1.875e-8 }, + "gemini/gemini-2.5-computer-use-preview-10-2025": { p: "gemini", i: 0.00000125, o: 0.00001 }, + "gemini/gemini-2.5-flash": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 3e-8 }, + "gemini/gemini-2.5-flash-lite": { p: "gemini", i: 1e-7, o: 4e-7, cr: 1e-8 }, + "gemini/gemini-2.5-flash-lite-preview-06-17": { p: "gemini", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gemini/gemini-2.5-flash-lite-preview-09-2025": { p: "gemini", i: 1e-7, o: 4e-7, cr: 1e-8 }, + "gemini/gemini-2.5-flash-native-audio-latest": { p: "gemini", i: 3e-7, o: 0.0000025 }, + "gemini/gemini-2.5-flash-native-audio-preview-09-2025": { p: "gemini", i: 3e-7, o: 0.0000025 }, + "gemini/gemini-2.5-flash-native-audio-preview-12-2025": { p: "gemini", i: 3e-7, o: 0.0000025 }, + "gemini/gemini-2.5-flash-preview-09-2025": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 7.5e-8 }, + "gemini/gemini-2.5-pro": { p: "gemini", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gemini/gemini-2.5-pro-preview-tts": { p: "gemini", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gemini/gemini-3-flash-preview": { p: "gemini", i: 5e-7, o: 0.000003, cr: 5e-8 }, + "gemini/gemini-3-pro-preview": { p: "gemini", i: 0.000002, o: 0.000012, cr: 2e-7 }, + "gemini/gemini-3.1-flash-lite": { p: "gemini", i: 2.5e-7, o: 0.0000015, cr: 2.5e-8 }, + "gemini/gemini-3.1-flash-lite-preview": { p: "gemini", i: 2.5e-7, o: 0.0000015, cr: 2.5e-8 }, + "gemini/gemini-3.1-flash-live-preview": { p: "gemini", i: 7.5e-7, o: 0.0000045 }, + "gemini/gemini-3.1-pro-preview": { p: "gemini", i: 0.000002, o: 0.000012, cr: 2e-7 }, + "gemini/gemini-3.1-pro-preview-customtools": { p: "gemini", i: 0.000002, o: 0.000012, cr: 2e-7 }, + "gemini/gemini-3.5-flash": { p: "gemini", i: 0.0000015, o: 0.000009, cr: 1.5e-7 }, + "gemini/gemini-3.5-flash-lite": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 3e-8 }, + "gemini/gemini-3.6-flash": { p: "gemini", i: 0.0000015, o: 0.0000075, cr: 1.5e-7 }, + "gemini/gemini-exp-1114": { p: "gemini", i: 0, o: 0 }, + "gemini/gemini-exp-1206": { p: "gemini", i: 0, o: 0 }, + "gemini/gemini-flash-latest": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 7.5e-8 }, + "gemini/gemini-flash-lite-latest": { p: "gemini", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gemini/gemini-gemma-2-27b-it": { p: "gemini", i: 3.5e-7, o: 0.00000105 }, + "gemini/gemini-gemma-2-9b-it": { p: "gemini", i: 3.5e-7, o: 0.00000105 }, + "gemini/gemini-omni-flash-preview": { p: "gemini", i: 0.0000015, o: 0.000009 }, + "gemini/gemini-pro-latest": { p: "gemini", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gemini/gemini-robotics-er-1.5-preview": { p: "gemini", i: 3e-7, o: 0.0000025, cr: 0 }, + "gemini/gemma-3-27b-it": { p: "gemini", i: 0, o: 0 }, + "gemini/learnlm-1.5-pro-experimental": { p: "gemini", i: 0, o: 0 }, + "gemini/lyria-3-clip-preview": { p: "gemini", i: 0, o: 0 }, + "gemini/lyria-3-pro-preview": { p: "gemini", i: 0, o: 0 }, + "gpt-3.5-turbo": { p: "openai", i: 5e-7, o: 0.0000015 }, + "gpt-3.5-turbo-0125": { p: "openai", i: 5e-7, o: 0.0000015 }, + "gpt-3.5-turbo-1106": { p: "openai", i: 0.000001, o: 0.000002 }, + "gpt-3.5-turbo-16k": { p: "openai", i: 0.000003, o: 0.000004 }, + "gpt-4": { p: "openai", i: 0.00003, o: 0.00006 }, + "gpt-4-0125-preview": { p: "openai", i: 0.00001, o: 0.00003 }, + "gpt-4-0314": { p: "openai", i: 0.00003, o: 0.00006 }, + "gpt-4-0613": { p: "openai", i: 0.00003, o: 0.00006 }, + "gpt-4-1106-preview": { p: "openai", i: 0.00001, o: 0.00003 }, + "gpt-4-turbo": { p: "openai", i: 0.00001, o: 0.00003 }, + "gpt-4-turbo-2024-04-09": { p: "openai", i: 0.00001, o: 0.00003 }, + "gpt-4-turbo-preview": { p: "openai", i: 0.00001, o: 0.00003 }, + "gpt-4.1": { p: "openai", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "gpt-4.1-2025-04-14": { p: "openai", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "gpt-4.1-mini": { p: "openai", i: 4e-7, o: 0.0000016, cr: 1e-7 }, + "gpt-4.1-mini-2025-04-14": { p: "openai", i: 4e-7, o: 0.0000016, cr: 1e-7 }, + "gpt-4.1-nano": { p: "openai", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gpt-4.1-nano-2025-04-14": { p: "openai", i: 1e-7, o: 4e-7, cr: 2.5e-8 }, + "gpt-4o": { p: "openai", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "gpt-4o-2024-05-13": { p: "openai", i: 0.000005, o: 0.000015 }, + "gpt-4o-2024-08-06": { p: "openai", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "gpt-4o-2024-11-20": { p: "openai", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "gpt-4o-audio-preview": { p: "openai", i: 0.0000025, o: 0.00001 }, + "gpt-4o-audio-preview-2024-12-17": { p: "openai", i: 0.0000025, o: 0.00001 }, + "gpt-4o-audio-preview-2025-06-03": { p: "openai", i: 0.0000025, o: 0.00001 }, + "gpt-4o-mini": { p: "openai", i: 1.5e-7, o: 6e-7, cr: 7.5e-8 }, + "gpt-4o-mini-2024-07-18": { p: "openai", i: 1.5e-7, o: 6e-7, cr: 7.5e-8 }, + "gpt-4o-mini-audio-preview": { p: "openai", i: 1.5e-7, o: 6e-7 }, + "gpt-4o-mini-audio-preview-2024-12-17": { p: "openai", i: 1.5e-7, o: 6e-7 }, + "gpt-4o-mini-search-preview": { p: "openai", i: 1.5e-7, o: 6e-7, cr: 7.5e-8 }, + "gpt-4o-mini-search-preview-2025-03-11": { p: "openai", i: 1.5e-7, o: 6e-7, cr: 7.5e-8 }, + "gpt-4o-search-preview": { p: "openai", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "gpt-4o-search-preview-2025-03-11": { p: "openai", i: 0.0000025, o: 0.00001, cr: 0.00000125 }, + "gpt-5": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5-2025-08-07": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5-chat": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5-chat-latest": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5-codex": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5-mini": { p: "openai", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "gpt-5-mini-2025-08-07": { p: "openai", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "gpt-5-nano": { p: "openai", i: 5e-8, o: 4e-7, cr: 5e-9 }, + "gpt-5-nano-2025-08-07": { p: "openai", i: 5e-8, o: 4e-7, cr: 5e-9 }, + "gpt-5-pro": { p: "openai", i: 0.000015, o: 0.00012 }, + "gpt-5-pro-2025-10-06": { p: "openai", i: 0.000015, o: 0.00012 }, + "gpt-5-search-api": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5-search-api-2025-10-14": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5.1": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5.1-2025-11-13": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5.1-chat-latest": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5.1-codex": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5.1-codex-max": { p: "openai", i: 0.00000125, o: 0.00001, cr: 1.25e-7 }, + "gpt-5.1-codex-mini": { p: "openai", i: 2.5e-7, o: 0.000002, cr: 2.5e-8 }, + "gpt-5.2": { p: "openai", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "gpt-5.2-2025-12-11": { p: "openai", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "gpt-5.2-chat-latest": { p: "openai", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "gpt-5.2-codex": { p: "openai", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "gpt-5.2-pro": { p: "openai", i: 0.000021, o: 0.000168 }, + "gpt-5.2-pro-2025-12-11": { p: "openai", i: 0.000021, o: 0.000168 }, + "gpt-5.3-chat-latest": { p: "openai", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "gpt-5.3-codex": { p: "openai", i: 0.00000175, o: 0.000014, cr: 1.75e-7 }, + "gpt-5.4": { p: "openai", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "gpt-5.4-2026-03-05": { p: "openai", i: 0.0000025, o: 0.000015, cr: 2.5e-7 }, + "gpt-5.4-mini": { p: "openai", i: 7.5e-7, o: 0.0000045, cr: 7.5e-8 }, + "gpt-5.4-mini-2026-03-17": { p: "openai", i: 7.5e-7, o: 0.0000045, cr: 7.5e-8 }, + "gpt-5.4-nano": { p: "openai", i: 2e-7, o: 0.00000125, cr: 2e-8 }, + "gpt-5.4-nano-2026-03-17": { p: "openai", i: 2e-7, o: 0.00000125, cr: 2e-8 }, + "gpt-5.4-pro": { p: "openai", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "gpt-5.4-pro-2026-03-05": { p: "openai", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "gpt-5.5": { p: "openai", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "gpt-5.5-2026-04-23": { p: "openai", i: 0.000005, o: 0.00003, cr: 5e-7 }, + "gpt-5.5-pro": { p: "openai", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "gpt-5.5-pro-2026-04-23": { p: "openai", i: 0.00003, o: 0.00018, cr: 0.000003 }, + "gpt-5.6": { p: "openai", i: 0.000005, o: 0.00003, cr: 5e-7, cw: 0.00000625 }, + "gpt-5.6-luna": { p: "openai", i: 2e-7, o: 0.0000012, cr: 2e-8, cw: 2.5e-7 }, + "gpt-5.6-sol": { p: "openai", i: 0.000005, o: 0.00003, cr: 5e-7, cw: 0.00000625 }, + "gpt-5.6-terra": { p: "openai", i: 0.000002, o: 0.000012, cr: 2e-7, cw: 0.0000025 }, + "gpt-audio": { p: "openai", i: 0.0000025, o: 0.00001 }, + "gpt-audio-1.5": { p: "openai", i: 0.0000025, o: 0.00001 }, + "gpt-audio-2025-08-28": { p: "openai", i: 0.0000025, o: 0.00001 }, + "gpt-audio-mini": { p: "openai", i: 6e-7, o: 0.0000024 }, + "gpt-audio-mini-2025-10-06": { p: "openai", i: 6e-7, o: 0.0000024 }, + "gpt-audio-mini-2025-12-15": { p: "openai", i: 6e-7, o: 0.0000024 }, + "groq/gemma-7b-it": { p: "groq", i: 5e-8, o: 8e-8 }, + "groq/llama-3.1-8b-instant": { p: "groq", i: 5e-8, o: 8e-8 }, + "groq/llama-3.3-70b-versatile": { p: "groq", i: 5.9e-7, o: 7.9e-7 }, + "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { p: "groq", i: 2e-7, o: 6e-7 }, + "groq/meta-llama/llama-4-scout-17b-16e-instruct": { p: "groq", i: 1.1e-7, o: 3.4e-7 }, + "groq/meta-llama/llama-guard-4-12b": { p: "groq", i: 2e-7, o: 2e-7 }, + "groq/moonshotai/kimi-k2-instruct-0905": { p: "groq", i: 0.000001, o: 0.000003, cr: 5e-7 }, + "groq/openai/gpt-oss-120b": { p: "groq", i: 1.5e-7, o: 6e-7, cr: 7.5e-8 }, + "groq/openai/gpt-oss-20b": { p: "groq", i: 7.5e-8, o: 3e-7, cr: 3.75e-8 }, + "groq/openai/gpt-oss-safeguard-20b": { p: "groq", i: 7.5e-8, o: 3e-7, cr: 3.7e-8 }, + "groq/qwen/qwen3-32b": { p: "groq", i: 2.9e-7, o: 5.9e-7 }, + "mistral/codestral-2405": { p: "mistral", i: 0.000001, o: 0.000003 }, + "mistral/codestral-2508": { p: "mistral", i: 3e-7, o: 9e-7 }, + "mistral/codestral-latest": { p: "mistral", i: 0.000001, o: 0.000003 }, + "mistral/codestral-mamba-latest": { p: "mistral", i: 2.5e-7, o: 2.5e-7 }, + "mistral/devstral-2512": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/devstral-latest": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/devstral-medium-2507": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/devstral-medium-latest": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/devstral-small-2505": { p: "mistral", i: 1e-7, o: 3e-7 }, + "mistral/devstral-small-2507": { p: "mistral", i: 1e-7, o: 3e-7 }, + "mistral/devstral-small-latest": { p: "mistral", i: 1e-7, o: 3e-7 }, + "mistral/labs-devstral-small-2512": { p: "mistral", i: 1e-7, o: 3e-7 }, + "mistral/magistral-medium-1-2-2509": { p: "mistral", i: 0.000002, o: 0.000005 }, + "mistral/magistral-medium-2506": { p: "mistral", i: 0.000002, o: 0.000005 }, + "mistral/magistral-medium-2509": { p: "mistral", i: 0.000002, o: 0.000005 }, + "mistral/magistral-medium-latest": { p: "mistral", i: 0.000002, o: 0.000005 }, + "mistral/magistral-small-1-2-2509": { p: "mistral", i: 5e-7, o: 0.0000015 }, + "mistral/magistral-small-2506": { p: "mistral", i: 5e-7, o: 0.0000015 }, + "mistral/magistral-small-latest": { p: "mistral", i: 5e-7, o: 0.0000015 }, + "mistral/ministral-3-14b-2512": { p: "mistral", i: 2e-7, o: 2e-7 }, + "mistral/ministral-3-3b-2512": { p: "mistral", i: 1e-7, o: 1e-7 }, + "mistral/ministral-3-8b-2512": { p: "mistral", i: 1.5e-7, o: 1.5e-7 }, + "mistral/ministral-8b-2512": { p: "mistral", i: 1.5e-7, o: 1.5e-7 }, + "mistral/ministral-8b-latest": { p: "mistral", i: 1.5e-7, o: 1.5e-7 }, + "mistral/mistral-large-2402": { p: "mistral", i: 0.000004, o: 0.000012 }, + "mistral/mistral-large-2407": { p: "mistral", i: 0.000003, o: 0.000009 }, + "mistral/mistral-large-2411": { p: "mistral", i: 0.000002, o: 0.000006 }, + "mistral/mistral-large-2512": { p: "mistral", i: 5e-7, o: 0.0000015 }, + "mistral/mistral-large-3": { p: "mistral", i: 5e-7, o: 0.0000015 }, + "mistral/mistral-large-latest": { p: "mistral", i: 5e-7, o: 0.0000015 }, + "mistral/mistral-medium": { p: "mistral", i: 0.0000027, o: 0.0000081 }, + "mistral/mistral-medium-2312": { p: "mistral", i: 0.0000027, o: 0.0000081 }, + "mistral/mistral-medium-2505": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/mistral-medium-2508": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/mistral-medium-2604": { p: "mistral", i: 0.0000015, o: 0.0000075 }, + "mistral/mistral-medium-3-1-2508": { p: "mistral", i: 4e-7, o: 0.000002 }, + "mistral/mistral-medium-3-5": { p: "mistral", i: 0.0000015, o: 0.0000075 }, + "mistral/mistral-medium-latest": { p: "mistral", i: 0.0000015, o: 0.0000075 }, + "mistral/mistral-small": { p: "mistral", i: 1e-7, o: 3e-7 }, + "mistral/mistral-small-3-2-2506": { p: "mistral", i: 6e-8, o: 1.8e-7 }, + "mistral/mistral-small-latest": { p: "mistral", i: 6e-8, o: 1.8e-7 }, + "mistral/mistral-tiny": { p: "mistral", i: 2.5e-7, o: 2.5e-7 }, + "mistral/open-codestral-mamba": { p: "mistral", i: 2.5e-7, o: 2.5e-7 }, + "mistral/open-mistral-7b": { p: "mistral", i: 2.5e-7, o: 2.5e-7 }, + "mistral/open-mistral-nemo": { p: "mistral", i: 3e-7, o: 3e-7 }, + "mistral/open-mistral-nemo-2407": { p: "mistral", i: 3e-7, o: 3e-7 }, + "mistral/open-mixtral-8x22b": { p: "mistral", i: 0.000002, o: 0.000006 }, + "mistral/open-mixtral-8x7b": { p: "mistral", i: 7e-7, o: 7e-7 }, + "mistral/pixtral-12b-2409": { p: "mistral", i: 1.5e-7, o: 1.5e-7 }, + "mistral/pixtral-large-2411": { p: "mistral", i: 0.000002, o: 0.000006 }, + "mistral/pixtral-large-latest": { p: "mistral", i: 0.000002, o: 0.000006 }, + o1: { p: "openai", i: 0.000015, o: 0.00006, cr: 0.0000075 }, + "o1-2024-12-17": { p: "openai", i: 0.000015, o: 0.00006, cr: 0.0000075 }, + "o1-pro": { p: "openai", i: 0.00015, o: 0.0006 }, + "o1-pro-2025-03-19": { p: "openai", i: 0.00015, o: 0.0006 }, + o3: { p: "openai", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "o3-2025-04-16": { p: "openai", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "o3-deep-research": { p: "openai", i: 0.00001, o: 0.00004, cr: 0.0000025 }, + "o3-deep-research-2025-06-26": { p: "openai", i: 0.00001, o: 0.00004, cr: 0.0000025 }, + "o3-mini": { p: "openai", i: 0.0000011, o: 0.0000044, cr: 5.5e-7 }, + "o3-mini-2025-01-31": { p: "openai", i: 0.0000011, o: 0.0000044, cr: 5.5e-7 }, + "o3-pro": { p: "openai", i: 0.00002, o: 0.00008 }, + "o3-pro-2025-06-10": { p: "openai", i: 0.00002, o: 0.00008 }, + "o4-mini": { p: "openai", i: 0.0000011, o: 0.0000044, cr: 2.75e-7 }, + "o4-mini-2025-04-16": { p: "openai", i: 0.0000011, o: 0.0000044, cr: 2.75e-7 }, + "o4-mini-deep-research": { p: "openai", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "o4-mini-deep-research-2025-06-26": { p: "openai", i: 0.000002, o: 0.000008, cr: 5e-7 }, + "vertex_ai/gemini-3.1-flash-lite": { + p: "vertex_ai-language-models", + i: 2.5e-7, + o: 0.0000015, + cr: 2.5e-8, + }, + "vertex_ai/gemini-3.1-flash-lite-preview": { + p: "vertex_ai-language-models", + i: 2.5e-7, + o: 0.0000015, + cr: 2.5e-8, + }, + "vertex_ai/gemini-3.5-flash-lite": { + p: "vertex_ai-language-models", + i: 3e-7, + o: 0.0000025, + cr: 3e-8, + }, + "xai/grok-2": { p: "xai", i: 0.000002, o: 0.00001 }, + "xai/grok-2-1212": { p: "xai", i: 0.000002, o: 0.00001 }, + "xai/grok-2-latest": { p: "xai", i: 0.000002, o: 0.00001 }, + "xai/grok-2-vision": { p: "xai", i: 0.000002, o: 0.00001 }, + "xai/grok-2-vision-1212": { p: "xai", i: 0.000002, o: 0.00001 }, + "xai/grok-2-vision-latest": { p: "xai", i: 0.000002, o: 0.00001 }, + "xai/grok-3": { p: "xai", i: 0.000003, o: 0.000015, cr: 7.5e-7 }, + "xai/grok-3-beta": { p: "xai", i: 0.000003, o: 0.000015, cr: 7.5e-7 }, + "xai/grok-3-fast-beta": { p: "xai", i: 0.000005, o: 0.000025, cr: 0.00000125 }, + "xai/grok-3-fast-latest": { p: "xai", i: 0.000005, o: 0.000025, cr: 0.00000125 }, + "xai/grok-3-latest": { p: "xai", i: 0.000003, o: 0.000015, cr: 7.5e-7 }, + "xai/grok-3-mini": { p: "xai", i: 3e-7, o: 5e-7, cr: 7.5e-8 }, + "xai/grok-3-mini-beta": { p: "xai", i: 3e-7, o: 5e-7, cr: 7.5e-8 }, + "xai/grok-3-mini-fast": { p: "xai", i: 6e-7, o: 0.000004, cr: 1.5e-7 }, + "xai/grok-3-mini-fast-beta": { p: "xai", i: 6e-7, o: 0.000004, cr: 1.5e-7 }, + "xai/grok-3-mini-fast-latest": { p: "xai", i: 6e-7, o: 0.000004, cr: 1.5e-7 }, + "xai/grok-3-mini-latest": { p: "xai", i: 3e-7, o: 5e-7, cr: 7.5e-8 }, + "xai/grok-4": { p: "xai", i: 0.000003, o: 0.000015 }, + "xai/grok-4-0709": { p: "xai", i: 0.000003, o: 0.000015 }, + "xai/grok-4-1-fast": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-1-fast-non-reasoning": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-1-fast-non-reasoning-latest": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-1-fast-reasoning": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-1-fast-reasoning-latest": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-fast-non-reasoning": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-fast-reasoning": { p: "xai", i: 2e-7, o: 5e-7, cr: 5e-8 }, + "xai/grok-4-latest": { p: "xai", i: 0.000003, o: 0.000015 }, + "xai/grok-4.20-0309-reasoning": { p: "xai", i: 0.000002, o: 0.000006, cr: 2e-7 }, + "xai/grok-4.20-beta-0309-non-reasoning": { p: "xai", i: 0.000002, o: 0.000006, cr: 2e-7 }, + "xai/grok-4.20-beta-0309-reasoning": { p: "xai", i: 0.000002, o: 0.000006, cr: 2e-7 }, + "xai/grok-4.20-multi-agent-beta-0309": { p: "xai", i: 0.000002, o: 0.000006, cr: 2e-7 }, + "xai/grok-4.3": { p: "xai", i: 0.00000125, o: 0.0000025, cr: 2e-7 }, + "xai/grok-4.3-latest": { p: "xai", i: 0.00000125, o: 0.0000025, cr: 2e-7 }, + "xai/grok-4.5": { p: "xai", i: 0.000002, o: 0.000006, cr: 5e-7 }, + "xai/grok-4.5-latest": { p: "xai", i: 0.000002, o: 0.000006, cr: 5e-7 }, + "xai/grok-beta": { p: "xai", i: 0.000005, o: 0.000015 }, + "xai/grok-code-fast": { p: "xai", i: 2e-7, o: 0.0000015, cr: 2e-8 }, + "xai/grok-code-fast-1": { p: "xai", i: 2e-7, o: 0.0000015, cr: 2e-8 }, + "xai/grok-code-fast-1-0825": { p: "xai", i: 2e-7, o: 0.0000015, cr: 2e-8 }, + "xai/grok-vision-beta": { p: "xai", i: 0.000005, o: 0.000015 }, +}; diff --git a/core/src/pricing/providerAliases.ts b/core/src/pricing/providerAliases.ts new file mode 100644 index 00000000..cd50dd4f --- /dev/null +++ b/core/src/pricing/providerAliases.ts @@ -0,0 +1,45 @@ +/** + * Maps each opfor provider to the `litellm_provider` values that legitimately + * price it in the upstream price map. + * + * Two names for one thing is the norm, not the exception: opfor's `google` + * appears upstream as both `gemini` (direct API) and `vertex_ai-language-models` + * (the same models via Vertex), and `azure` splits into `azure` / `azure_ai`. + * + * This map is the single source of truth for both sides of the feature: + * `scripts/build-pricing.ts` filters the vendored table by the union of these + * values, and `lookupPrice` uses them to reject a match from the wrong provider. + * Keeping one definition is what stops the table from containing entries the + * lookup can never accept, or vice versa. + * + * `null` means "cannot be verified" — a proxy speaking the OpenAI protocol can + * serve any model from any vendor, so there is no provider to check against. + */ +export const LITELLM_PROVIDER_ALIASES: Record = { + openai: ["openai"], + anthropic: ["anthropic"], + groq: ["groq"], + google: ["gemini", "vertex_ai-language-models"], + deepseek: ["deepseek"], + azure: ["azure", "azure_ai"], + "openai-compatible": null, +}; + +/** + * Upstream providers whose entries are kept in the vendored table. + * + * The union of the aliases above, plus a few vendors that are commonly reached + * *through* an `openai-compatible` proxy and are cheap to include. Deliberately + * excluded: bedrock, fireworks, openrouter and friends — together they are an + * order of magnitude more entries, and a setup routing through them is one where + * the proxy can report exact cost directly, making a list price redundant. + */ +export const VENDORED_LITELLM_PROVIDERS: string[] = [ + ...new Set([ + ...Object.values(LITELLM_PROVIDER_ALIASES) + .filter((v): v is string[] => v !== null) + .flat(), + "mistral", + "xai", + ]), +]; diff --git a/core/src/pricing/types.ts b/core/src/pricing/types.ts new file mode 100644 index 00000000..0287615b --- /dev/null +++ b/core/src/pricing/types.ts @@ -0,0 +1,68 @@ +/** + * Types for run cost estimation. + * + * Prices are USD per single token (the unit the upstream LiteLLM price map uses), + * not per million — keeping the raw unit avoids a scaling step at every call site + * and a class of off-by-1e6 bugs. Format for display at the edge. + */ + +/** Per-token USD prices for one model. */ +export interface ModelPrice { + /** USD per input token. */ + inputPerToken: number; + /** USD per output token. */ + outputPerToken: number; + /** + * USD per cached (repeat) input token, when the provider publishes one. + * Not applied yet — token counting does not separate repeat tokens today. + */ + cacheReadPerToken?: number; + /** USD per cache-write token, when the provider publishes one. Not applied yet. */ + cacheWritePerToken?: number; +} + +/** Where a price came from. Ordered best-first; today only `table` and `unknown` occur. */ +export type CostSource = "table" | "unknown"; + +/** Cost attributed to one model. */ +export interface ModelCost { + /** `":"` — matches the token breakdown's key. */ + key: string; + provider: string; + model: string; + /** Run phases this model served, e.g. `["attacker"]`. */ + roles: string[]; + inputTokens: number; + outputTokens: number; + totalTokens: number; + /** USD, or undefined when the model could not be priced. */ + usd?: number; + source: CostSource; + /** The price-table key this matched, for auditability. Absent when unpriced. */ + matchedKey?: string; +} + +/** Cost of one run (or one evaluator), split by model. */ +export interface RunCost { + /** Sum of every priced model's cost. Excludes unpriced models by definition. */ + totalUsd: number; + currency: "USD"; + byModel: ModelCost[]; + /** + * Models whose price was not found, as `":"`. Non-empty means + * `totalUsd` is a lower bound — surface this rather than implying $0. + */ + unpricedModels: string[]; + /** + * True when every model **that reported tokens** was priced. + * + * Deliberately narrower than "all spend is accounted for": a call site that + * records no usage at all produces no bucket, so there is nothing here to flag. + * A few helper paths (trace curation, session summarisation, `generateJsonObject`) + * still do not report usage, so `complete: true` means "nothing we saw was + * unpriced", not "nothing was missed". Treat `totalUsd` as a floor either way. + */ + complete: boolean; + /** Which price snapshot produced this, for reproducibility. */ + priceTableVersion: string; +} diff --git a/core/src/providers/factory.ts b/core/src/providers/factory.ts index 90589dc8..667fc1b1 100644 --- a/core/src/providers/factory.ts +++ b/core/src/providers/factory.ts @@ -7,6 +7,7 @@ import { createAzure } from "@ai-sdk/azure"; import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; import type { LlmConfig } from "../config/types.js"; import { getEnv } from "../lib/env.js"; +import { rememberModelIdentity } from "./modelIdentity.js"; export interface ProviderCapabilities { supportsJsonMode: boolean; @@ -201,5 +202,9 @@ export function createModel(llm: LlmConfig): LanguageModel { if (adapter.capabilities.requiresBaseURL && !llm.baseURL) { throw new Error(adapter.baseUrlError ?? `baseURL is required for provider '${llm.provider}'`); } - return adapter.build({ apiKey, model: llm.model, baseURL: llm.baseURL }); + const built = adapter.build({ apiKey, model: llm.model, baseURL: llm.baseURL }); + // The built model no longer carries the opfor provider name (openai-compatible + // becomes "custom.chat"), so record it here for token-usage attribution. + rememberModelIdentity(built, llm); + return built; } diff --git a/core/src/providers/modelIdentity.ts b/core/src/providers/modelIdentity.ts new file mode 100644 index 00000000..994b08ec --- /dev/null +++ b/core/src/providers/modelIdentity.ts @@ -0,0 +1,88 @@ +/** + * Model identity registry — maps a built AI SDK model back to the opfor + * provider/model pair it was created from. + * + * Why this exists: {@link createModel} returns an opaque AI SDK object whose own + * `provider` field is SDK-flavored and lossy — `openai-compatible` surfaces as + * `"custom.chat"`, `anthropic` as `"anthropic.messages"`. Token-usage + * attribution (and, later, cost lookup) needs the provider name the *user* + * configured, because that is what the price tables are keyed against. + * + * Registering identity at creation time means every downstream recording site + * can attribute usage from the model object it already holds, instead of + * threading `LlmConfig` through a dozen call signatures. + */ + +import type { LanguageModel } from "ai"; +import type { LlmConfig } from "../config/types.js"; + +/** The opfor-side identity of one model: provider name + model string, as configured. */ +export interface ModelIdentity { + /** opfor provider name, e.g. `"anthropic"` or `"openai-compatible"`. */ + provider: string; + /** Model string exactly as the user configured it, e.g. `"deepseek/deepseek-v4-pro"`. */ + model: string; +} + +/** Bucket key for usage that could not be attributed to any model. */ +export const UNKNOWN_MODEL_KEY = "unknown"; + +/** Stable map/display key for one identity. */ +export function modelKey(identity: ModelIdentity): string { + return `${identity.provider}:${identity.model}`; +} + +// WeakMap so registering a model never keeps it alive. Keyed by the model object +// itself; `createModel` is the single factory for every path (CLI, SDK, MCP +// runner, browser extension), so one registration point covers all of them. +const registry = new WeakMap(); + +/** Record which provider/model a built AI SDK model came from. Called by `createModel`. */ +export function rememberModelIdentity(model: LanguageModel, llm: LlmConfig): void { + if (model && typeof model === "object") { + registry.set(model, { provider: llm.provider, model: llm.model }); + } +} + +/** Anything a call site might have on hand when recording token usage. */ +export type ModelRef = LanguageModel | LlmConfig | ModelIdentity; + +/** Shape test for the `{ provider, model }` pair shared by LlmConfig and ModelIdentity. */ +function isIdentityLike(value: object): value is ModelIdentity { + const v = value as Partial; + return typeof v.provider === "string" && typeof v.model === "string"; +} + +/** Shape test for a built AI SDK language model. */ +function isLanguageModelObject(value: object): value is { modelId: string; provider?: string } { + return typeof (value as { modelId?: unknown }).modelId === "string"; +} + +/** + * Resolve any model reference to an identity. + * + * Prefers the registry (exact configured provider + model). Falls back to the AI + * SDK object's own fields, narrowing `"anthropic.messages"` to `"anthropic"` — + * lossy for `openai-compatible` (reports `"custom"`), which is why the registry + * is consulted first. Returns undefined when nothing identifying is available, + * so callers can bucket the usage as unattributed rather than guess. + */ +export function resolveModelIdentity(ref?: ModelRef): ModelIdentity | undefined { + if (!ref) return undefined; + + // `ai` allows a bare model string; there is no provider to recover from it. + if (typeof ref === "string") return { provider: "unknown", model: ref }; + if (typeof ref !== "object") return undefined; + + const registered = registry.get(ref); + if (registered) return registered; + + if (isLanguageModelObject(ref)) { + const sdkProvider = typeof ref.provider === "string" ? ref.provider.split(".")[0] : "unknown"; + return { provider: sdkProvider || "unknown", model: ref.modelId }; + } + + if (isIdentityLike(ref)) return { provider: ref.provider, model: ref.model }; + + return undefined; +} diff --git a/core/src/report/buildReport.ts b/core/src/report/buildReport.ts index bd5f1e16..74081aae 100644 --- a/core/src/report/buildReport.ts +++ b/core/src/report/buildReport.ts @@ -90,6 +90,8 @@ function toEvaluatorViewModel(ev: EvaluatorResult): EvaluatorViewModel { passRate: ev.passRate, results: ev.attacks.map(toResultViewModel), tokenUsage: ev.tokenUsage, + tokenUsageByModel: ev.tokenUsageByModel, + cost: ev.cost, }; } diff --git a/core/src/report/render.ts b/core/src/report/render.ts index 2227c9db..14d33528 100644 --- a/core/src/report/render.ts +++ b/core/src/report/render.ts @@ -22,6 +22,52 @@ import { roleLabel, SEV_HEX, } from "./format.js"; +import type { RunCost } from "../pricing/types.js"; +import { formatUsd } from "../pricing/estimateCost.js"; + +/** + * Cost for display, carrying its own confidence. + * + * `totalUsd` sums only the models that could be priced, so when none of them + * could it is legitimately `0` — and `formatUsd(0)` is `"$0.00"`, which reads as + * "this run was free" rather than "we don't know". + * + * A complete total is shown plain; that it is a list-price estimate is stated in + * the card's tooltip and the docs rather than decorating every figure. The other + * two cases keep their marker because the number alone would mislead: `≥` says + * the real total is higher, and an unpriced run has no number worth printing. + */ +export function formatCostDisplay(cost: RunCost): string { + if (cost.complete) return formatUsd(cost.totalUsd); + return cost.totalUsd > 0 ? `≥${formatUsd(cost.totalUsd)}` : "unpriced"; +} + +/** + * Sub-label for the cost card: the attacker/judge split when both are present + * (the number people act on — "the judge is most of the spend"), otherwise a + * coverage note. Always states when models went unpriced, so a partial total is + * never mistaken for a complete one. + */ +function costSubLabel(cost: RunCost): string { + if (cost.unpricedModels.length > 0) { + const priced = cost.byModel.length - cost.unpricedModels.length; + return `${priced} of ${cost.byModel.length} models priced — lower bound`; + } + const byRole = new Map(); + for (const m of cost.byModel) { + // One role → label it. Several → "mixed". None → "unknown": a bucket with no + // recorded phase is unattributed, which is not the same as serving several. + const role = m.roles.length === 1 ? m.roles[0] : m.roles.length > 1 ? "mixed" : "unknown"; + byRole.set(role, (byRole.get(role) ?? 0) + (m.usd ?? 0)); + } + if (byRole.size > 1) { + return [...byRole.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([role, usd]) => `${role} ${formatUsd(usd)}`) + .join(" · "); + } + return `${cost.byModel.length} model${cost.byModel.length === 1 ? "" : "s"} · attacker + judge only`; +} // ── Mode-specific labels ───────────────────────────────────────── @@ -131,7 +177,7 @@ export function renderReport(model: ReportViewModel): string {
- ${e.tokenUsage ? `${formatTokenCount(e.tokenUsage.totalTokens)} tokens|` : ""} + ${e.tokenUsage ? `${formatTokenCount(e.tokenUsage.totalTokens)} tokens${e.cost ? ` · ${formatCostDisplay(e.cost)}` : ""}|` : ""} ${evalVerdict === "PASS" ? "Pass" : evalVerdict === "ERROR" ? "Error" : "Fail"} Safety score: ${avgScore ?? "—"}/10
@@ -436,6 +482,15 @@ export function renderReport(model: ReportViewModel): string { ` : "" } + ${ + summary.cost + ? `
+
Testing Cost
+
${formatCostDisplay(summary.cost)}
+
${esc(costSubLabel(summary.cost))}
+
` + : "" + } ${ summary.durationMs !== undefined ? `
diff --git a/core/src/report/types.ts b/core/src/report/types.ts index 468b9a63..f86d1e25 100644 --- a/core/src/report/types.ts +++ b/core/src/report/types.ts @@ -4,6 +4,8 @@ */ import type { JudgeResult } from "../lib/judgeTypes.js"; +import type { ModelTokenUsage } from "../execute/tokenTracker.js"; +import type { RunCost } from "../pricing/types.js"; /** @deprecated Use JudgeResult from @keyvaluesystems/agent-opfor-core/lib/judgeTypes.js directly. */ export type ReportJudge = JudgeResult; @@ -45,6 +47,10 @@ export interface EvaluatorViewModel { passRate: number; results: ResultViewModel[]; tokenUsage?: { inputTokens: number; outputTokens: number; totalTokens: number }; + /** Same tokens as `tokenUsage`, split by the model that spent them. */ + tokenUsageByModel?: ModelTokenUsage[]; + /** Estimated USD cost of this evaluator's attacker + judge calls. */ + cost?: RunCost; } export interface ReportViewModel { @@ -74,6 +80,13 @@ export interface ReportViewModel { safetyScore: number; attackSuccessRate: number; tokenUsage?: { inputTokens: number; outputTokens: number; totalTokens: number }; + /** Same tokens as `tokenUsage`, split by the model that spent them. */ + tokenUsageByModel?: ModelTokenUsage[]; + /** + * Estimated USD cost of the attacker + judge LLM calls. Excludes the + * target's own inference cost, which opfor cannot observe. + */ + cost?: RunCost; /** Wall-clock time for the whole run, from the first evaluator to the last. */ durationMs?: number; }; diff --git a/core/src/run/judge.ts b/core/src/run/judge.ts index 3e33a8bb..7e3f264b 100644 --- a/core/src/run/judge.ts +++ b/core/src/run/judge.ts @@ -194,6 +194,7 @@ export async function judgeToolResponse( // Safe as a predicate because parseJson never throws — it degrades to ERROR. isAcceptableJson: (json) => verdictParser.parseJson(json).verdict !== "ERROR", tokenTracker: args.tokenTracker, + role: "judge", }); return verdictParser.parseJson(raw); diff --git a/core/tests/modelIdentity.test.ts b/core/tests/modelIdentity.test.ts new file mode 100644 index 00000000..49a0b0cf --- /dev/null +++ b/core/tests/modelIdentity.test.ts @@ -0,0 +1,163 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import type { LanguageModel } from "ai"; +import { + modelKey, + rememberModelIdentity, + resolveModelIdentity, + UNKNOWN_MODEL_KEY, +} from "../src/providers/modelIdentity.js"; +import type { LlmConfig } from "../src/config/types.js"; +import { createModel } from "../src/providers/factory.js"; +import { withRetry, roleFromContext } from "../src/lib/llmRetry.js"; +import { TokenTracker } from "../src/execute/tokenTracker.js"; + +/** Stand-in for a built AI SDK model — only the fields the resolver reads. */ +function fakeSdkModel(modelId: string, provider: string): LanguageModel { + return { modelId, provider } as unknown as LanguageModel; +} + +test("modelKey joins provider and model", () => { + assert.equal( + modelKey({ provider: "anthropic", model: "claude-opus-5" }), + "anthropic:claude-opus-5" + ); +}); + +test("UNKNOWN_MODEL_KEY is distinct from any real key", () => { + assert.notEqual(UNKNOWN_MODEL_KEY, modelKey({ provider: "openai", model: "gpt-4o-mini" })); +}); + +test("resolveModelIdentity returns undefined for no reference", () => { + assert.equal(resolveModelIdentity(undefined), undefined); +}); + +test("resolveModelIdentity handles a bare model string with no recoverable provider", () => { + assert.deepStrictEqual(resolveModelIdentity("gpt-4o-mini"), { + provider: "unknown", + model: "gpt-4o-mini", + }); +}); + +test("resolveModelIdentity accepts an LlmConfig directly", () => { + const llm: LlmConfig = { + provider: "openai-compatible", + model: "deepseek/deepseek-v4-pro", + apiKeyEnv: "OPFOR_API_KEY", + baseURL: "https://llm.keyvalue.systems/v1", + }; + assert.deepStrictEqual(resolveModelIdentity(llm), { + provider: "openai-compatible", + model: "deepseek/deepseek-v4-pro", + }); +}); + +test("resolveModelIdentity falls back to SDK fields, narrowing the dotted provider", () => { + const m = fakeSdkModel("claude-opus-5", "anthropic.messages"); + assert.deepStrictEqual(resolveModelIdentity(m), { + provider: "anthropic", + model: "claude-opus-5", + }); +}); + +test("SDK fallback yields the lossy provider that makes the registry necessary", () => { + // An openai-compatible model reports "custom.chat" — the configured provider + // name is unrecoverable from the object alone. This is why createModel registers. + const m = fakeSdkModel("deepseek/deepseek-v4-pro", "custom.chat"); + assert.equal(resolveModelIdentity(m)?.provider, "custom"); +}); + +test("a registered identity wins over the SDK's own fields", () => { + const m = fakeSdkModel("deepseek/deepseek-v4-pro", "custom.chat"); + rememberModelIdentity(m, { + provider: "openai-compatible", + model: "deepseek/deepseek-v4-pro", + apiKeyEnv: "OPFOR_API_KEY", + baseURL: "https://llm.keyvalue.systems/v1", + }); + assert.deepStrictEqual(resolveModelIdentity(m), { + provider: "openai-compatible", + model: "deepseek/deepseek-v4-pro", + }); +}); + +test("rememberModelIdentity is a no-op for a non-object model reference", () => { + const llm: LlmConfig = { provider: "openai", model: "gpt-4o-mini", apiKeyEnv: "OPENAI_API_KEY" }; + // Must not throw — `ai` permits a bare string model. + rememberModelIdentity("gpt-4o-mini" as unknown as LanguageModel, llm); + assert.deepStrictEqual(resolveModelIdentity("gpt-4o-mini"), { + provider: "unknown", + model: "gpt-4o-mini", + }); +}); + +test("resolveModelIdentity returns undefined for an object with nothing identifying", () => { + assert.equal(resolveModelIdentity({} as unknown as LanguageModel), undefined); +}); + +// --------------------------------------------------------------------------- +// Role derivation + end-to-end attribution +// +// roleFromContext reads the free-text label withRetry already takes for logging. +// That coupling is load-bearing but invisible: renaming a log string would +// silently drop the attacker/judge split from every report. These pin it. +// --------------------------------------------------------------------------- + +test("roleFromContext maps the labels actually used at call sites", () => { + // These four strings are the literal `context:` values passed by + // generateAttacks.ts, judge.ts and withRetry's default. Changing one without + // updating roleFromContext silently removes roles from the cost breakdown. + assert.equal(roleFromContext("Attacker"), "attacker"); + assert.equal(roleFromContext("Attacker (MCP)"), "attacker"); + assert.equal(roleFromContext("Judge"), "judge"); + assert.equal(roleFromContext("LLM"), undefined); +}); + +test("roleFromContext is case- and whitespace-insensitive, and safe on empty", () => { + assert.equal(roleFromContext(" jUdGe "), "judge"); + assert.equal(roleFromContext(undefined), undefined); + assert.equal(roleFromContext(" "), undefined); +}); + +test("withRetry attributes usage to the model built by createModel", async () => { + // The load-bearing path: createModel registers identity in a WeakMap, and + // withRetry resolves it from the model object alone. Every unit test above + // uses hand-built identities, so this is the only check that the real + // factory -> tracker chain works. + process.env.MODEL_IDENTITY_TEST_KEY = "dummy-key-not-used"; + const llm: LlmConfig = { + provider: "openai-compatible", + model: "deepseek/deepseek-v4-pro", + apiKeyEnv: "MODEL_IDENTITY_TEST_KEY", + baseURL: "https://example.invalid/v1", + }; + const model = createModel(llm); + + const tracker = new TokenTracker(); + // withRetry auto-records from a successful result's `.usage`; no network needed. + await withRetry(async () => ({ usage: { inputTokens: 120, outputTokens: 30 } }), { + context: "Judge", + tokenTracker: tracker, + model, + }); + + assert.equal(tracker.breakdown.length, 1); + const [bucket] = tracker.breakdown; + // Provider survives as the CONFIGURED name, not the SDK's "custom" — this is + // the whole reason the identity registry exists. + assert.equal(bucket.key, "openai-compatible:deepseek/deepseek-v4-pro"); + assert.equal(bucket.provider, "openai-compatible"); + assert.deepStrictEqual(bucket.roles, ["judge"]); + assert.equal(bucket.totalTokens, 150); +}); + +test("withRetry without a model still counts the tokens, unattributed", async () => { + const tracker = new TokenTracker(); + await withRetry(async () => ({ usage: { inputTokens: 10, outputTokens: 5 } }), { + context: "Attacker", + tokenTracker: tracker, + }); + assert.equal(tracker.totals.totalTokens, 15); + assert.equal(tracker.breakdown[0].key, "unknown"); + assert.deepStrictEqual(tracker.breakdown[0].roles, ["attacker"]); +}); diff --git a/core/tests/pricing.test.ts b/core/tests/pricing.test.ts new file mode 100644 index 00000000..7e710a59 --- /dev/null +++ b/core/tests/pricing.test.ts @@ -0,0 +1,286 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { lookupPrice, priceCandidates } from "../src/pricing/lookupPrice.js"; +import { estimateRunCost, formatUsd } from "../src/pricing/estimateCost.js"; +import { PRICE_TABLE, PRICE_TABLE_VERSION } from "../src/pricing/priceTable.generated.js"; +import { LITELLM_PROVIDER_ALIASES } from "../src/pricing/providerAliases.js"; +import type { ModelTokenUsage } from "../src/execute/tokenTracker.js"; +import type { RunCost } from "../src/pricing/types.js"; +import { formatCostDisplay } from "../src/report/render.js"; + +/** Build a token-breakdown row without repeating the boilerplate. */ +function usage( + provider: string, + model: string, + inputTokens: number, + outputTokens: number, + roles: string[] = ["attacker"] +): ModelTokenUsage { + return { + key: `${provider}:${model}`, + provider, + model, + roles, + calls: 1, + inputTokens, + outputTokens, + totalTokens: inputTokens + outputTokens, + }; +} + +// --------------------------------------------------------------------------- +// Vendored table sanity +// --------------------------------------------------------------------------- + +test("the vendored price table is populated and versioned", () => { + assert.ok(Object.keys(PRICE_TABLE).length > 100); + assert.match(PRICE_TABLE_VERSION, /^litellm-[0-9a-f]{12}$/); +}); + +test("every table row carries a provider the alias map can accept", () => { + const acceptable = new Set( + Object.values(LITELLM_PROVIDER_ALIASES) + .filter((v): v is string[] => v !== null) + .flat() + ); + // Extra vendors are vendored deliberately for proxy setups; the invariant is + // that no row is unreachable *because its provider was never in the aliases* + // for a provider that claims to support it. + const rows = Object.values(PRICE_TABLE); + assert.ok(rows.every((r) => typeof r.p === "string" && r.p.length > 0)); + assert.ok(rows.some((r) => acceptable.has(r.p))); +}); + +test("every table row has a usable input price", () => { + for (const [key, row] of Object.entries(PRICE_TABLE)) { + assert.equal(typeof row.i, "number", `${key} has no numeric input price`); + assert.ok(row.i >= 0, `${key} has a negative input price`); + } +}); + +// --------------------------------------------------------------------------- +// Candidate ladder +// --------------------------------------------------------------------------- + +test("candidates try the provider-scoped form before the bare name", () => { + const c = priceCandidates("azure", "gpt-4o-mini"); + assert.equal(c[0], "azure/gpt-4o-mini"); + assert.ok(c.indexOf("gpt-4o-mini") > 0); +}); + +test("candidates include the prefix-stripped form for vendor-prefixed names", () => { + assert.ok(priceCandidates("anthropic", "anthropic/claude-opus-5").includes("claude-opus-5")); +}); + +test("candidates are deduped and empty models yield none", () => { + const c = priceCandidates("deepseek", "deepseek-chat"); + assert.equal(new Set(c).size, c.length); + assert.deepStrictEqual(priceCandidates("openai", " "), []); +}); + +// --------------------------------------------------------------------------- +// Lookup +// --------------------------------------------------------------------------- + +test("prices a bare direct-API model", () => { + const r = lookupPrice("openai", "gpt-4o-mini"); + assert.ok(r); + assert.ok(r.price.inputPerToken > 0); + assert.equal(r.matchedKey, "gpt-4o-mini"); +}); + +test("prices a vendor-prefixed name by stripping the prefix", () => { + const r = lookupPrice("anthropic", "anthropic/claude-opus-5"); + assert.ok(r); + assert.equal(r.matchedKey, "claude-opus-5"); +}); + +test("prices a provider-scoped row when one exists", () => { + const r = lookupPrice("groq", "llama-3.3-70b-versatile"); + assert.ok(r); + assert.equal(r.matchedKey, "groq/llama-3.3-70b-versatile"); +}); + +test("azure resells at its own rate, not OpenAI's", () => { + const azure = lookupPrice("azure", "gpt-4o-mini"); + const openai = lookupPrice("openai", "gpt-4o-mini"); + assert.ok(azure && openai); + assert.equal(azure.matchedKey, "azure/gpt-4o-mini"); + assert.notEqual(azure.price.inputPerToken, openai.price.inputPerToken); +}); + +test("google accepts both the direct-API and Vertex naming of a model", () => { + const r = lookupPrice("google", "gemini-2.0-flash"); + assert.ok(r, "gemini-2.0-flash should price under the google provider"); + assert.ok(r.price.inputPerToken > 0); +}); + +test("a model from the wrong provider is refused rather than mispriced", () => { + // gpt-4o exists, but not on Groq. Matching it would bill Groq usage at + // OpenAI's rates — a confidently wrong number. + assert.equal(lookupPrice("groq", "gpt-4o"), undefined); +}); + +test("an unknown model is unpriced, not free", () => { + assert.equal(lookupPrice("openai", "totally-made-up-model-v9"), undefined); +}); + +test("openai-compatible skips the provider guard so proxied models still price", () => { + const r = lookupPrice("openai-compatible", "deepseek/deepseek-v4-pro"); + assert.ok(r); + assert.ok(r.price.inputPerToken > 0); +}); + +test("openai-compatible resolves a vendor-prefixed Anthropic model", () => { + const r = lookupPrice("openai-compatible", "anthropic/claude-opus-5"); + assert.ok(r); + assert.equal(r.matchedKey, "claude-opus-5"); +}); + +test("the unattributed bucket does not accidentally match a row", () => { + assert.equal(lookupPrice("unknown", "unknown"), undefined); +}); + +// --------------------------------------------------------------------------- +// Cost estimation +// --------------------------------------------------------------------------- + +test("estimateRunCost returns undefined when nothing was spent", () => { + assert.equal(estimateRunCost(undefined), undefined); + assert.equal(estimateRunCost([]), undefined); + assert.equal(estimateRunCost([usage("openai", "gpt-4o-mini", 0, 0)]), undefined); +}); + +test("cost is tokens times the published per-token rate", () => { + const price = lookupPrice("openai", "gpt-4o-mini"); + assert.ok(price); + const cost = estimateRunCost([usage("openai", "gpt-4o-mini", 1_000_000, 500_000)]); + assert.ok(cost); + const expected = 1_000_000 * price.price.inputPerToken + 500_000 * price.price.outputPerToken; + assert.ok(Math.abs(cost.totalUsd - expected) < 1e-9); + assert.equal(cost.byModel[0].source, "table"); + assert.equal(cost.complete, true); + assert.deepStrictEqual(cost.unpricedModels, []); +}); + +test("attacker and judge are priced separately at their own rates", () => { + const cost = estimateRunCost([ + usage("openai-compatible", "deepseek/deepseek-v4-pro", 80_000, 12_000, ["attacker"]), + usage("anthropic", "claude-opus-5", 20_000, 3_000, ["judge"]), + ]); + assert.ok(cost); + assert.equal(cost.byModel.length, 2); + const judge = cost.byModel.find((m) => m.roles.includes("judge")); + const attacker = cost.byModel.find((m) => m.roles.includes("attacker")); + assert.ok(judge?.usd && attacker?.usd); + // Opus is far pricier per token, so despite ~4x fewer tokens it dominates. + assert.ok(judge.usd > attacker.usd); + assert.ok(Math.abs(cost.totalUsd - (judge.usd + attacker.usd)) < 1e-9); +}); + +test("an unpriced model is reported, not silently counted as free", () => { + const cost = estimateRunCost([ + usage("openai", "gpt-4o-mini", 1000, 100), + usage("openai", "totally-made-up-model-v9", 999_999, 999_999), + ]); + assert.ok(cost); + assert.equal(cost.complete, false); + assert.deepStrictEqual(cost.unpricedModels, ["openai:totally-made-up-model-v9"]); + const unpriced = cost.byModel.find((m) => m.model === "totally-made-up-model-v9"); + assert.equal(unpriced?.usd, undefined); + assert.equal(unpriced?.source, "unknown"); + // The total is a lower bound: it reflects only the model we could price. + const priced = cost.byModel.find((m) => m.model === "gpt-4o-mini"); + assert.equal(cost.totalUsd, priced?.usd); +}); + +test("cost records which price snapshot produced it", () => { + const cost = estimateRunCost([usage("openai", "gpt-4o-mini", 10, 10)]); + assert.equal(cost?.priceTableVersion, PRICE_TABLE_VERSION); +}); + +test("the matched table key is retained for auditability", () => { + const cost = estimateRunCost([usage("groq", "llama-3.3-70b-versatile", 10, 10)]); + assert.equal(cost?.byModel[0].matchedKey, "groq/llama-3.3-70b-versatile"); +}); + +// --------------------------------------------------------------------------- +// Display +// --------------------------------------------------------------------------- + +test("sub-cent runs keep enough precision not to read as free", () => { + assert.equal(formatUsd(0), "$0.00"); + // A real per-model cost from a smoke run. Fixed decimals would show "$0.0000". + assert.equal(formatUsd(0.0000305), "$0.00003"); + assert.equal(formatUsd(0.0000355), "$0.000036"); + assert.equal(formatUsd(0.000001), "$0.000001"); + assert.equal(formatUsd(0.0034), "$0.0034"); + assert.equal(formatUsd(0.0099), "$0.0099"); + assert.equal(formatUsd(12.3456), "$12.35"); +}); + +test("cent-to-dime amounts keep the digit that separates one evaluator from another", () => { + // Real per-evaluator figures from a run. At two decimals these collapse to + // $0.04 / $0.05 / $0.06 and you can no longer see which evaluator is dearest. + assert.equal(formatUsd(0.037), "$0.037"); + assert.equal(formatUsd(0.049), "$0.049"); + assert.equal(formatUsd(0.055), "$0.055"); +}); + +test("a dime and up reads as plain money, not false precision", () => { + assert.equal(formatUsd(0.1), "$0.10"); + assert.equal(formatUsd(0.177), "$0.18"); + assert.equal(formatUsd(0.25), "$0.25"); + assert.equal(formatUsd(0.999), "$1.00"); +}); + +test("amounts too small to show are labelled, never rounded to zero", () => { + assert.equal(formatUsd(0.0000000001), "<$0.000001"); +}); + +test("no displayed amount collapses to a bare zero unless it is truly zero", () => { + for (const v of [1e-9, 1e-7, 1e-6, 3.05e-5, 0.0001, 0.009, 0.5, 3.2]) { + const s = formatUsd(v); + assert.notEqual(s, "$0.00", `${v} rendered as $0.00`); + assert.ok(!/^\$0\.0+$/.test(s), `${v} rendered as all-zero string ${s}`); + } +}); + +// --------------------------------------------------------------------------- +// Cost display confidence +// +// Regression: totalUsd sums only *priced* models, so a run where nothing could +// be priced legitimately totals 0 — and rendering that as "$0.00" makes an +// unknown cost look free, which is the exact failure the design forbids. +// --------------------------------------------------------------------------- + +/** Minimal RunCost for display tests. */ +function runCost(totalUsd: number, unpriced: string[]): RunCost { + return { + totalUsd, + currency: "USD", + byModel: [], + unpricedModels: unpriced, + complete: unpriced.length === 0, + priceTableVersion: PRICE_TABLE_VERSION, + }; +} + +test("a fully-priced run reads as an estimate", () => { + assert.equal(formatCostDisplay(runCost(1.25, [])), "$1.25"); +}); + +test("a partially-priced run reads as a floor, not an estimate", () => { + assert.equal(formatCostDisplay(runCost(1.25, ["openai:mystery"])), "≥$1.25"); +}); + +test("a run where nothing could be priced never renders as $0.00", () => { + const display = formatCostDisplay(runCost(0, ["openai:mystery"])); + assert.equal(display, "unpriced"); + assert.ok(!display.includes("0.00"), "unknown cost must not look free"); +}); + +test("a genuinely free run is still allowed to show zero", () => { + // Everything priced, everything free (e.g. a self-hosted model at $0). + assert.equal(formatCostDisplay(runCost(0, [])), "$0.00"); +}); diff --git a/core/tests/tokenTracker.test.ts b/core/tests/tokenTracker.test.ts index 82957ddb..9ed369a9 100644 --- a/core/tests/tokenTracker.test.ts +++ b/core/tests/tokenTracker.test.ts @@ -126,3 +126,109 @@ test("parseUsage strips unknown provider metadata and returns normalized usage", }); assert.deepStrictEqual(result, { inputTokens: 100, outputTokens: 20, totalTokens: 150 }); }); + +// --------------------------------------------------------------------------- +// Per-model attribution +// --------------------------------------------------------------------------- + +const ATTACKER = { provider: "openai-compatible", model: "deepseek/deepseek-v4-pro" }; +const JUDGE = { provider: "anthropic", model: "claude-opus-5" }; + +test("fresh tracker has an empty breakdown", () => { + const t = new TokenTracker(); + assert.deepStrictEqual(t.breakdown, []); +}); + +test("usage recorded without attribution lands in the unknown bucket, not dropped", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 100, outputTokens: 20 }); + assert.equal(t.breakdown.length, 1); + assert.equal(t.breakdown[0].key, "unknown"); + assert.equal(t.breakdown[0].totalTokens, 120); + assert.deepStrictEqual(t.totals, { inputTokens: 100, outputTokens: 20, totalTokens: 120 }); +}); + +test("attributed usage is keyed by provider and model", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 100, outputTokens: 20 }, { model: ATTACKER, role: "attacker" }); + const [b] = t.breakdown; + assert.equal(b.key, "openai-compatible:deepseek/deepseek-v4-pro"); + assert.equal(b.provider, "openai-compatible"); + assert.equal(b.model, "deepseek/deepseek-v4-pro"); + assert.deepStrictEqual(b.roles, ["attacker"]); + assert.equal(b.calls, 1); +}); + +test("two models are tracked separately and still sum to run totals", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 80_000, outputTokens: 12_000 }, { model: ATTACKER, role: "attacker" }); + t.record({ inputTokens: 20_000, outputTokens: 3_000 }, { model: JUDGE, role: "judge" }); + + assert.equal(t.breakdown.length, 2); + const sum = t.breakdown.reduce((n, b) => n + b.totalTokens, 0); + assert.equal(sum, t.totals.totalTokens); + assert.equal(t.totals.totalTokens, 115_000); +}); + +test("breakdown is ordered heaviest first", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 10, outputTokens: 1 }, { model: JUDGE }); + t.record({ inputTokens: 500, outputTokens: 50 }, { model: ATTACKER }); + assert.equal(t.breakdown[0].model, "deepseek/deepseek-v4-pro"); +}); + +test("one model used for both phases accumulates both roles, deduped and sorted", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 10, outputTokens: 2 }, { model: JUDGE, role: "judge" }); + t.record({ inputTokens: 10, outputTokens: 2 }, { model: JUDGE, role: "attacker" }); + t.record({ inputTokens: 10, outputTokens: 2 }, { model: JUDGE, role: "judge" }); + + assert.equal(t.breakdown.length, 1); + assert.deepStrictEqual(t.breakdown[0].roles, ["attacker", "judge"]); + assert.equal(t.breakdown[0].calls, 3); +}); + +test("roles are normalized to lowercase so 'Judge' and 'judge' do not split", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 10, outputTokens: 2 }, { model: JUDGE, role: "Judge" }); + t.record({ inputTokens: 10, outputTokens: 2 }, { model: JUDGE, role: "judge" }); + assert.deepStrictEqual(t.breakdown[0].roles, ["judge"]); +}); + +test("attributed and unattributed usage coexist without losing tokens", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 100, outputTokens: 20 }, { model: ATTACKER, role: "attacker" }); + t.record({ inputTokens: 7, outputTokens: 3 }); + const sum = t.breakdown.reduce((n, b) => n + b.totalTokens, 0); + assert.equal(sum, t.totals.totalTokens); + assert.ok(t.breakdown.some((b) => b.key === "unknown")); +}); + +test("child tracker propagates attribution to its parent, not just totals", () => { + const parent = new TokenTracker(); + const child = parent.child(); + child.record({ inputTokens: 100, outputTokens: 20 }, { model: JUDGE, role: "judge" }); + + assert.equal(child.breakdown[0].key, "anthropic:claude-opus-5"); + assert.equal(parent.breakdown[0].key, "anthropic:claude-opus-5"); + assert.deepStrictEqual(parent.breakdown[0].roles, ["judge"]); +}); + +test("sibling children merge into one parent bucket per model", () => { + const parent = new TokenTracker(); + const a = parent.child(); + const b = parent.child(); + a.record({ inputTokens: 100, outputTokens: 20 }, { model: ATTACKER, role: "attacker" }); + b.record({ inputTokens: 200, outputTokens: 40 }, { model: ATTACKER, role: "attacker" }); + + assert.equal(parent.breakdown.length, 1); + assert.equal(parent.breakdown[0].calls, 2); + assert.equal(parent.breakdown[0].totalTokens, 360); +}); + +test("explicit totalTokens is preserved in the per-model bucket too", () => { + const t = new TokenTracker(); + t.record({ inputTokens: 100, outputTokens: 20, totalTokens: 150 }, { model: JUDGE }); + assert.equal(t.breakdown[0].totalTokens, 150); + assert.equal(t.totals.totalTokens, 150); +}); diff --git a/docs/browser-extension.md b/docs/browser-extension.md index a7bf35e7..ad4d8578 100644 --- a/docs/browser-extension.md +++ b/docs/browser-extension.md @@ -88,7 +88,7 @@ The extension uses a **single LLM configuration** for all operations (attack gen The extension runs up to **20 turns per evaluator** (default 10). It stops a given evaluator early when the judge returns a definitive verdict. -**Token usage** is tracked per evaluator and shown on the Done screen and in the downloadable HTML report. +**Token usage and testing cost** are tracked per evaluator and shown on the Done screen and in the downloadable HTML report. Cost covers the attacker and judge LLMs you configured in Options. Whatever the target chat spends on its own inference is excluded — opfor drives it through the browser and cannot observe or meter that spend. See [Token usage and testing cost](cli.md#token-usage-and-testing-cost) for how the figure is derived and its caveats. --- diff --git a/docs/cli.md b/docs/cli.md index 60f982ad..cd2e5e0f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -215,21 +215,39 @@ Partial reports include all completed evaluator results and are marked with `sto --- -## Token usage tracking +## Token usage and testing cost -Every LLM call (attacker generation, adaptive follow-ups, judge) is metered. After the run completes, the CLI prints a summary line: +The instrumented LLM calls (attacker generation, adaptive follow-ups, judge, MCP baseline scans) are metered, and each is attributed to the model that made it. After the run completes, the CLI prints: -``` +```text Results: 5 passed, 2 failed, 0 errors Safety score: 71% Token usage: 51,323 input / 6,057 output (57,380 total) +Testing cost: $0.18 + deepseek/deepseek-v4-pro [attacker]: $0.037 + anthropic/claude-opus-5 [judge]: $0.14 ``` -Token usage is also included in the JSON report (`summary.tokenUsage` and per-evaluator `tokenUsage` fields) and in the HTML report's executive summary card. When using `--events`, the `run_finish` event includes token counts in its `summary` payload. +The per-model split is the actionable part — the judge is frequently the larger share, and switching it to a cheaper model is usually the easiest saving. + +Both are included in the JSON report (`summary.tokenUsage`, `summary.tokenUsageByModel`, `summary.cost`, plus the same fields per evaluator) and in the HTML report's executive summary. When using `--events`, the `run_finish` event includes them in its `summary` payload. + +The browser extension shows the same figures on its Done screen. + +### What the cost figure covers + +**"Testing cost" is opfor's own spend — the attacker and judge LLM calls.** It excludes your target's inference cost, which opfor cannot observe from the outside. + +Prices come from a snapshot of LiteLLM's public price map, vendored into the package. Nothing is downloaded at runtime, so runs work offline and a report re-rendered months later produces the same figure. The snapshot version is recorded in the JSON as `summary.cost.priceTableVersion`. + +Maintainers refresh it with `npm run build:pricing` (`-- --check` reports whether it has drifted from upstream). -The browser extension shows a `Tokens` stat on its Done screen. +### Accuracy caveats -> Token counts reflect raw model usage (input + output tokens). No cost estimation is performed — provider pricing varies and changes frequently. +- **Multi-turn runs are over-estimated.** Providers discount repeated context — and multi-turn attacks re-send the whole conversation each turn — but opfor prices every input token at the full rate. The more turns, the more conservative the figure. +- **List prices only.** Negotiated rates, credits, and proxy markup are not reflected. +- **Unknown models are never counted as free.** If a model isn't in the price table, the report says so and marks the total a lower bound (`≥` prefix, or `unpriced` when nothing could be priced). +- **Not every call is instrumented yet.** Trace curation, session summarisation, and `generateJsonObject` record no token usage, so their spend never reaches the total. `summary.cost.complete` reports only that every model opfor _saw_ was priced — it cannot vouch for calls that reported nothing. Treat `totalUsd` as a floor in all cases. --- diff --git a/package.json b/package.json index 882c8801..4f346950 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "format:check": "prettier --check .", "validate:skills": "tsx scripts/validate-skills.ts", "build:catalog": "tsx scripts/build-catalog.ts", + "build:pricing": "tsx scripts/build-pricing.ts", "build:catalog:check": "tsx scripts/build-catalog.ts --check", "test:catalog-generators": "node --import tsx/esm --test scripts/tests/*.test.mjs scripts/tests/*.test.ts runners/extension/tests/*.test.mjs", "smoke:pack": "tsx scripts/smoke-pack.ts", diff --git a/runners/cli/src/commands/run.ts b/runners/cli/src/commands/run.ts index 96edf824..a139c5b5 100644 --- a/runners/cli/src/commands/run.ts +++ b/runners/cli/src/commands/run.ts @@ -7,6 +7,7 @@ import { writeReport } from "@keyvaluesystems/agent-opfor-core/report/buildRepor import type { RunConfig } from "@keyvaluesystems/agent-opfor-core/execute/types.js"; import { parseRunConfig } from "@keyvaluesystems/agent-opfor-core/config/schema.js"; import { normalizeEffort } from "@keyvaluesystems/agent-opfor-core/execute/effortCompat.js"; +import { formatUsd } from "@keyvaluesystems/agent-opfor-core/pricing/estimateCost.js"; import { runSetupAndWrite } from "./setup.js"; import { ensureOpforDirs, OPFOR_DIR, OPFOR_REPORTS_DIR } from "../lib/artifacts.js"; import { ConsoleProgressListener } from "../lib/consoleProgressListener.js"; @@ -260,6 +261,19 @@ export function registerRunCommand(program: Command): void { `Token usage: ${inputTokens.toLocaleString()} input / ${outputTokens.toLocaleString()} output (${totalTokens.toLocaleString()} total)` ); } + if (summary.cost) { + // "Testing cost", not "total cost" — the target's own inference spend is + // invisible to opfor, and an unpriced model makes this a lower bound. + const note = summary.cost.complete + ? "" + : ` (lower bound — unpriced: ${summary.cost.unpricedModels.join(", ")})`; + log.info(`Testing cost: ${formatUsd(summary.cost.totalUsd)}${note}`); + for (const m of summary.cost.byModel) { + const amount = m.usd === undefined ? "not priced" : formatUsd(m.usd); + const roles = m.roles.length ? ` [${m.roles.join(", ")}]` : ""; + log.info(` ${m.model}${roles}: ${amount}`); + } + } log.success(`\nReport: ${html}`); log.info(` JSON: ${json}`); diff --git a/scripts/build-pricing.ts b/scripts/build-pricing.ts new file mode 100644 index 00000000..aee11a8b --- /dev/null +++ b/scripts/build-pricing.ts @@ -0,0 +1,253 @@ +/** + * Generate the vendored model price table. + * + * Downloads LiteLLM's community price map, prunes it to the providers opfor can + * actually reach, and writes core/src/pricing/priceTable.generated.ts. + * + * Why vendored rather than fetched at runtime: opfor runs in air-gapped CI and + * locked-down corporate networks, and a security report should produce the same + * numbers when re-run months later. Pruned, the table is ~60 KB — small enough + * to inline, which also lets the browser extension use it (esbuild bundles the + * generated module; nothing here touches node:fs at runtime). + * + * Usage: + * npm run build:pricing # refresh the table + * npm run build:pricing -- --check # exit 1 if the table differs from upstream + * + * NOTE: --check is intentionally NOT a blocking CI gate. Unlike the evaluator + * catalog (which goes stale only when this repo changes), this table goes stale + * when a third party reprices a model — gating merges on that would let outside + * events break the build. Run it on a schedule and open a PR instead. + */ + +import { createHash } from "node:crypto"; +import { readFile, writeFile, mkdir } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { format, resolveConfig } from "prettier"; +import { z } from "zod"; +import { VENDORED_LITELLM_PROVIDERS } from "../core/src/pricing/providerAliases.js"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, ".."); +const OUT_FILE = path.join(REPO_ROOT, "core/src/pricing/priceTable.generated.ts"); +const CHECK_ONLY = process.argv.includes("--check"); + +const SOURCE_URL = + "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"; + +/** + * Upstream entry shape — only the fields we consume; everything else is ignored. + * + * The map is third-party data fetched over the network, so it is validated + * rather than cast (AGENTS.md: Zod for all external input). Rows are parsed + * individually and a row that fails is skipped and counted, so one malformed + * entry upstream cannot fail the whole build — but a wholesale format change + * shows up as a collapsed row count and trips the empty-table guard below. + */ +const UpstreamEntrySchema = z.object({ + litellm_provider: z.string().min(1).optional(), + mode: z.string().optional(), + input_cost_per_token: z.number().nonnegative().optional(), + output_cost_per_token: z.number().nonnegative().optional(), + cache_read_input_token_cost: z.number().nonnegative().optional(), + cache_creation_input_token_cost: z.number().nonnegative().optional(), +}); + +/** The document itself must be an object of named entries. */ +const UpstreamMapSchema = z.record(z.string(), z.unknown()); + +/** Compact on-disk shape. Short keys because this file is generated, not read. */ +interface CompactPrice { + /** litellm_provider */ + p: string; + /** input cost per token */ + i: number; + /** output cost per token */ + o: number; + /** cache-read cost per token */ + cr?: number; + /** cache-write cost per token */ + cw?: number; +} + +/** Only chat-shaped models can be an attacker or a judge; skip image/audio/embedding rows. */ +const CHAT_MODES = new Set(["chat", "responses"]); + +function prune(raw: Record): { + table: Record; + droppedProviders: Map; + malformed: string[]; +} { + const keep = new Set(VENDORED_LITELLM_PROVIDERS); + const table: Record = {}; + const droppedProviders = new Map(); + const malformed: string[] = []; + + for (const [key, value] of Object.entries(raw)) { + if (!value || typeof value !== "object") continue; + const parsed = UpstreamEntrySchema.safeParse(value); + if (!parsed.success) { + malformed.push(key); + continue; + } + const e = parsed.data; + const provider = e.litellm_provider; + if (!provider) continue; + + if (!keep.has(provider)) { + droppedProviders.set(provider, (droppedProviders.get(provider) ?? 0) + 1); + continue; + } + if (!e.mode || !CHAT_MODES.has(e.mode)) continue; + // A row without an input price cannot produce a cost; drop it rather than + // let it match and silently price at zero. + if (typeof e.input_cost_per_token !== "number") continue; + + const entry: CompactPrice = { + p: provider, + i: e.input_cost_per_token, + o: typeof e.output_cost_per_token === "number" ? e.output_cost_per_token : 0, + }; + if (typeof e.cache_read_input_token_cost === "number") { + entry.cr = e.cache_read_input_token_cost; + } + if (typeof e.cache_creation_input_token_cost === "number") { + entry.cw = e.cache_creation_input_token_cost; + } + table[key] = entry; + } + + return { table, droppedProviders, malformed }; +} + +/** Stable stringify: sorted keys so an unchanged upstream yields an identical file. */ +function serializeTable(table: Record): string { + const keys = Object.keys(table).sort(); + const lines = keys.map((k) => ` ${JSON.stringify(k)}: ${JSON.stringify(table[k])},`); + return lines.join("\n"); +} + +function renderModule(body: string, version: string, entryCount: number): string { + return `/** + * GENERATED FILE — do not edit by hand. + * + * Model prices in USD per token, pruned from LiteLLM's community price map: + * ${SOURCE_URL} + * + * Regenerate with: npm run build:pricing + * + * Entries: ${entryCount} + * + * Contains no Node imports so the browser extension can bundle it. + */ + +/** Compact price row. Short keys keep the generated table small. */ +export interface CompactPrice { + /** Upstream \`litellm_provider\` — used to reject a match from the wrong vendor. */ + p: string; + /** USD per input token. */ + i: number; + /** USD per output token. */ + o: number; + /** USD per cached input token, when published. */ + cr?: number; + /** USD per cache-write token, when published. */ + cw?: number; +} + +/** Identifies this snapshot; recorded in reports so a cost figure is reproducible. */ +export const PRICE_TABLE_VERSION = ${JSON.stringify(version)}; + +export const PRICE_TABLE: Record = { +${body} +}; +`; +} + +async function main(): Promise { + process.stdout.write(`[build-pricing] fetching ${SOURCE_URL}\n`); + const res = await fetch(SOURCE_URL); + if (!res.ok) { + throw new Error( + `Failed to download the price map: HTTP ${res.status}. ` + + `Check network access, or retry later — the vendored table is still usable meanwhile.` + ); + } + const rawText = await res.text(); + const parsedDoc = UpstreamMapSchema.safeParse(JSON.parse(rawText)); + if (!parsedDoc.success) { + throw new Error( + `The downloaded price map is not an object of named entries — upstream may have ` + + `changed format. Refusing to regenerate from it. (${parsedDoc.error.issues[0]?.message})` + ); + } + const raw = parsedDoc.data; + + const { table, droppedProviders, malformed } = prune(raw); + const entryCount = Object.keys(table).length; + if (entryCount === 0) { + throw new Error( + "Pruning produced an empty table — upstream may have changed shape. " + + "Refusing to overwrite the vendored table with nothing." + ); + } + + const body = serializeTable(table); + // Version off the pruned content, not the fetch time: regenerating against an + // unchanged upstream must be a no-op diff. + const version = `litellm-${createHash("sha256").update(body).digest("hex").slice(0, 12)}`; + // Run the rendered module through the repo's Prettier config before comparing + // or writing. Without this the committed file (reformatted by the pre-commit + // hook) never equals this script's raw output, which would make `--check` + // report "stale" forever and turn every regeneration into a full-file diff. + const contents = await format(renderModule(body, version, entryCount), { + ...(await resolveConfig(OUT_FILE)), + parser: "typescript", + }); + + const existing = await readFile(OUT_FILE, "utf8").catch(() => null); + + process.stdout.write( + `[build-pricing] upstream ${Object.keys(raw).length} entries -> kept ${entryCount} ` + + `across ${VENDORED_LITELLM_PROVIDERS.length} providers (${(body.length / 1024).toFixed(1)} KB)\n` + ); + // Say what was left out — a silent prune reads as full coverage when it isn't. + const topDropped = [...droppedProviders.entries()].sort((a, b) => b[1] - a[1]).slice(0, 5); + process.stdout.write( + `[build-pricing] dropped providers (top 5): ${topDropped.map(([p, n]) => `${p}=${n}`).join(", ")}\n` + ); + // Surface schema rejections rather than folding them into the drop count — + // a sudden spike means upstream changed a field's type on us. + if (malformed.length > 0) { + process.stdout.write( + `[build-pricing] ${malformed.length} row(s) failed validation and were skipped: ` + + `${malformed.slice(0, 5).join(", ")}${malformed.length > 5 ? ", …" : ""}\n` + ); + } + + if (CHECK_ONLY) { + if (existing !== contents) { + process.stderr.write( + `[build-pricing] STALE — ${OUT_FILE} differs from upstream. Run: npm run build:pricing\n` + ); + process.exit(1); + } + process.stdout.write(`[build-pricing] up to date (${version})\n`); + return; + } + + if (existing === contents) { + process.stdout.write(`[build-pricing] unchanged (${version})\n`); + return; + } + + await mkdir(path.dirname(OUT_FILE), { recursive: true }); + await writeFile(OUT_FILE, contents, "utf8"); + process.stdout.write(`[build-pricing] wrote ${OUT_FILE} (${version})\n`); +} + +main().catch((err: unknown) => { + process.stderr.write(`[build-pricing] ${err instanceof Error ? err.message : String(err)}\n`); + process.exit(1); +});