From e518ca74edc98396dd28c08fadb729f0630f8d62 Mon Sep 17 00:00:00 2001 From: testikun <320479488+testikun@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:44:58 +0800 Subject: [PATCH] feat(model-info): add per-turn cache diagnostics --- extensions/model-info/cache-diagnostics.ts | 220 ++++++++++++++++++ extensions/model-info/index.ts | 37 ++- .../model-info/cache-diagnostics.test.ts | 152 ++++++++++++ tests/extensions/model-info/index.test.ts | 26 +++ 4 files changed, 434 insertions(+), 1 deletion(-) create mode 100644 extensions/model-info/cache-diagnostics.ts create mode 100644 tests/extensions/model-info/cache-diagnostics.test.ts diff --git a/extensions/model-info/cache-diagnostics.ts b/extensions/model-info/cache-diagnostics.ts new file mode 100644 index 00000000..16648517 --- /dev/null +++ b/extensions/model-info/cache-diagnostics.ts @@ -0,0 +1,220 @@ +import { createHash } from "node:crypto"; +import type { Usage } from "@earendil-works/pi-ai"; + +export const CACHE_DIAGNOSTICS_CHANNEL = "model-info:cache-diagnostics"; + +export const CACHE_WARM_MINIMUM_TOKENS = 2_048; + +export type CacheSemantics = + | "explicit-prefix" + | "implicit-best-effort" + | "unknown"; + +export type CacheObservationKind = + | "first-turn" + | "cold" + | "warm" + | "partial-hit" + | "miss-after-warm-prefix" + | "unknown"; + +export type CacheCorrelation = + | "model-change" + | "thinking-change" + | "tool-surface-change" + | "system-prompt-change" + | "compaction" + | "branch-change"; + +export interface CacheTurnIdentity { + provider: string; + modelId: string; + thinking: string; + toolSurfaceFingerprint: string; + systemPromptFingerprint: string; +} + +export interface CacheTurnObservation { + turnIndex: number; + provider: string; + semantics: CacheSemantics; + kind: CacheObservationKind; + usage: { + input: number; + cacheRead: number; + cacheWrite: number; + promptTokens: number; + }; + previousCacheRead: number | null; + reprocessedTokens: number | null; + correlations: CacheCorrelation[]; + evidence: "observation"; + verifiedCause: null; + explanation: string; +} + +type TurnSample = { + identity: CacheTurnIdentity; + usage: CacheTurnObservation["usage"]; +}; + +const IMPLICIT_CACHE_PROVIDERS = new Set([ + "azure-openai-responses", + "google", + "google-antigravity", + "google-gemini-cli", + "openai", + "openai-codex", + "openai-responses", +]); + +export function cacheSemanticsForProvider(provider: string): CacheSemantics { + const normalized = provider.trim().toLowerCase(); + if (normalized === "anthropic") return "explicit-prefix"; + if (IMPLICIT_CACHE_PROVIDERS.has(normalized)) return "implicit-best-effort"; + return "unknown"; +} + +export function fingerprintCacheSurface(value: unknown) { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +function finiteUsage(value: number) { + return Number.isFinite(value) && value > 0 ? value : 0; +} + +function promptUsage(usage: Usage): CacheTurnObservation["usage"] { + const input = finiteUsage(usage.input); + const cacheRead = finiteUsage(usage.cacheRead); + const cacheWrite = finiteUsage(usage.cacheWrite); + return { + input, + cacheRead, + cacheWrite, + promptTokens: input + cacheRead + cacheWrite, + }; +} + +function identityCorrelations( + previous: CacheTurnIdentity, + current: CacheTurnIdentity, +) { + const correlations: CacheCorrelation[] = []; + if ( + previous.provider !== current.provider || + previous.modelId !== current.modelId + ) { + correlations.push("model-change"); + } + if (previous.thinking !== current.thinking) { + correlations.push("thinking-change"); + } + if (previous.toolSurfaceFingerprint !== current.toolSurfaceFingerprint) { + correlations.push("tool-surface-change"); + } + if (previous.systemPromptFingerprint !== current.systemPromptFingerprint) { + correlations.push("system-prompt-change"); + } + return correlations; +} + +function classify( + semantics: CacheSemantics, + current: CacheTurnObservation["usage"], + previous: TurnSample | undefined, +): Pick { + if (!previous) { + return { + kind: "first-turn", + reprocessedTokens: null, + explanation: + "No prior turn exists, so cache continuity cannot be inferred.", + }; + } + + const previousWarm = previous.usage.cacheRead >= CACHE_WARM_MINIMUM_TOKENS; + if (current.cacheRead > 0) { + return { + kind: + current.cacheRead < previous.usage.cacheRead ? "partial-hit" : "warm", + reprocessedTokens: null, + explanation: + current.cacheRead < previous.usage.cacheRead + ? "The provider reported a smaller cache read than on the prior turn." + : "The provider reported cached prompt tokens on this turn.", + }; + } + if (!previousWarm) { + return { + kind: "cold", + reprocessedTokens: null, + explanation: + "The prior turn had no sufficiently warm prefix, so this cold turn is not an invalidation signal.", + }; + } + if (semantics !== "explicit-prefix") { + return { + kind: "unknown", + reprocessedTokens: null, + explanation: + semantics === "implicit-best-effort" + ? "A warm-to-cold transition was observed, but this provider exposes only best-effort cache usage." + : "A warm-to-cold transition was observed, but the provider cache contract is unknown.", + }; + } + return { + kind: "miss-after-warm-prefix", + reprocessedTokens: current.input, + explanation: + "An explicit-prefix provider reported a warm prior turn and zero cache read now; local boundaries are correlations, not verified causes.", + }; +} + +export function createCacheDiagnosticsTracker() { + let previous: TurnSample | undefined; + let pendingCorrelations = new Set(); + + const reset = () => { + previous = undefined; + pendingCorrelations = new Set(); + }; + + const mark = (correlation: CacheCorrelation) => { + pendingCorrelations.add(correlation); + }; + + const observe = (options: { + turnIndex: number; + identity: CacheTurnIdentity; + usage: Usage; + }) => { + const usage = promptUsage(options.usage); + const semantics = cacheSemanticsForProvider(options.identity.provider); + const classification = classify(semantics, usage, previous); + const correlations = previous + ? identityCorrelations(previous.identity, options.identity) + : []; + for (const pending of pendingCorrelations) correlations.push(pending); + const uniqueCorrelations = [...new Set(correlations)]; + + const observation: CacheTurnObservation = { + turnIndex: options.turnIndex, + provider: options.identity.provider, + semantics, + kind: classification.kind, + usage, + previousCacheRead: previous?.usage.cacheRead ?? null, + reprocessedTokens: classification.reprocessedTokens, + correlations: uniqueCorrelations, + evidence: "observation", + verifiedCause: null, + explanation: classification.explanation, + }; + + previous = { identity: options.identity, usage }; + pendingCorrelations.clear(); + return observation; + }; + + return { mark, observe, reset }; +} diff --git a/extensions/model-info/index.ts b/extensions/model-info/index.ts index eb8e6000..e1d82c4b 100644 --- a/extensions/model-info/index.ts +++ b/extensions/model-info/index.ts @@ -8,6 +8,12 @@ import { REFRESH_CHANNEL, } from "../shared/dashboard-state.ts"; import { createSessionMetricsTracker } from "./session-metrics.ts"; +import { + CACHE_DIAGNOSTICS_CHANNEL, + createCacheDiagnosticsTracker, + fingerprintCacheSurface, + type CacheTurnIdentity, +} from "./cache-diagnostics.ts"; const CHARS_PER_ESTIMATED_TOKEN = 4; const LIVE_UPDATE_INTERVAL_MS = 200; @@ -29,6 +35,8 @@ export default function modelInfo(pi: ExtensionAPI) { let lastLiveUpdate = 0; let currentContext: ExtensionContext | undefined; const sessionMetrics = createSessionMetricsTracker(); + const cacheDiagnostics = createCacheDiagnosticsTracker(); + let cacheIdentity: CacheTurnIdentity | undefined; const publish = () => pi.events.emit(MODEL_INFO_CHANNEL, { ...state }); @@ -74,11 +82,14 @@ export default function modelInfo(pi: ExtensionAPI) { runContentStreamMs = 0; state = { ...state, tokensPerSecond: null, generating: false }; sessionMetrics.reset(); + cacheDiagnostics.reset(); + cacheIdentity = undefined; syncSessionMetrics(ctx); refresh(ctx); }); pi.on("model_select", (event, ctx) => { + cacheDiagnostics.mark("model-change"); state = { ...state, provider: event.model.provider, @@ -91,6 +102,7 @@ export default function modelInfo(pi: ExtensionAPI) { }); pi.on("thinking_level_select", (event) => { + cacheDiagnostics.mark("thinking-change"); state = { ...state, thinking: event.level }; publish(); }); @@ -103,6 +115,17 @@ export default function modelInfo(pi: ExtensionAPI) { refresh(ctx); }); + pi.on("before_agent_start", (event, ctx) => { + const selectedTools = event.systemPromptOptions.selectedTools ?? []; + cacheIdentity = { + provider: ctx.model?.provider ?? "", + modelId: ctx.model?.id ?? "no-model", + thinking: ctx.model?.reasoning ? pi.getThinkingLevel() : "off", + toolSurfaceFingerprint: fingerprintCacheSurface(selectedTools), + systemPromptFingerprint: fingerprintCacheSurface(event.systemPrompt), + }; + }); + pi.on("message_start", (event) => { if (event.message.role === "assistant") resetMessageTracking(); }); @@ -191,20 +214,32 @@ export default function modelInfo(pi: ExtensionAPI) { refresh(ctx); }); - pi.on("turn_end", (_event, ctx) => { + pi.on("turn_end", (event, ctx) => { syncSessionMetrics(ctx); refresh(ctx); + if (event.message?.role === "assistant" && cacheIdentity) { + pi.events.emit( + CACHE_DIAGNOSTICS_CHANNEL, + cacheDiagnostics.observe({ + turnIndex: event.turnIndex, + identity: cacheIdentity, + usage: event.message.usage, + }), + ); + } }); // Compaction and branch moves rewrite history, so the cached percentage is // stale the moment they land. Pi reports unknown occupancy until the next // assistant reply, which is the honest state to show. pi.on("session_compact", (_event, ctx) => { + cacheDiagnostics.mark("compaction"); syncSessionMetrics(ctx); refresh(ctx); }); pi.on("session_tree", (_event, ctx) => { + cacheDiagnostics.mark("branch-change"); syncSessionMetrics(ctx); refresh(ctx); }); diff --git a/tests/extensions/model-info/cache-diagnostics.test.ts b/tests/extensions/model-info/cache-diagnostics.test.ts new file mode 100644 index 00000000..bf48e487 --- /dev/null +++ b/tests/extensions/model-info/cache-diagnostics.test.ts @@ -0,0 +1,152 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import type { Usage } from "@earendil-works/pi-ai"; +import { + cacheSemanticsForProvider, + createCacheDiagnosticsTracker, + fingerprintCacheSurface, + type CacheTurnIdentity, +} from "../../../extensions/model-info/cache-diagnostics.ts"; + +const identity: CacheTurnIdentity = { + provider: "anthropic", + modelId: "claude-test", + thinking: "high", + toolSurfaceFingerprint: fingerprintCacheSurface(["read"]), + systemPromptFingerprint: fingerprintCacheSurface("prompt"), +}; + +function usage(overrides: Partial): Usage { + return { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + total: 0, + }, + ...overrides, + }; +} + +test("classifies provider cache contracts conservatively", () => { + assert.equal(cacheSemanticsForProvider("anthropic"), "explicit-prefix"); + assert.equal(cacheSemanticsForProvider("openai"), "implicit-best-effort"); + assert.equal(cacheSemanticsForProvider("google"), "implicit-best-effort"); + assert.equal(cacheSemanticsForProvider("custom"), "unknown"); +}); + +test("first and consecutive cold turns do not create an invalidation", () => { + const tracker = createCacheDiagnosticsTracker(); + const first = tracker.observe({ + turnIndex: 0, + identity, + usage: usage({ input: 500 }), + }); + const second = tracker.observe({ + turnIndex: 1, + identity, + usage: usage({ input: 700 }), + }); + + assert.equal(first.kind, "first-turn"); + assert.equal(second.kind, "cold"); + assert.equal(second.reprocessedTokens, null); + assert.equal(second.verifiedCause, null); +}); + +test("explicit warm to cold reports reprocessed tokens without inventing a cause", () => { + const tracker = createCacheDiagnosticsTracker(); + tracker.observe({ + turnIndex: 0, + identity, + usage: usage({ cacheRead: 4_096 }), + }); + tracker.mark("compaction"); + const observation = tracker.observe({ + turnIndex: 1, + identity: { + ...identity, + systemPromptFingerprint: fingerprintCacheSurface("changed"), + }, + usage: usage({ input: 4_400 }), + }); + + assert.equal(observation.kind, "miss-after-warm-prefix"); + assert.equal(observation.reprocessedTokens, 4_400); + assert.deepEqual(observation.correlations, [ + "system-prompt-change", + "compaction", + ]); + assert.equal(observation.evidence, "observation"); + assert.equal(observation.verifiedCause, null); +}); + +test("implicit and unknown providers keep warm-to-cold transitions unknown", () => { + for (const provider of ["openai", "custom-provider"]) { + const tracker = createCacheDiagnosticsTracker(); + const current = { ...identity, provider }; + tracker.observe({ + turnIndex: 0, + identity: current, + usage: usage({ cacheRead: 3_000 }), + }); + const observation = tracker.observe({ + turnIndex: 1, + identity: current, + usage: usage({ input: 3_200 }), + }); + assert.equal(observation.kind, "unknown"); + assert.equal(observation.reprocessedTokens, null); + } +}); + +test("partial hits and local identity changes are reported separately", () => { + const tracker = createCacheDiagnosticsTracker(); + tracker.observe({ + turnIndex: 0, + identity, + usage: usage({ cacheRead: 6_000 }), + }); + const observation = tracker.observe({ + turnIndex: 1, + identity: { + ...identity, + modelId: "claude-next", + thinking: "low", + toolSurfaceFingerprint: fingerprintCacheSurface(["read", "bash"]), + }, + usage: usage({ cacheRead: 2_500 }), + }); + + assert.equal(observation.kind, "partial-hit"); + assert.deepEqual(observation.correlations, [ + "model-change", + "thinking-change", + "tool-surface-change", + ]); +}); + +test("reset removes the prior warm baseline and pending correlations", () => { + const tracker = createCacheDiagnosticsTracker(); + tracker.observe({ + turnIndex: 0, + identity, + usage: usage({ cacheRead: 4_000 }), + }); + tracker.mark("branch-change"); + tracker.reset(); + const observation = tracker.observe({ + turnIndex: 0, + identity, + usage: usage({ input: 4_000 }), + }); + + assert.equal(observation.kind, "first-turn"); + assert.deepEqual(observation.correlations, []); +}); diff --git a/tests/extensions/model-info/index.test.ts b/tests/extensions/model-info/index.test.ts index fd51150d..bf4267c1 100644 --- a/tests/extensions/model-info/index.test.ts +++ b/tests/extensions/model-info/index.test.ts @@ -7,6 +7,10 @@ import type { } from "@earendil-works/pi-coding-agent"; import type { Usage } from "@earendil-works/pi-ai"; import modelInfo from "../../../extensions/model-info/index.ts"; +import { + CACHE_DIAGNOSTICS_CHANNEL, + type CacheTurnObservation, +} from "../../../extensions/model-info/cache-diagnostics.ts"; import { MODEL_INFO_CHANNEL, REFRESH_CHANNEL, @@ -131,6 +135,7 @@ class ModelInfoHarness { >(); readonly listeners = new Map void>>(); readonly publications: ModelInfoState[] = []; + readonly cacheObservations: CacheTurnObservation[] = []; readonly manager: InstrumentedSessionManager; contextTokens = 100; private model = { @@ -171,6 +176,9 @@ class ModelInfoHarness { if (channel === MODEL_INFO_CHANNEL) { this.publications.push(value as ModelInfoState); } + if (channel === CACHE_DIAGNOSTICS_CHANNEL) { + this.cacheObservations.push(value as CacheTurnObservation); + } for (const listener of this.listeners.get(channel) ?? []) { listener(value); } @@ -207,6 +215,24 @@ class ModelInfoHarness { } } +test("emits per-turn cache observations without adding them to dashboard state", async () => { + const harness = new ModelInfoHarness([]); + await harness.emit("session_start"); + await harness.emit("before_agent_start", { + systemPrompt: "system", + systemPromptOptions: { cwd: "/repo", selectedTools: ["read"] }, + }); + await harness.emit("turn_end", { + turnIndex: 0, + message: assistant("assistant", null, usage({ input: 120 })).message, + toolResults: [], + }); + + assert.equal(harness.cacheObservations.length, 1); + assert.equal(harness.cacheObservations[0]?.kind, "first-turn"); + assert.equal("cacheDiagnostic" in harness.state, false); +}); + test("synchronizes initial history and waits for turn_end before counting an assistant message", async () => { const initialUsage = usage({ input: 10,