Prompt cache debugging

Author: jahooma

packages/agent-runtime/src/constants.ts

@@ -1,3 +1,11 @@
 import { endsAgentStepParam } from '@codebuff/common/tools/constants'
 
 export const globalStopSequence = `${JSON.stringify(endsAgentStepParam)}`
+
+/**
+ * Set to `true` to log the full LLM request (system prompt, tools, messages)
+ * to `debug/cache-debug/` on each user prompt. Use with:
+ *   bun scripts/compare-cache-debug.ts
+ * to diff sequential requests and find what's breaking prompt caching.
+ */
+export const CACHE_DEBUG_FULL_LOGGING = false

packages/agent-runtime/src/run-agent-step.ts

@@ -1,3 +1,5 @@
+import { createHash } from 'crypto'
+
 import { AnalyticsEvent } from '@codebuff/common/constants/analytics-events'
 import { supportsCacheControl } from '@codebuff/common/old-constants'
 import { TOOLS_WHICH_WONT_FORCE_NEXT_STEP } from '@codebuff/common/tools/constants'
@@ -7,6 +9,7 @@ import { systemMessage, userMessage } from '@codebuff/common/util/messages'
 import { APICallError, type ToolSet } from 'ai'
 import { cloneDeep, mapValues } from 'lodash'
 
+import { CACHE_DEBUG_FULL_LOGGING } from './constants'
 import { callTokenCountAPI } from './llm-api/codebuff-web-api'
 import { getMCPToolData } from './mcp'
 import { getAgentStreamFromTemplate } from './prompt-agent-stream'
@@ -18,6 +21,7 @@ import { getAgentPrompt } from './templates/strings'
 import { getToolSet } from './tools/prompts'
 import { processStream } from './tools/stream-parser'
 import { getAgentOutput } from './util/agent-output'
+import { writeCacheDebugSnapshot } from './util/cache-debug'
 import {
   withSystemInstructionTags,
   withSystemTags as withSystemTags,
@@ -461,7 +465,7 @@ export async function loopAgentSteps(
   params: {
     addAgentStep: AddAgentStepFn
     agentState: AgentState
-    agentType: AgentTemplateType
+    agentType: string
     clearUserPromptMessagesAfterResponse?: boolean
     clientSessionId: string
     content?: Array<TextPart | ImagePart>
@@ -711,6 +715,36 @@ export async function loopAgentSteps(
     inputSchema: tool.inputSchema as {},
   }))
 
+  if (CACHE_DEBUG_FULL_LOGGING) {
+    // Debug: hash the system prompt and tool definitions to detect prompt cache invalidation
+    const systemHash = createHash('sha256').update(system).digest('hex').slice(0, 8)
+    const sortedToolDefs = Object.keys(toolDefinitions).sort().reduce((acc, key) => {
+      acc[key] = toolDefinitions[key]
+      return acc
+    }, {} as Record<string, unknown>)
+    const toolsHash = createHash('sha256').update(JSON.stringify(sortedToolDefs)).digest('hex').slice(0, 8)
+    logger.debug(
+      {
+        systemHash,
+        toolsHash,
+        systemLength: system.length,
+        toolCount: Object.keys(toolDefinitions).length,
+        toolNames: Object.keys(toolDefinitions).sort(),
+        agentType,
+      },
+      `[Cache Debug] System prompt hash: ${systemHash}, Tools hash: ${toolsHash}`,
+    )
+
+    writeCacheDebugSnapshot({
+      agentType: String(agentType),
+      system,
+      toolDefinitions: sortedToolDefs,
+      messages: initialMessages,
+      logger,
+      projectRoot: fileContext.projectRoot,
+    })
+  }
+
   const additionalToolDefinitionsWithCache = async () => {
     if (!cachedAdditionalToolDefinitions) {
       cachedAdditionalToolDefinitions = await additionalToolDefinitions({

packages/agent-runtime/src/util/cache-debug.ts

@@ -0,0 +1,50 @@
+import { mkdirSync, writeFileSync } from 'fs'
+import { join } from 'path'
+
+import type { Logger } from '@codebuff/common/types/contracts/logger'
+import type { Message } from '@codebuff/common/types/messages/codebuff-message'
+
+function getCacheDebugDir(projectRoot: string) {
+  return join(projectRoot, 'debug', 'cache-debug')
+}
+let cacheDebugCounter = 0
+
+export function writeCacheDebugSnapshot(params: {
+  agentType: string
+  system: string
+  toolDefinitions: Record<string, unknown>
+  messages: Message[]
+  logger: Logger
+  projectRoot: string
+}) {
+  const { agentType, system, toolDefinitions, messages, logger, projectRoot } = params
+  const cacheDebugDir = getCacheDebugDir(projectRoot)
+  try {
+    mkdirSync(cacheDebugDir, { recursive: true })
+    const index = String(cacheDebugCounter++).padStart(3, '0')
+    const filename = `${index}-${agentType}-${Date.now()}.json`
+    const snapshot = {
+      index: cacheDebugCounter - 1,
+      timestamp: new Date().toISOString(),
+      agentType,
+      systemPrompt: system,
+      toolDefinitions,
+      messages: messages.map((m) => ({
+        role: m.role,
+        content: m.content,
+        tags: 'tags' in m ? m.tags : undefined,
+        timeToLive: 'timeToLive' in m ? m.timeToLive : undefined,
+        sentAt: 'sentAt' in m ? m.sentAt : undefined,
+      })),
+    }
+    writeFileSync(
+      join(cacheDebugDir, filename),
+      JSON.stringify(snapshot, null, 2),
+    )
+    logger.debug(
+      `[Cache Debug] Wrote full snapshot to ${cacheDebugDir}/${filename}`,
+    )
+  } catch (err) {
+    logger.warn({ error: err }, '[Cache Debug] Failed to write snapshot')
+  }
+}