feat: preserve prompt cache across effort switches

packages/ai/AGENTS.md

@@ -165,6 +165,10 @@ Native chronological system messages are route/model-specific. Open Responses lo
 
 The wrapped-user fallback preserves ordering while visibly lowering authority. Never silently pass a raw chronological `role: "system"` through a route that might reject it. Do not insert raw retrieved documents, tool output, or web content into privileged chronological system updates; keep untrusted content in ordinary user/tool channels.
 
+### Effort Updates
+
+`Message.effort({ effort, previous })` is a chronological "reasoning effort changed here" marker (`undefined` means the model default). Changing a top-level effort invalidates the whole provider prompt cache, so protocols with a native per-message update (`Protocol.supportsEffortUpdates`) keep the top-level effort at the first marker's `previous` and lower each marker in place: Anthropic Messages emits an empty `role: "system"` message with `output_config.effort` plus the `mid-conversation-output-config-2026-07-01` beta, and OpenAI Responses emits `configuration_update` items. `applyEffortUpdates` runs in `prepareRequest` and strips the markers for every other route, so a protocol without support keeps today's plain top-level behaviour. When the last marker disagrees with the effort the request asks for (reverted or forked history), `resolveEffortUpdates` strips the markers and falls back to a plain top-level change.
+
 ### Tools
 
 Tool loops are represented in common messages and events:

packages/ai/src/cache-policy.ts

@@ -12,6 +12,7 @@
 // count against the four-breakpoint budget; auto only fills remaining slots.
 import { CacheHint, type CachePolicy, type CachePolicyObject } from "./schema/options.js"
 import { LLMRequest, Message, ToolDefinition, type ContentPart, type ToolEntry } from "./schema/messages.js"
+import { effortUpdate } from "./effort-updates.js"
 
 const AUTO: CachePolicyObject = {
   tools: true,
@@ -121,9 +122,15 @@ const markMessages = (
     return markMessageAt(messages, lastIndexOfRole(messages, "user"), hint, budget)
   if (strategy === "latest-assistant")
     return markMessageAt(messages, lastIndexOfRole(messages, "assistant"), hint, budget)
-  const start = Math.max(0, messages.length - strategy.tail)
+  let start = messages.length
+  let remaining = strategy.tail
+  while (remaining > 0 && start > 0) {
+    start -= 1
+    if (effortUpdate(messages[start]!) === undefined) remaining -= 1
+  }
   let next = messages
-  for (let i = start; i < messages.length; i++) next = markMessageAt(next, i, hint, budget)
+  for (let i = start; i < messages.length; i++)
+    if (effortUpdate(messages[i]!) === undefined) next = markMessageAt(next, i, hint, budget)
   return next
 }
 

packages/ai/src/effort-updates.ts

@@ -0,0 +1,27 @@
+// A top-level reasoning effort change invalidates the whole provider prompt cache, so a
+// mid-conversation switch travels as a `Message.effort(...)` marker: protocols with a native
+// per-message update freeze the top-level effort and lower the markers; every other route strips them.
+import { LLMRequest, type EffortPart, type Message } from "./schema/messages.js"
+
+export const effortUpdate = (message: Message): EffortPart | undefined => {
+  if (message.role !== "system" || message.content.length !== 1) return undefined
+  const part = message.content[0]
+  return part.type === "effort" ? part : undefined
+}
+
+export const stripEffortUpdates = (request: LLMRequest) => {
+  const messages = request.messages.filter((message) => effortUpdate(message) === undefined)
+  return messages.length === request.messages.length ? request : LLMRequest.update(request, { messages })
+}
+
+export const applyEffortUpdates = (request: LLMRequest): LLMRequest =>
+  request.model.route.supportsEffortUpdates?.(request) ? request : stripEffortUpdates(request)
+
+// The markers must end at `current`: `revert.ts` never touches `session.model` and forks may select
+// another variant, so on disagreement fall back to a plain top-level change instead of misreporting effort.
+export const resolveEffortUpdates = (request: LLMRequest, current: string | undefined) => {
+  const updates = request.messages.flatMap((message) => effortUpdate(message) ?? [])
+  if (updates.length === 0) return { request, effort: current }
+  if (updates.at(-1)?.effort !== current) return { request: stripEffortUpdates(request), effort: current }
+  return { request, effort: updates[0]?.previous }
+}

packages/ai/src/protocols/anthropic-messages.ts

@@ -27,6 +27,7 @@ import {
 } from "../schema/index.js"
 import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
 import { classifyProviderFailure } from "../provider-error.js"
+import { effortUpdate, resolveEffortUpdates } from "../effort-updates.js"
 import * as Cache from "./utils/cache.js"
 import { Lifecycle } from "./utils/lifecycle.js"
 import { ToolSchemaProjection } from "./utils/tool-schema.js"
@@ -36,6 +37,7 @@ const ADAPTER = "anthropic-messages"
 export const DEFAULT_BASE_URL = "https://api.anthropic.com/v1"
 export const PATH = "/messages"
 export const DEFAULT_MAX_TOKENS = 32_000
+const DEFAULT_EFFORT = "high"
 
 const SSE_EVENTS = new Set([
   "message",
@@ -286,7 +288,11 @@ type AnthropicToolResultBlock = Schema.Schema.Type<typeof AnthropicToolResultBlo
 const AnthropicMessage = Schema.Union([
   Schema.Struct({ role: Schema.Literal("user"), content: Schema.Array(AnthropicUserBlock) }),
   Schema.Struct({ role: Schema.Literal("assistant"), content: Schema.Array(AnthropicAssistantBlock) }),
-  Schema.Struct({ role: Schema.Literal("system"), content: Schema.Array(AnthropicTextBlock) }),
+  Schema.Struct({
+    role: Schema.Literal("system"),
+    content: Schema.Array(AnthropicTextBlock),
+    output_config: Schema.optional(Schema.Struct({ effort: Schema.String })),
+  }),
 ]).pipe(Schema.toTaggedUnion("role"))
 type AnthropicMessage = Schema.Schema.Type<typeof AnthropicMessage>
 
@@ -877,6 +883,12 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* (
 
   for (const [index, message] of request.messages.entries()) {
     if (message.role === "system") {
+      const update = effortUpdate(message)
+      if (update) {
+        // Accepted at any position, so the text-update placement rules do not apply.
+        messages.push({ role: "system", content: [], output_config: { effort: update.effort ?? DEFAULT_EFFORT } })
+        continue
+      }
       if (splitsLocalToolResults(request.messages, index))
         return yield* invalid("Anthropic Messages system updates cannot split a local tool call from its tool result")
       if (supportsNativeSystemUpdates(request) && canUseNativeSystemUpdate(request, index)) {
@@ -1034,18 +1046,11 @@ const resolveOptions = Effect.fn("AnthropicMessages.resolveOptions")(function* (
     ProviderShared.isRecord(rawOutputConfig) && ProviderShared.isRecord(rawOutputConfig.format)
       ? (rawOutputConfig.format as { type: "json_schema"; schema: Record<string, unknown> })
       : undefined
-  const output_config =
-    outputConfigEffort === undefined && outputConfigFormat === undefined
-      ? undefined
-      : {
-          ...(outputConfigEffort === undefined ? {} : { effort: outputConfigEffort }),
-          ...(outputConfigFormat === undefined ? {} : { format: outputConfigFormat }),
-        }
   const thinking = yield* resolveThinking(input?.thinking)
   return {
     thinking: applyThinkingBindingDefault(request.model, thinking),
     effort: outputConfigEffort,
-    output_config,
+    format: outputConfigFormat,
     service_tier,
     metadata,
     container,
@@ -1054,15 +1059,30 @@ const resolveOptions = Effect.fn("AnthropicMessages.resolveOptions")(function* (
   }
 })
 
+// Accept gateway namespaces and Vertex suffixes without treating a snapshot date as a minor version.
+const claudeVersion = (id: string) => {
+  const match = /(?:^|[./])claude-(?<family>[a-z]+)-(?<major>\d+)(?:[.-](?<minor>\d{1,2}))?(?:$|[-:@])/.exec(
+    id.toLowerCase(),
+  )?.groups
+  if (!match) return undefined
+  return { family: match.family, major: Number(match.major), minor: Number(match.minor ?? 0) }
+}
+
 const supportsThinkingBlockBinding = (model: LLMRequest["model"]) => {
   const override = model.compatibility?.supportsThinkingBlockBinding
   if (override !== undefined) return override
-  // Accept gateway namespaces and Vertex suffixes without treating a snapshot date as a minor version.
-  const version = /(?:^|[./])claude-[a-z]+-(?<major>\d+)(?:[.-](?<minor>\d{1,2}))?(?:$|[-:@])/i.exec(model.id)?.groups
-  if (!version) return false
-  const major = Number(version.major)
-  const minor = Number(version.minor ?? 0)
-  return major > 5 || (major === 5 && minor >= 1)
+  const version = claudeVersion(model.id)
+  return version !== undefined && (version.major > 5 || (version.major === 5 && version.minor >= 1))
+}
+
+const supportsEffortUpdates = (model: LLMRequest["model"]) => {
+  const override = model.compatibility?.supportsEffortUpdates
+  if (override !== undefined) return override
+  const version = claudeVersion(model.id)
+  if (version === undefined) return false
+  if (version.family === "opus") return version.major >= 5
+  if (version.family !== "fable" && version.family !== "mythos") return false
+  return version.major > 5 || (version.major === 5 && version.minor >= 1)
 }
 
 const applyThinkingBindingDefault = (model: LLMRequest["model"], thinking: AnthropicThinking | undefined) => {
@@ -1104,13 +1124,15 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
   const management = yield* ProviderShared.validateWith(
     Schema.decodeUnknownEffect(Schema.UndefinedOr(ContextManagement)),
   )(request.providerOptions?.contextManagement)
+  const options = yield* resolveOptions(request)
+  const updates = resolveEffortUpdates(request, options.effort)
   const generation = request.generation
   const toolSchemaCompatibility = request.model.compatibility?.toolSchema
   // Allocate the 4-breakpoint budget in invalidation order: tools → system →
   // messages. Tools live highest in the cache hierarchy, so when callers
   // over-mark we keep their tool hints and shed the message-tail ones first.
   const breakpoints = Cache.newBreakpoints(ANTHROPIC_BREAKPOINT_CAP)
-  const flattened = ProviderShared.flattenToolRequest(request)
+  const flattened = ProviderShared.flattenToolRequest(updates.request)
   const tools =
     flattened.tools.length === 0
       ? undefined
@@ -1138,7 +1160,13 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
       `Anthropic Messages: dropped ${breakpoints.dropped} cache breakpoint(s); the API allows at most ${ANTHROPIC_BREAKPOINT_CAP} per request.`,
     )
   }
-  const options = yield* resolveOptions(request)
+  const output_config =
+    updates.effort === undefined && options.format === undefined
+      ? undefined
+      : {
+          ...(updates.effort === undefined ? {} : { effort: updates.effort }),
+          ...(options.format === undefined ? {} : { format: options.format }),
+        }
   const body = {
     model: request.model.id,
     system,
@@ -1152,7 +1180,7 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
     top_k: generation?.topK,
     stop_sequences: generation?.stop,
     thinking: options.thinking,
-    output_config: options.output_config,
+    output_config,
     // top-level passthrough per SDK MessageCreateParamsBase:4638,4643,4649,4654,4670
     cache_control: options.cache_control,
     container: options.container,
@@ -1677,6 +1705,7 @@ export const protocol = Protocol.make({
     }),
     step,
   },
+  supportsEffortUpdates: (request) => supportsEffortUpdates(request.model),
 })
 
 export const transport = <
@@ -1715,6 +1744,9 @@ function requiredBetaHeaders(body: Pick<AnthropicMessagesBody, "messages" | "con
   )
   if (requestsCompaction || replaysCompaction) betas.push("compact-2026-01-12")
 
+  if (body.messages.some((message) => message.role === "system" && message.output_config !== undefined))
+    betas.push("mid-conversation-output-config-2026-07-01")
+
   const thinking = body.thinking
   if (thinking && thinking.type !== "disabled" && thinking.block_binding)
     betas.push("thinking-binding-controls-2026-08-01")

packages/ai/src/protocols/open-responses.ts

@@ -20,6 +20,7 @@ import {
 } from "../schema/index.js"
 import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
 import { classifyProviderFailure } from "../provider-error.js"
+import { effortUpdate } from "../effort-updates.js"
 import { OpenResponsesOptions } from "./utils/open-responses-options.js"
 import { Lifecycle } from "./utils/lifecycle.js"
 import { ToolSchemaProjection } from "./utils/tool-schema.js"
@@ -164,6 +165,13 @@ export const CompactionItem = Schema.Struct({
   encrypted_content: Schema.String,
 })
 
+// Kept out of the baseline `InputItem` union: only the OpenAI extension accepts it.
+export const ConfigurationUpdate = Schema.Struct({
+  type: Schema.Literal("configuration_update"),
+  reasoning: Schema.Struct({ effort: OpenResponsesOptions.ReasoningEffort }),
+})
+type ConfigurationUpdate = Schema.Schema.Type<typeof ConfigurationUpdate>
+
 export const InputItem = Schema.Union([
   CompactionItem,
   Schema.Struct({ role: Schema.tag("system"), content: Schema.String }),
@@ -208,6 +216,7 @@ export type HostedToolReplayItem = {
 type LoweredInputItem =
   | OpenResponsesInputItem
   | HostedToolReplayItem
+  | ConfigurationUpdate
   | {
       readonly type: "message"
       readonly id?: string
@@ -634,6 +643,8 @@ const lowerToolResultOutput = Effect.fnUntraced(function* (
   return yield* Effect.forEach(content, (item) => lowerToolResultContentItem(item, request, adapter))
 })
 
+const DEFAULT_EFFORT = "medium"
+
 const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (
   request: LLMRequest,
   adapter: ProviderAdapter,
@@ -646,6 +657,14 @@ const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (
       Schema.decodeUnknownEffect(Schema.UndefinedOr(MessageMetadata)),
     )(message.providerMetadata?.[providerMetadataKey])
     if (message.role === "system") {
+      const update = effortUpdate(message)
+      if (update) {
+        // Consecutive updates are rejected, so a newer one replaces its predecessor.
+        const last = input.at(-1)
+        if (last !== undefined && "type" in last && last.type === "configuration_update") input.pop()
+        input.push({ type: "configuration_update", reasoning: { effort: update.effort ?? DEFAULT_EFFORT } })
+        continue
+      }
       input.push({
         role: "developer",
         content: ProviderShared.joinText(yield* ProviderShared.systemUpdateText(adapter.name, message)),
@@ -789,8 +808,7 @@ export const lowerConversation = Effect.fn("OpenResponses.lowerConversation")(fu
   }
 })
 
-export const lowerGeneration = (request: LLMRequest) => {
-  const options = OpenResponsesOptions.resolve(request)
+export const lowerGeneration = (request: LLMRequest, options = OpenResponsesOptions.resolve(request)) => {
   const generation = request.generation
   const cacheKey = ProviderShared.promptCacheKey(request)
   const parallelToolCalls = resolveParallelToolCalls(request)

packages/ai/src/protocols/openai-responses.ts

@@ -6,7 +6,9 @@ import { Endpoint } from "../route/endpoint.js"
 import { Protocol } from "../route/protocol.js"
 import { HttpTransport } from "../route/transport/index.js"
 import { LLMRequest, mergeJsonRecords, type JsonSchema, type ToolDefinition, type ToolEntry } from "../schema/index.js"
+import { resolveEffortUpdates } from "../effort-updates.js"
 import { OpenResponses } from "./open-responses.js"
+import { OpenResponsesOptions } from "./utils/open-responses-options.js"
 import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
 import { OpenAIImage } from "./utils/openai-image.js"
 import { ResponsesHostedTools } from "./utils/responses-hosted-tools.js"
@@ -94,9 +96,15 @@ const OpenAIResponsesToolChoice = Schema.Union([
   Schema.Struct({ type: Schema.tag("image_generation") }),
 ])
 
+const OpenAIResponsesInputItem = Schema.Union([
+  OpenResponses.InputItem,
+  OpenAIResponsesHostedToolItem,
+  OpenResponses.ConfigurationUpdate,
+])
+
 const OpenAIResponsesCoreFields = {
   ...OpenResponses.coreFields,
-  input: Schema.Array(Schema.Union([OpenResponses.InputItem, OpenAIResponsesHostedToolItem])),
+  input: Schema.Array(OpenAIResponsesInputItem),
   tools: optionalArray(OpenAIResponsesTools),
   tool_choice: Schema.optional(OpenAIResponsesToolChoice),
   context_management: Schema.optional(
@@ -119,7 +127,7 @@ export type OpenAIResponsesBody = Schema.Schema.Type<typeof OpenAIResponsesBody>
 export const CompactionTrigger = Schema.Struct({ type: Schema.Literal("compaction_trigger") })
 const CheckpointBody = Schema.Struct({
   ...OpenAIResponsesBody.fields,
-  input: Schema.Array(Schema.Union([OpenResponses.InputItem, OpenAIResponsesHostedToolItem, CompactionTrigger])),
+  input: Schema.Array(Schema.Union([OpenAIResponsesInputItem, CompactionTrigger])),
   store: Schema.Literal(false),
   prompt_cache_retention: optionalNull(Schema.String),
   prompt_cache_options: optionalNull(
@@ -133,6 +141,14 @@ const adapter = {
   restoreHostedToolItem: (item: unknown) => (Schema.is(OpenAIResponsesHostedToolItem)(item) ? item : undefined),
 } satisfies OpenResponses.ProviderAdapter
 
+// Only GPT-6 Astra accepts `configuration_update`, and never alongside automatic `context_management` compaction.
+const supportsEffortUpdates = (request: LLMRequest) => {
+  if (request.providerOptions?.contextManagement !== undefined) return false
+  const override = request.model.compatibility?.supportsEffortUpdates
+  if (override !== undefined) return override
+  return /(?:^|\/)gpt-6-astra$/i.test(request.model.id)
+}
+
 const nativeImageToolInput = (tool: ToolDefinition) => {
   const native = tool.native?.openai
   return ProviderShared.isRecord(native) && native.type === "image_generation" ? native : undefined
@@ -189,10 +205,12 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request:
   const management = yield* ProviderShared.validateWith(
     Schema.decodeUnknownEffect(Schema.UndefinedOr(ContextManagement)),
   )(request.providerOptions?.contextManagement)
+  const options = OpenResponsesOptions.resolve(request)
+  const updates = resolveEffortUpdates(request, options.reasoningEffort)
   const toolSchemaCompatibility = request.model.compatibility?.toolSchema
   return yield* decodeBody({
-    ...(yield* OpenResponses.lowerConversation(request, adapter)),
-    ...OpenResponses.lowerGeneration(request),
+    ...(yield* OpenResponses.lowerConversation(updates.request, adapter)),
+    ...OpenResponses.lowerGeneration(request, { ...options, reasoningEffort: updates.effort }),
     context_management: management?.map((edit) => ({ type: edit.type, compact_threshold: edit.compactThreshold })),
     tools:
       request.tools.length === 0
@@ -295,6 +313,7 @@ export const protocol = Protocol.make({
     step,
     terminal: OpenResponses.terminal,
   },
+  supportsEffortUpdates,
 })
 
 const endpoint = Endpoint.path<OpenAIResponsesBody>(PATH, { baseURL: DEFAULT_BASE_URL })

packages/ai/src/protocols/utils/open-responses-options.ts

@@ -1,12 +1,7 @@
 import { Option, Schema } from "effect"
-import type { LLMRequest } from "../../schema/index.js"
+import { ReasoningEffort, ReasoningEfforts, type LLMRequest } from "../../schema/index.js"
 
-export const ReasoningEfforts = ["none", "minimal", "low", "medium", "high", "xhigh", "max"] as const
-export type ReasoningEffort = (typeof ReasoningEfforts)[number] | (string & {})
-export const ReasoningEffort = Schema.declare<ReasoningEffort>(
-  (value): value is ReasoningEffort => typeof value === "string",
-  { title: "ReasoningEffort" },
-)
+export { ReasoningEffort, ReasoningEfforts }
 
 export const TextVerbosities = ["low", "medium", "high"] as const
 export type TextVerbosity = (typeof TextVerbosities)[number] | (string & {})