From 6ddd9614a9817d86697b4e2da01468993d6fd752 Mon Sep 17 00:00:00 2001
From: nachiketb <nachiketb@nvidia.com>
Date: Fri, 31 Jul 2026 12:34:48 -0700
Subject: [PATCH] fix(llm-client): enable Anthropic prompt caching by default

Signed-off-by: nachiketb <nachiketb@nvidia.com>
---
 crates/libsy-llm-client/src/client.rs | 46 +++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index e60bdcd5..0860152f 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -172,6 +172,9 @@ impl TranslatingLlmClient {
         // the upstream always sees the target id.
         set_json_model(&mut body, model);
         merge_extra_body(&mut body, backend.extra_body());
+        if matches!(backend, Backend::Anthropic(_)) {
+            enable_anthropic_prompt_caching(&mut body);
+        }
         if matches!(backend, Backend::OpenAiChat(_)) {
             ensure_openai_stream_usage(&mut body);
         }
@@ -679,6 +682,35 @@ fn merge_extra_body(body: &mut Value, extra_body: &BTreeMap<String, Value>) {
     }
 }
 
+// Marks the final message content block as the Anthropic prompt-cache breakpoint.
+fn enable_anthropic_prompt_caching(body: &mut Value) {
+    let Some(content) = body
+        .get_mut("messages")
+        .and_then(Value::as_array_mut)
+        .and_then(|messages| messages.last_mut())
+        .and_then(|message| message.get_mut("content"))
+    else {
+        return;
+    };
+    match content {
+        Value::String(text) => {
+            *content = serde_json::json!([{
+                "type": "text",
+                "text": std::mem::take(text),
+                "cache_control": {"type": "ephemeral"}
+            }]);
+        }
+        Value::Array(blocks) => {
+            if let Some(block) = blocks.last_mut().and_then(Value::as_object_mut) {
+                block
+                    .entry("cache_control".to_string())
+                    .or_insert_with(|| serde_json::json!({"type": "ephemeral"}));
+            }
+        }
+        _ => {}
+    }
+}
+
 // Requests streamed Chat usage by default while preserving an explicit caller choice.
 fn ensure_openai_stream_usage(body: &mut Value) {
     let Value::Object(object) = body else {
@@ -834,6 +866,20 @@ mod tests {
         }
     }
 
+    #[test]
+    fn anthropic_prompt_caching_marks_final_message() {
+        let mut body = json!({
+            "messages": [{"role": "user", "content": "hello"}]
+        });
+
+        enable_anthropic_prompt_caching(&mut body);
+
+        assert_eq!(
+            body["messages"][0]["content"][0]["cache_control"],
+            json!({"type": "ephemeral"})
+        );
+    }
+
     // A request that pins `format` in its metadata, so the client resolves that
     // wire format instead of the model's default backend.
     fn request_with_wire_format(model: &str, format: WireFormat) -> Request {