From 32414ff373e063b41e5989f1b02df120d9fde590 Mon Sep 17 00:00:00 2001 From: Ibrahim Khalil Date: Mon, 17 Nov 2025 16:40:37 -0500 Subject: [PATCH 01/11] Update core.py Added the gemini caching parameters to __init__ --- lisette/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lisette/core.py b/lisette/core.py index e1f8d77..7c4d0e9 100644 --- a/lisette/core.py +++ b/lisette/core.py @@ -244,6 +244,7 @@ def __init__( cache=False, # Anthropic prompt caching cache_idxs:list=[-1], # Anthropic cache breakpoint idxs, use `0` for sys prompt if provided ttl=None, # Anthropic prompt caching ttl + cached_content = None # Gemini prompt caching ): "LiteLLM chat client." self.model = model @@ -251,6 +252,7 @@ def __init__( if ns is None and tools: ns = mk_ns(tools) elif ns is None: ns = globals() self.tool_schemas = [lite_mk_func(t) for t in tools] if tools else None + self.cache_name = cached_content store_attr() def _prep_msg(self, msg=None, prefill=None): From d68ad467405d6fdb62e32fdf76d67731bbedba98 Mon Sep 17 00:00:00 2001 From: Ibrahim Khalil Date: Mon, 17 Nov 2025 22:59:26 -0500 Subject: [PATCH 02/11] Add Gemini explicit caching support --- lisette/core.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/lisette/core.py b/lisette/core.py index 7c4d0e9..d2f741c 100644 --- a/lisette/core.py +++ b/lisette/core.py @@ -273,6 +273,7 @@ def _call(self, msg=None, prefill=None, temp=None, think=None, search=None, stre if not get_model_info(self.model).get("supports_assistant_prefill"): prefill=None if _has_search(self.model) and (s:=ifnone(search,self.search)): kwargs['web_search_options'] = {"search_context_size": effort[s]} else: _=kwargs.pop('web_search_options',None) + if self.cache_name: kwargs['cached_content'] = self.cache_name res = completion(model=self.model, messages=self._prep_msg(msg, prefill), stream=stream, tools=self.tool_schemas, reasoning_effort = effort.get(think), tool_choice=tool_choice, # temperature is not supported when reasoning @@ -469,3 +470,77 @@ async def adisplay_stream(rs): md+=o display(Markdown(md),clear=True) return fmt + +def create_cache(self, system_instruction=None, contents=None, tools=None, ttl="3600s"): + from google import genai + client = genai.Client() + + # if model is "gemini/gemini-2.0-flash", extract "gemini-2.0-flash" + if "/" in self.model: + model_name = self.model.split("/")[1] + else: + model_name = self.model + + #check if model has `-001` suffix + if "-001" not in model_name: + model_name += "-001" + + # Check if cache already exists + if self.cache_name: + raise ValueError("Cache already exists. Delete it first with delete_cache()") + + # Use defaults from Chat if not provided + system_instruction = system_instruction or self.sp + tools = tools or self.tools_schema + + # Create cache using google.genai client + cache = client.caches.create( + model=model_name, + config = types.CreateCachedContentConfig( + system_instruction= system_instruction, + if contents: + contents = [contents], + tools = tools, + ttl=ttl + ) + ) + # Store cache.name in self.cache_name + self.cache_name = cache.name + # Return cache object + return cache + + +def delete_cache(self): + from google import genai + + if not self.cache_name: + raise ValueError("No cache exists to delete.") + + client = genai.Client() + client.caches.delete(self.cache_name) + self.cache_name = None + +def get_cache(self): + from google import genai + + if not self.cache_name: + raise ValueError("No cache exists") + + client = genai.Client() + return client.caches.get(name=self.cache_name) + +def update_cache(self,ttl='300s'): + ## ttl needs to be in seconds in string format i.e., '300s' + from google import genai + from google.genai import types + + if not self.cache_name: + raise ValueError("No cache exists to update") + + client = genai.Client() + client.caches.update( + name = self.cache_name, + config = types.UpdateCachedContentConfig( + ttl=ttl + ) +) \ No newline at end of file From 85640f6915cf74cb6b116ddd0aa040988b718329 Mon Sep 17 00:00:00 2001 From: Ibrahim Khalil Date: Mon, 17 Nov 2025 23:14:59 -0500 Subject: [PATCH 03/11] Fix syntax error in create_cache method --- lisette/core.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/lisette/core.py b/lisette/core.py index d2f741c..0811a54 100644 --- a/lisette/core.py +++ b/lisette/core.py @@ -493,13 +493,15 @@ def create_cache(self, system_instruction=None, contents=None, tools=None, ttl=" system_instruction = system_instruction or self.sp tools = tools or self.tools_schema + if contents: + contents = [contents] + # Create cache using google.genai client cache = client.caches.create( model=model_name, config = types.CreateCachedContentConfig( system_instruction= system_instruction, - if contents: - contents = [contents], + contents = [contents], tools = tools, ttl=ttl ) @@ -529,18 +531,3 @@ def get_cache(self): client = genai.Client() return client.caches.get(name=self.cache_name) -def update_cache(self,ttl='300s'): - ## ttl needs to be in seconds in string format i.e., '300s' - from google import genai - from google.genai import types - - if not self.cache_name: - raise ValueError("No cache exists to update") - - client = genai.Client() - client.caches.update( - name = self.cache_name, - config = types.UpdateCachedContentConfig( - ttl=ttl - ) -) \ No newline at end of file From bf0177ce7480578994e982df84ed46951b8c5866 Mon Sep 17 00:00:00 2001 From: Ibrahim Khalil Date: Mon, 17 Nov 2025 23:17:14 -0500 Subject: [PATCH 04/11] fix indent in if statement --- lisette/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lisette/core.py b/lisette/core.py index 0811a54..97ea91d 100644 --- a/lisette/core.py +++ b/lisette/core.py @@ -494,7 +494,7 @@ def create_cache(self, system_instruction=None, contents=None, tools=None, ttl=" tools = tools or self.tools_schema if contents: - contents = [contents] + contents = [contents] # Create cache using google.genai client cache = client.caches.create( From 1df85be582a290265a542c19343c5516fb631359 Mon Sep 17 00:00:00 2001 From: Ibrahim Khalil Date: Mon, 17 Nov 2025 23:21:57 -0500 Subject: [PATCH 05/11] Adding it to Chat class. Had added the create cache and associated functions to core.py --- lisette/core.py | 124 +++++++++++++++++++++++++----------------------- 1 file changed, 65 insertions(+), 59 deletions(-) diff --git a/lisette/core.py b/lisette/core.py index 97ea91d..90c8c82 100644 --- a/lisette/core.py +++ b/lisette/core.py @@ -314,6 +314,71 @@ def __call__(self, elif return_all: return list(result_gen) # toolloop behavior else: return last(result_gen) # normal chat behavior + + def create_cache(self, system_instruction=None, contents=None, tools=None, ttl="3600s"): + from google import genai + client = genai.Client() + + # if model is "gemini/gemini-2.0-flash", extract "gemini-2.0-flash" + if "/" in self.model: + model_name = self.model.split("/")[1] + else: + model_name = self.model + + #check if model has `-001` suffix + if "-001" not in model_name: + model_name += "-001" + + # Check if cache already exists + if self.cache_name: + raise ValueError("Cache already exists. Delete it first with delete_cache()") + + # Use defaults from Chat if not provided + system_instruction = system_instruction or self.sp + tools = tools or self.tools_schema + + if contents: + contents = [contents] + + # Create cache using google.genai client + cache = client.caches.create( + model=model_name, + config = types.CreateCachedContentConfig( + system_instruction= system_instruction, + contents = [contents], + tools = tools, + ttl=ttl + ) + ) + # Store cache.name in self.cache_name + self.cache_name = cache.name + # Return cache object + return cache + + + def delete_cache(self): + from google import genai + + if not self.cache_name: + raise ValueError("No cache exists to delete.") + + client = genai.Client() + client.caches.delete(self.cache_name) + self.cache_name = None + + def get_cache(self): + from google import genai + + if not self.cache_name: + raise ValueError("No cache exists") + + client = genai.Client() + return client.caches.get(name=self.cache_name) + + + + + # %% ../nbs/00_core.ipynb @patch def print_hist(self:Chat): @@ -471,63 +536,4 @@ async def adisplay_stream(rs): display(Markdown(md),clear=True) return fmt -def create_cache(self, system_instruction=None, contents=None, tools=None, ttl="3600s"): - from google import genai - client = genai.Client() - - # if model is "gemini/gemini-2.0-flash", extract "gemini-2.0-flash" - if "/" in self.model: - model_name = self.model.split("/")[1] - else: - model_name = self.model - - #check if model has `-001` suffix - if "-001" not in model_name: - model_name += "-001" - - # Check if cache already exists - if self.cache_name: - raise ValueError("Cache already exists. Delete it first with delete_cache()") - - # Use defaults from Chat if not provided - system_instruction = system_instruction or self.sp - tools = tools or self.tools_schema - - if contents: - contents = [contents] - - # Create cache using google.genai client - cache = client.caches.create( - model=model_name, - config = types.CreateCachedContentConfig( - system_instruction= system_instruction, - contents = [contents], - tools = tools, - ttl=ttl - ) - ) - # Store cache.name in self.cache_name - self.cache_name = cache.name - # Return cache object - return cache - - -def delete_cache(self): - from google import genai - - if not self.cache_name: - raise ValueError("No cache exists to delete.") - - client = genai.Client() - client.caches.delete(self.cache_name) - self.cache_name = None - -def get_cache(self): - from google import genai - - if not self.cache_name: - raise ValueError("No cache exists") - - client = genai.Client() - return client.caches.get(name=self.cache_name) From 55c147bff16441abb50d7c4171839458a1d27fba Mon Sep 17 00:00:00 2001 From: Ibrahim Khalil Date: Mon, 17 Nov 2025 23:59:35 -0500 Subject: [PATCH 06/11] added import types. added update method --- lisette/core.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lisette/core.py b/lisette/core.py index 90c8c82..0f1d2e9 100644 --- a/lisette/core.py +++ b/lisette/core.py @@ -317,6 +317,7 @@ def __call__(self, def create_cache(self, system_instruction=None, contents=None, tools=None, ttl="3600s"): from google import genai + from google.genai import types client = genai.Client() # if model is "gemini/gemini-2.0-flash", extract "gemini-2.0-flash" @@ -375,7 +376,21 @@ def get_cache(self): client = genai.Client() return client.caches.get(name=self.cache_name) + def update_cache(self,ttl='300s'): + ## ttl needs to be in seconds in string format i.e., '300s' + from google import genai + from google.genai import types + + if not self.cache_name: + raise ValueError("No cache exists to update") + client = genai.Client() + client.caches.update( + name = self.cache_name, + config = types.UpdateCachedContentConfig( + ttl=ttl + ) +) From 1d9a7e5d1346bca89092d39ade2dfb7ff2cb4b52 Mon Sep 17 00:00:00 2001 From: Ibrahim Khalil Date: Tue, 18 Nov 2025 00:18:20 -0500 Subject: [PATCH 07/11] added s in schemas --- lisette/core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lisette/core.py b/lisette/core.py index 0f1d2e9..5c4dd9e 100644 --- a/lisette/core.py +++ b/lisette/core.py @@ -313,7 +313,6 @@ def __call__(self, if stream: return result_gen # streaming elif return_all: return list(result_gen) # toolloop behavior else: return last(result_gen) # normal chat behavior - def create_cache(self, system_instruction=None, contents=None, tools=None, ttl="3600s"): from google import genai @@ -336,7 +335,7 @@ def create_cache(self, system_instruction=None, contents=None, tools=None, ttl=" # Use defaults from Chat if not provided system_instruction = system_instruction or self.sp - tools = tools or self.tools_schema + tools = tools or self.tools_schemas if contents: contents = [contents] From d83c8d96231ea9fe9a7e9dc70daee8d0f559f2d1 Mon Sep 17 00:00:00 2001 From: Ibrahim Khalil Date: Tue, 18 Nov 2025 11:43:14 -0500 Subject: [PATCH 08/11] fix tool schema spelling --- lisette/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lisette/core.py b/lisette/core.py index 5c4dd9e..9ea0b2c 100644 --- a/lisette/core.py +++ b/lisette/core.py @@ -335,7 +335,7 @@ def create_cache(self, system_instruction=None, contents=None, tools=None, ttl=" # Use defaults from Chat if not provided system_instruction = system_instruction or self.sp - tools = tools or self.tools_schemas + tools = tools or self.tool_schemas if contents: contents = [contents] From be4ee22d15365c52676fe0e40e62cc24445fab8c Mon Sep 17 00:00:00 2001 From: Ibrahim Khalil Date: Tue, 18 Nov 2025 12:16:53 -0500 Subject: [PATCH 09/11] corrected the NONE contents to be made a list --- lisette/core.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/lisette/core.py b/lisette/core.py index 9ea0b2c..d9753e3 100644 --- a/lisette/core.py +++ b/lisette/core.py @@ -337,19 +337,26 @@ def create_cache(self, system_instruction=None, contents=None, tools=None, ttl=" system_instruction = system_instruction or self.sp tools = tools or self.tool_schemas - if contents: - contents = [contents] - # Create cache using google.genai client - cache = client.caches.create( - model=model_name, - config = types.CreateCachedContentConfig( - system_instruction= system_instruction, - contents = [contents], - tools = tools, - ttl=ttl - ) + if contents: + cache = client.caches.create( + model=model_name, + config=types.CreateCachedContentConfig( + system_instruction=system_instruction, + contents=contents, + tools=tools, + ttl=ttl ) + ) + else: + cache = client.caches.create( + model=model_name, + config=types.CreateCachedContentConfig( + system_instruction=system_instruction, + tools=tools, + ttl=ttl + ) + ) # Store cache.name in self.cache_name self.cache_name = cache.name # Return cache object From 6d408ee1d3836af0d8627be2ac03d02ad0e541d4 Mon Sep 17 00:00:00 2001 From: Ibrahim Khalil Date: Tue, 18 Nov 2025 12:44:46 -0500 Subject: [PATCH 10/11] added system instruction flag --- lisette/core.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lisette/core.py b/lisette/core.py index d9753e3..6cb5507 100644 --- a/lisette/core.py +++ b/lisette/core.py @@ -257,7 +257,8 @@ def __init__( def _prep_msg(self, msg=None, prefill=None): "Prepare the messages list for the API call" - sp = [{"role": "system", "content": self.sp}] if self.sp else [] + # Don't include sp if using cache (it's already in the cache) + sp = [{"role": "system", "content": self.sp}] if self.sp and not getattr(self, '_sp_in_cache', False) else [] if sp: if 0 in self.cache_idxs: sp[0] = _add_cache_control(sp[0]) cache_idxs = L(self.cache_idxs).filter().map(lambda o: o-1 if o>0 else o) @@ -359,6 +360,10 @@ def create_cache(self, system_instruction=None, contents=None, tools=None, ttl=" ) # Store cache.name in self.cache_name self.cache_name = cache.name + + # Set flag if system prompt is in cache + self._sp_in_cache = bool(system_instruction) + # Return cache object return cache From f20042f5c9e2ba67658c7715408cc4c0248d446d Mon Sep 17 00:00:00 2001 From: Ibrahim Khalil Date: Tue, 18 Nov 2025 13:27:40 -0500 Subject: [PATCH 11/11] added name= to self.cache_name --- lisette/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lisette/core.py b/lisette/core.py index 6cb5507..7fc4bee 100644 --- a/lisette/core.py +++ b/lisette/core.py @@ -375,7 +375,7 @@ def delete_cache(self): raise ValueError("No cache exists to delete.") client = genai.Client() - client.caches.delete(self.cache_name) + client.caches.delete(name=self.cache_name) self.cache_name = None def get_cache(self):