From 1e4ec4dae975936101cd2d413407e05b3dd04f02 Mon Sep 17 00:00:00 2001 From: thomaslprr Date: Mon, 16 Nov 2026 17:25:52 +0100 Subject: [PATCH] feat(models): send the ADK session id to a LiteLLM proxy LiteLlm did not tell a LiteLLM proxy which conversation a call belongs to, so the proxy logged every call of an ADK session under a different session_id and calls could not be grouped per conversation. The LLM flow now sets the session id on LlmRequest.session_id, a field excluded from serialization, and LiteLlm sends it as the x-litellm-session-id header when the model is served through a LiteLLM proxy. A session id header set by the caller is kept, and no header is sent to other providers or when the request has no session id. --- .../adk/flows/llm_flows/core/_model_call.py | 5 + src/google/adk/models/lite_llm.py | 16 +++ src/google/adk/models/llm_request.py | 9 ++ .../flows/llm_flows/test_other_configs.py | 34 +++++ tests/unittests/models/test_fallback_model.py | 1 + tests/unittests/models/test_litellm.py | 135 ++++++++++++++++++ 6 files changed, 200 insertions(+) diff --git a/src/google/adk/flows/llm_flows/core/_model_call.py b/src/google/adk/flows/llm_flows/core/_model_call.py index 896f89bffe4..7b7297ba417 100644 --- a/src/google/adk/flows/llm_flows/core/_model_call.py +++ b/src/google/adk/flows/llm_flows/core/_model_call.py @@ -226,6 +226,11 @@ async def _call_llm_with_tracing() -> AsyncGenerator[LlmResponse, None]: if ADK_AGENT_NAME_LABEL_KEY not in llm_request.config.labels: llm_request.config.labels[ADK_AGENT_NAME_LABEL_KEY] = agent.name + # Tell the model which session the call belongs to, so that it can + # forward it to a backend that groups calls by conversation. + if llm_request.session_id is None: + llm_request.session_id = invocation_context.session.id + # Calls the LLM. llm = await flow._get_llm(invocation_context) diff --git a/src/google/adk/models/lite_llm.py b/src/google/adk/models/lite_llm.py index ad7bc63b81e..654e7d8754c 100644 --- a/src/google/adk/models/lite_llm.py +++ b/src/google/adk/models/lite_llm.py @@ -416,6 +416,7 @@ def _decode_tool_call_arguments( # Routing-only prefix: requests go through a LiteLLM Proxy deployment, but the # payload must still be shaped for the provider named in the next segment. _PROXY_PROVIDER = "litellm_proxy" +_PROXY_SESSION_ID_HEADER = "x-litellm-session-id" _MIME_TYPE_TO_EXTENSION = { "application/pdf": ".pdf", @@ -4822,6 +4823,21 @@ async def generate_content_async( if http_opts.extra_body is not None: completion_args["extra_body"] = http_opts.extra_body + # Lets a LiteLLM Proxy group the calls of an ADK session. Done after the + # headers are merged so that a session ID header set by the caller wins. + if llm_request.session_id and _is_proxied_model( + effective_model, completion_args + ): + extra_headers = completion_args.get("extra_headers") + extra_headers = ( + dict(extra_headers) if isinstance(extra_headers, dict) else {} + ) + if not any( + name.lower() == _PROXY_SESSION_ID_HEADER for name in extra_headers + ): + extra_headers[_PROXY_SESSION_ID_HEADER] = llm_request.session_id + completion_args["extra_headers"] = extra_headers + include_thoughts = ( llm_request.config.thinking_config.include_thoughts if llm_request.config and llm_request.config.thinking_config diff --git a/src/google/adk/models/llm_request.py b/src/google/adk/models/llm_request.py index 3166b41234f..6390715591f 100644 --- a/src/google/adk/models/llm_request.py +++ b/src/google/adk/models/llm_request.py @@ -65,6 +65,7 @@ class LlmRequest(BaseModel): tools_dict: The tools dictionary. cache_config: Context cache configuration for this request. cache_metadata: Cache metadata from previous requests, used for cache management. + session_id: The ID of the ADK session this request belongs to. """ model_config = ConfigDict(arbitrary_types_allowed=True) @@ -113,6 +114,14 @@ class LlmRequest(BaseModel): serving tier. `None` means the run asked for no tier, so the default applies. """ + session_id: str | None = Field(default=None, exclude=True) + """The ID of the ADK session this request belongs to. + + Set by the LLM flow so that a model can forward it to a backend that groups + calls by conversation. Excluded from serialization, and None for a request + built outside of a Runner. + """ + _dynamic_instructions: list[str] = PrivateAttr(default_factory=list) """Dynamic instructions generated by tools that should be resolved in finalized contents/system instructions.""" diff --git a/tests/unittests/flows/llm_flows/test_other_configs.py b/tests/unittests/flows/llm_flows/test_other_configs.py index d4b19f52749..3136c29efea 100644 --- a/tests/unittests/flows/llm_flows/test_other_configs.py +++ b/tests/unittests/flows/llm_flows/test_other_configs.py @@ -45,3 +45,37 @@ class CustomOutput(BaseModel): assert mockModel.requests[0].config.response_schema == CustomOutput assert mockModel.requests[0].config.response_mime_type == 'application/json' assert mockModel.requests[0].config.labels == {'adk_agent_name': 'root_agent'} + + +def test_session_id_is_set_on_llm_request(): + mock_model = testing_utils.MockModel.create(responses=['response1']) + root_agent = Agent(name='root_agent', model=mock_model) + + runner = testing_utils.InMemoryRunner(root_agent) + runner.run('test1') + + assert len(mock_model.requests) == 1 + assert mock_model.requests[0].session_id == runner.session.id + + +def test_session_id_is_the_same_for_a_transferred_sub_agent(): + sub_model = testing_utils.MockModel.create(responses=['sub response']) + sub_agent = Agent(name='sub_agent', model=sub_model) + root_model = testing_utils.MockModel.create( + responses=[ + Part.from_function_call( + name='transfer_to_agent', args={'agent_name': 'sub_agent'} + ) + ] + ) + root_agent = Agent( + name='root_agent', model=root_model, sub_agents=[sub_agent] + ) + + runner = testing_utils.InMemoryRunner(root_agent) + runner.run('test1') + + assert len(root_model.requests) == 1 + assert len(sub_model.requests) == 1 + assert root_model.requests[0].session_id == runner.session.id + assert sub_model.requests[0].session_id == runner.session.id diff --git a/tests/unittests/models/test_fallback_model.py b/tests/unittests/models/test_fallback_model.py index 757fedc33a8..c6154d0af02 100644 --- a/tests/unittests/models/test_fallback_model.py +++ b/tests/unittests/models/test_fallback_model.py @@ -947,6 +947,7 @@ def test_every_public_field_is_accounted_for(): 'cacheable_contents_token_count', # Response-side too. 'previous_interaction_id', # Set by the interactions processor pre-call. 'service_tier', # Set by the interactions processor pre-call too. + 'session_id', # Set by the flow before the call; models only read it. } assert ( diff --git a/tests/unittests/models/test_litellm.py b/tests/unittests/models/test_litellm.py index 18176f313e4..e7e0764dc73 100644 --- a/tests/unittests/models/test_litellm.py +++ b/tests/unittests/models/test_litellm.py @@ -14419,3 +14419,138 @@ def test_strict_json_does_not_log_warning(self, caplog): with caplog.at_level(logging.WARNING, logger="google_adk"): _parse_tool_call_arguments('{"a": 1}', function_name="test_fn") assert "Repaired" not in caplog.text + + +def _session_llm_request( + session_id: str | None = "session-123", + headers: dict[str, str] | None = None, +) -> LlmRequest: + return LlmRequest( + contents=[ + types.Content( + role="user", parts=[types.Part.from_text(text="Test prompt")] + ) + ], + config=types.GenerateContentConfig( + http_options=types.HttpOptions(headers=headers) if headers else None + ), + session_id=session_id, + ) + + +async def _session_completion_kwargs( + mock_acompletion, lite_llm: LiteLlm, llm_request: LlmRequest +) -> dict: + async for _ in lite_llm.generate_content_async(llm_request): + pass + mock_acompletion.assert_called_once() + _, kwargs = mock_acompletion.call_args + return kwargs + + +def test_llm_request_session_id_is_not_serialized(): + llm_request = _session_llm_request() + + assert llm_request.session_id == "session-123" + assert "session_id" not in llm_request.model_dump() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model, extra_kwargs", + [ + ("litellm_proxy/gpt-4o", {}), + ("gpt-4o", {"custom_llm_provider": "litellm_proxy"}), + ], +) +async def test_generate_content_async_sends_session_id_header_to_proxy( + mock_acompletion, mock_client, model, extra_kwargs +): + lite_llm = LiteLlm(model=model, llm_client=mock_client, **extra_kwargs) + + kwargs = await _session_completion_kwargs( + mock_acompletion, lite_llm, _session_llm_request() + ) + + assert kwargs["extra_headers"] == {"x-litellm-session-id": "session-123"} + + +@pytest.mark.asyncio +async def test_generate_content_async_keeps_session_id_from_direct_provider( + mock_acompletion, mock_client +): + lite_llm = LiteLlm(model="openai/gpt-4o", llm_client=mock_client) + + kwargs = await _session_completion_kwargs( + mock_acompletion, lite_llm, _session_llm_request() + ) + + assert "extra_headers" not in kwargs + + +@pytest.mark.asyncio +async def test_generate_content_async_without_session_id_sends_no_header( + mock_acompletion, mock_client +): + lite_llm = LiteLlm(model="litellm_proxy/gpt-4o", llm_client=mock_client) + + kwargs = await _session_completion_kwargs( + mock_acompletion, lite_llm, _session_llm_request(session_id=None) + ) + + assert "extra_headers" not in kwargs + + +@pytest.mark.asyncio +async def test_generate_content_async_keeps_session_id_header_from_constructor( + mock_acompletion, mock_client +): + lite_llm = LiteLlm( + model="litellm_proxy/gpt-4o", + llm_client=mock_client, + extra_headers={"X-LiteLLM-Session-Id": "caller-session"}, + ) + + kwargs = await _session_completion_kwargs( + mock_acompletion, lite_llm, _session_llm_request() + ) + + assert kwargs["extra_headers"] == {"X-LiteLLM-Session-Id": "caller-session"} + + +@pytest.mark.asyncio +async def test_generate_content_async_keeps_session_id_header_from_http_options( + mock_acompletion, mock_client +): + lite_llm = LiteLlm(model="litellm_proxy/gpt-4o", llm_client=mock_client) + + kwargs = await _session_completion_kwargs( + mock_acompletion, + lite_llm, + _session_llm_request(headers={"x-litellm-session-id": "caller-session"}), + ) + + assert kwargs["extra_headers"] == {"x-litellm-session-id": "caller-session"} + + +@pytest.mark.asyncio +async def test_generate_content_async_adds_session_id_to_other_extra_headers( + mock_acompletion, mock_client +): + constructor_headers = {"X-Api-Key": "secret-key"} + lite_llm = LiteLlm( + model="litellm_proxy/gpt-4o", + llm_client=mock_client, + extra_headers=constructor_headers, + ) + + kwargs = await _session_completion_kwargs( + mock_acompletion, lite_llm, _session_llm_request() + ) + + assert kwargs["extra_headers"] == { + "X-Api-Key": "secret-key", + "x-litellm-session-id": "session-123", + } + # The headers given at construction are not changed by a call. + assert constructor_headers == {"X-Api-Key": "secret-key"}