From d1d5802bc7ed69c0051dc7fecc63954ad0d184be Mon Sep 17 00:00:00 2001 From: David Sanchez <64162682+dsfaccini@users.noreply.github.com> Date: Sun, 30 Nov 2025 19:01:28 -0500 Subject: [PATCH 1/6] fix broken note and add another note about not falling back on validation errors link to fallbackmodel in agnets.md --- docs/agents.md | 2 +- docs/models/overview.md | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/agents.md b/docs/agents.md index 0633fb88ba..59be4a588a 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -708,7 +708,7 @@ print(result_sync.output) The final request uses `temperature=0.0` (run-time), `max_tokens=500` (from model), demonstrating how settings merge with run-time taking precedence. !!! note "Model Settings Support" - Model-level settings are supported by all concrete model implementations (OpenAI, Anthropic, Google, etc.). Wrapper models like `FallbackModel`, `WrapperModel`, and `InstrumentedModel` don't have their own settings - they use the settings of their underlying models. + Model-level settings are supported by all concrete model implementations (OpenAI, Anthropic, Google, etc.). Wrapper models like [`FallbackModel`](models/overview.md#fallback-model), [`WrapperModel`][pydantic_ai.models.wrapper.WrapperModel], and [`InstrumentedModel`][pydantic_ai.models.instrumented.InstrumentedModel] don't have their own settings - they use the settings of their underlying models. ### Model specific settings diff --git a/docs/models/overview.md b/docs/models/overview.md index f1b7eb0781..a568a20161 100644 --- a/docs/models/overview.md +++ b/docs/models/overview.md @@ -87,8 +87,7 @@ in sequence until one successfully returns a result. Under the hood, Pydantic AI from one model to the next if the current model returns a 4xx or 5xx status code. !!! note - - The provider SDKs on which Models are based (like OpenAI, Anthropic, etc.) often have built-in retry logic that can delay the `FallbackModel` from activating. + The provider SDKs on which Models are based (like OpenAI, Anthropic, etc.) often have built-in retry logic that can delay the `FallbackModel` from activating. When using `FallbackModel`, it's recommended to disable provider SDK retries to ensure immediate fallback, for example by setting `max_retries=0` on a [custom OpenAI client](openai.md#custom-openai-client). @@ -173,7 +172,9 @@ In the year 2157, Captain Maya Chen piloted her spacecraft through the vast expa In this example, if the OpenAI model fails, the agent will automatically fall back to the Anthropic model with its own configured settings. The `FallbackModel` itself doesn't have settings - it uses the individual settings of whichever model successfully handles the request. -In this next example, we demonstrate the exception-handling capabilities of `FallbackModel`. +### Exception Handling + +The next example demonstrates the exception-handling capabilities of `FallbackModel`. If all models fail, a [`FallbackExceptionGroup`][pydantic_ai.exceptions.FallbackExceptionGroup] is raised, which contains all the exceptions encountered during the `run` execution. @@ -230,3 +231,6 @@ By default, the `FallbackModel` only moves on to the next model if the current m [`ModelAPIError`][pydantic_ai.exceptions.ModelAPIError], which includes [`ModelHTTPError`][pydantic_ai.exceptions.ModelHTTPError]. You can customize this behavior by passing a custom `fallback_on` argument to the `FallbackModel` constructor. + +!!! note + Validation errors (from [structured output](../output.md#structured-output) or [tool parameters](../tools.md)) do **not** trigger fallback. These errors use the [retry mechanism](../agents.md#reflection-and-self-correction) instead, which re-prompts the same model to try again. This is intentional: validation errors stem from the non-deterministic nature of LLMs and may succeed on retry, whereas API errors (4xx/5xx) generally indicate issues that won't resolve by retrying the same request. From 2c4166eb1d8846d1b235dc2959c2b8b5846491bf Mon Sep 17 00:00:00 2001 From: David Sanchez <64162682+dsfaccini@users.noreply.github.com> Date: Tue, 2 Dec 2025 13:16:13 -0500 Subject: [PATCH 2/6] interim json schema fix for gemini models via openrouter --- .../pydantic_ai/providers/openrouter.py | 69 ++++++++++++++++++- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/providers/openrouter.py b/pydantic_ai_slim/pydantic_ai/providers/openrouter.py index 70f962e047..a73aa5efe2 100644 --- a/pydantic_ai_slim/pydantic_ai/providers/openrouter.py +++ b/pydantic_ai_slim/pydantic_ai/providers/openrouter.py @@ -7,13 +7,14 @@ from openai import AsyncOpenAI from pydantic_ai import ModelProfile +from pydantic_ai._json_schema import JsonSchema, JsonSchemaTransformer from pydantic_ai.exceptions import UserError from pydantic_ai.models import cached_async_http_client from pydantic_ai.profiles.amazon import amazon_model_profile from pydantic_ai.profiles.anthropic import anthropic_model_profile from pydantic_ai.profiles.cohere import cohere_model_profile from pydantic_ai.profiles.deepseek import deepseek_model_profile -from pydantic_ai.profiles.google import google_model_profile +from pydantic_ai.profiles.google import GoogleModelProfile from pydantic_ai.profiles.grok import grok_model_profile from pydantic_ai.profiles.meta import meta_model_profile from pydantic_ai.profiles.mistral import mistral_model_profile @@ -31,6 +32,70 @@ ) from _import_error +class OpenRouterGoogleJsonSchemaTransformer(JsonSchemaTransformer): + """Legacy Google JSON schema transformer for OpenRouter compatibility. + + OpenRouter's compatibility layer doesn't fully support modern JSON Schema features + like $defs/$ref and anyOf for nullable types. This transformer restores v1.19.0 + behavior by inlining definitions and simplifying nullable unions. + + See: https://github.com/pydantic/pydantic-ai/issues/3617 + """ + + def __init__(self, schema: JsonSchema, *, strict: bool | None = None): + super().__init__(schema, strict=strict, prefer_inlined_defs=True, simplify_nullable_unions=True) + + def transform(self, schema: JsonSchema) -> JsonSchema: + # Remove properties not supported by Gemini + schema.pop('$schema', None) + schema.pop('title', None) + schema.pop('discriminator', None) + schema.pop('examples', None) + schema.pop('exclusiveMaximum', None) + schema.pop('exclusiveMinimum', None) + + if (const := schema.pop('const', None)) is not None: + schema['enum'] = [const] + + # Convert enums to string type (legacy Gemini requirement) + if enum := schema.get('enum'): + schema['type'] = 'string' + schema['enum'] = [str(val) for val in enum] + + # Convert oneOf to anyOf for discriminated unions + if 'oneOf' in schema and 'type' not in schema: + schema['anyOf'] = schema.pop('oneOf') + + # Handle string format -> description + type_ = schema.get('type') + if type_ == 'string' and (fmt := schema.pop('format', None)): + description = schema.get('description') + if description: + schema['description'] = f'{description} (format: {fmt})' + else: + schema['description'] = f'Format: {fmt}' + + return schema + + +def openrouter_google_model_profile(model_name: str) -> ModelProfile | None: + """Get the model profile for a Google model accessed via OpenRouter. + + Uses the legacy transformer to maintain compatibility with OpenRouter's + translation layer, which doesn't fully support modern JSON Schema features. + """ + is_image_model = 'image' in model_name + is_3_or_newer = 'gemini-3' in model_name + return GoogleModelProfile( + json_schema_transformer=OpenRouterGoogleJsonSchemaTransformer, + supports_image_output=is_image_model, + supports_json_schema_output=is_3_or_newer or not is_image_model, + supports_json_object_output=is_3_or_newer or not is_image_model, + supports_tools=not is_image_model, + google_supports_native_output_with_builtin_tools=is_3_or_newer, + ) + + class OpenRouterProvider(Provider[AsyncOpenAI]): """Provider for OpenRouter API.""" @@ -48,7 +113,7 @@ def client(self) -> AsyncOpenAI: def model_profile(self, model_name: str) -> ModelProfile | None: provider_to_profile = { - 'google': google_model_profile, + 'google': openrouter_google_model_profile, 'openai': openai_model_profile, 'anthropic': anthropic_model_profile, 'mistralai': mistral_model_profile, From 53ccd39f2b3912cd062409b78832e544a71cc26b Mon Sep 17 00:00:00 2001 From: David Sanchez <64162682+dsfaccini@users.noreply.github.com> Date: Wed, 3 Dec 2025 11:39:56 -0500 Subject: [PATCH 3/6] review adjsutments --- .../pydantic_ai/providers/openrouter.py | 23 ++++++++----------- tests/providers/test_openrouter.py | 11 +++++---- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/providers/openrouter.py b/pydantic_ai_slim/pydantic_ai/providers/openrouter.py index a73aa5efe2..5df71f1143 100644 --- a/pydantic_ai_slim/pydantic_ai/providers/openrouter.py +++ b/pydantic_ai_slim/pydantic_ai/providers/openrouter.py @@ -1,6 +1,7 @@ from __future__ import annotations as _annotations import os +from dataclasses import replace from typing import overload import httpx @@ -14,7 +15,7 @@ from pydantic_ai.profiles.anthropic import anthropic_model_profile from pydantic_ai.profiles.cohere import cohere_model_profile from pydantic_ai.profiles.deepseek import deepseek_model_profile -from pydantic_ai.profiles.google import GoogleModelProfile +from pydantic_ai.profiles.google import google_model_profile from pydantic_ai.profiles.grok import grok_model_profile from pydantic_ai.profiles.meta import meta_model_profile from pydantic_ai.profiles.mistral import mistral_model_profile @@ -32,7 +33,7 @@ ) from _import_error -class OpenRouterGoogleJsonSchemaTransformer(JsonSchemaTransformer): +class _OpenRouterGoogleJsonSchemaTransformer(JsonSchemaTransformer): """Legacy Google JSON schema transformer for OpenRouter compatibility. OpenRouter's compatibility layer doesn't fully support modern JSON Schema features @@ -78,22 +79,16 @@ def transform(self, schema: JsonSchema) -> JsonSchema: return schema -def openrouter_google_model_profile(model_name: str) -> ModelProfile | None: +def _openrouter_google_model_profile(model_name: str) -> ModelProfile | None: """Get the model profile for a Google model accessed via OpenRouter. Uses the legacy transformer to maintain compatibility with OpenRouter's translation layer, which doesn't fully support modern JSON Schema features. """ - is_image_model = 'image' in model_name - is_3_or_newer = 'gemini-3' in model_name - return GoogleModelProfile( - json_schema_transformer=OpenRouterGoogleJsonSchemaTransformer, - supports_image_output=is_image_model, - supports_json_schema_output=is_3_or_newer or not is_image_model, - supports_json_object_output=is_3_or_newer or not is_image_model, - supports_tools=not is_image_model, - google_supports_native_output_with_builtin_tools=is_3_or_newer, - ) + profile = google_model_profile(model_name) + if profile is None: + return None + return replace(profile, json_schema_transformer=_OpenRouterGoogleJsonSchemaTransformer) class OpenRouterProvider(Provider[AsyncOpenAI]): @@ -113,7 +108,7 @@ def client(self) -> AsyncOpenAI: def model_profile(self, model_name: str) -> ModelProfile | None: provider_to_profile = { - 'google': openrouter_google_model_profile, + 'google': _openrouter_google_model_profile, 'openai': openai_model_profile, 'anthropic': anthropic_model_profile, 'mistralai': mistral_model_profile, diff --git a/tests/providers/test_openrouter.py b/tests/providers/test_openrouter.py index ffd54bdab5..c484cbb0c8 100644 --- a/tests/providers/test_openrouter.py +++ b/tests/providers/test_openrouter.py @@ -12,7 +12,7 @@ from pydantic_ai.profiles.anthropic import AnthropicJsonSchemaTransformer, anthropic_model_profile from pydantic_ai.profiles.cohere import cohere_model_profile from pydantic_ai.profiles.deepseek import deepseek_model_profile -from pydantic_ai.profiles.google import GoogleJsonSchemaTransformer, google_model_profile +from pydantic_ai.profiles.google import google_model_profile from pydantic_ai.profiles.grok import grok_model_profile from pydantic_ai.profiles.meta import meta_model_profile from pydantic_ai.profiles.mistral import mistral_model_profile @@ -26,7 +26,10 @@ import openai from pydantic_ai.models.openrouter import OpenRouterModel - from pydantic_ai.providers.openrouter import OpenRouterProvider + from pydantic_ai.providers.openrouter import ( + OpenRouterProvider, + _OpenRouterGoogleJsonSchemaTransformer, # pyright: ignore[reportPrivateUsage] + ) pytestmark = [ @@ -109,12 +112,12 @@ def test_openrouter_provider_model_profile(mocker: MockerFixture): google_profile = provider.model_profile('google/gemini-2.5-pro-preview') google_model_profile_mock.assert_called_with('gemini-2.5-pro-preview') assert google_profile is not None - assert google_profile.json_schema_transformer == GoogleJsonSchemaTransformer + assert google_profile.json_schema_transformer == _OpenRouterGoogleJsonSchemaTransformer google_profile = provider.model_profile('google/gemma-3n-e4b-it:free') google_model_profile_mock.assert_called_with('gemma-3n-e4b-it') assert google_profile is not None - assert google_profile.json_schema_transformer == GoogleJsonSchemaTransformer + assert google_profile.json_schema_transformer == _OpenRouterGoogleJsonSchemaTransformer openai_profile = provider.model_profile('openai/o1-mini') openai_model_profile_mock.assert_called_with('o1-mini') From ae2ee1bd5ad467c81e4d0101da6afebe3b45c76e Mon Sep 17 00:00:00 2001 From: David Sanchez <64162682+dsfaccini@users.noreply.github.com> Date: Wed, 3 Dec 2025 12:03:20 -0500 Subject: [PATCH 4/6] add test for coverage --- .../pydantic_ai/models/openrouter.py | 5 +- .../pydantic_ai/providers/openrouter.py | 2 +- .../test_openrouter_google_nested_schema.yaml | 136 ++++++++++++++++++ tests/models/test_openrouter.py | 69 +++++++++ 4 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 tests/models/cassettes/test_openrouter/test_openrouter_google_nested_schema.yaml diff --git a/pydantic_ai_slim/pydantic_ai/models/openrouter.py b/pydantic_ai_slim/pydantic_ai/models/openrouter.py index 07550b9a03..a219344fe4 100644 --- a/pydantic_ai_slim/pydantic_ai/models/openrouter.py +++ b/pydantic_ai_slim/pydantic_ai/models/openrouter.py @@ -399,7 +399,10 @@ class _OpenRouterChoice(chat_completion.Choice): class _OpenRouterCostDetails: """OpenRouter specific cost details.""" - upstream_inference_cost: int | None = None + upstream_inference_cost: float | None = None + + # TODO rework fields, tests/models/cassettes/test_openrouter/test_openrouter_google_nested_schema.yaml + # shows an `upstream_inference_completions_cost` field as well class _OpenRouterPromptTokenDetails(completion_usage.PromptTokensDetails): diff --git a/pydantic_ai_slim/pydantic_ai/providers/openrouter.py b/pydantic_ai_slim/pydantic_ai/providers/openrouter.py index 5df71f1143..63a73eb05d 100644 --- a/pydantic_ai_slim/pydantic_ai/providers/openrouter.py +++ b/pydantic_ai_slim/pydantic_ai/providers/openrouter.py @@ -86,7 +86,7 @@ def _openrouter_google_model_profile(model_name: str) -> ModelProfile | None: translation layer, which doesn't fully support modern JSON Schema features. """ profile = google_model_profile(model_name) - if profile is None: + if profile is None: # pragma: no cover return None return replace(profile, json_schema_transformer=_OpenRouterGoogleJsonSchemaTransformer) diff --git a/tests/models/cassettes/test_openrouter/test_openrouter_google_nested_schema.yaml b/tests/models/cassettes/test_openrouter/test_openrouter_google_nested_schema.yaml new file mode 100644 index 0000000000..bed734f33a --- /dev/null +++ b/tests/models/cassettes/test_openrouter/test_openrouter_google_nested_schema.yaml @@ -0,0 +1,136 @@ +interactions: +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '914' + content-type: + - application/json + host: + - openrouter.ai + method: POST + parsed_body: + messages: + - content: It's a house with a ground floor that has an entryway, a living room and a garage. + role: user + model: google/gemini-2.5-flash + stream: false + tool_choice: required + tools: + - function: + description: Insert a level with its spaces. + name: insert_level_with_spaces + parameters: + description: Insert a level with its spaces. + properties: + level: + default: null + nullable: true + properties: + level_name: + type: string + level_type: + enum: + - ground + - basement + - floor + - attic + type: string + required: + - level_name + - level_type + type: object + spaces: + items: + properties: + space_name: + type: string + space_type: + enum: + - entryway + - living-room + - kitchen + - bedroom + - bathroom + - garage + type: string + required: + - space_name + - space_type + type: object + type: array + required: + - spaces + type: object + strict: true + type: function + uri: https://openrouter.ai/api/v1/chat/completions + response: + headers: + access-control-allow-origin: + - '*' + connection: + - keep-alive + content-length: + - '1163' + content-type: + - application/json + permissions-policy: + - payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com" "https://js.stripe.com" "https://*.js.stripe.com" + "https://hooks.stripe.com") + referrer-policy: + - no-referrer, strict-origin-when-cross-origin + transfer-encoding: + - chunked + vary: + - Accept-Encoding + parsed_body: + choices: + - finish_reason: tool_calls + index: 0 + logprobs: null + message: + content: '' + reasoning: null + refusal: null + role: assistant + tool_calls: + - function: + arguments: '{"level":{"level_name":"ground floor","level_type":"ground"},"spaces":[{"space_name":"entryway","space_type":"entryway"},{"space_type":"living-room","space_name":"living + room"},{"space_type":"garage","space_name":"garage"}]}' + name: insert_level_with_spaces + id: tool_insert_level_with_spaces_Dp2OGkGjAwsmqUUwHwmj + index: 0 + type: function + native_finish_reason: STOP + created: 1764781178 + id: gen-1764781178-SLI9BlpnJoXSOwuii48G + model: google/gemini-2.5-flash + object: chat.completion + provider: Google AI Studio + usage: + completion_tokens: 89 + completion_tokens_details: + image_tokens: 0 + reasoning_tokens: 0 + cost: 0 + cost_details: + upstream_inference_completions_cost: 0.0002225 + upstream_inference_cost: 0.000296 + upstream_inference_prompt_cost: 7.35e-05 + is_byok: true + prompt_tokens: 245 + prompt_tokens_details: + audio_tokens: 0 + cached_tokens: 0 + video_tokens: 0 + total_tokens: 334 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/models/test_openrouter.py b/tests/models/test_openrouter.py index c52ec6e546..1b8e9c1112 100644 --- a/tests/models/test_openrouter.py +++ b/tests/models/test_openrouter.py @@ -406,3 +406,72 @@ class FindEducationContentFilters(BaseModel): } ] ) + + +async def test_openrouter_google_nested_schema(allow_model_requests: None, openrouter_api_key: str) -> None: + """Test that nested schemas with $defs/$ref work correctly with OpenRouter + Gemini. + + This verifies the fix for https://github.com/pydantic/pydantic-ai/issues/3617 + where OpenRouter's translation layer didn't support modern JSON Schema features. + """ + from enum import Enum + + provider = OpenRouterProvider(api_key=openrouter_api_key) + + class LevelType(str, Enum): + ground = 'ground' + basement = 'basement' + floor = 'floor' + attic = 'attic' + + class SpaceType(str, Enum): + entryway = 'entryway' + living_room = 'living-room' + kitchen = 'kitchen' + bedroom = 'bedroom' + bathroom = 'bathroom' + garage = 'garage' + + class InsertLevelArg(BaseModel): + level_name: str + level_type: LevelType + + class SpaceArg(BaseModel): + space_name: str + space_type: SpaceType + + class InsertLevelWithSpaces(BaseModel): + """Insert a level with its spaces.""" + + level: InsertLevelArg | None = None + spaces: list[SpaceArg] + + model = OpenRouterModel('google/gemini-2.5-flash', provider=provider) + response = await model_request( + model, + [ + ModelRequest.user_text_prompt( + "It's a house with a ground floor that has an entryway, a living room and a garage." + ) + ], + model_request_parameters=ModelRequestParameters( + function_tools=[ + ToolDefinition( + name='insert_level_with_spaces', + description=InsertLevelWithSpaces.__doc__ or '', + parameters_json_schema=InsertLevelWithSpaces.model_json_schema(), + ) + ], + allow_text_output=False, + ), + ) + + assert response.parts == snapshot( + [ + ToolCallPart( + tool_name='insert_level_with_spaces', + args='{"level":{"level_name":"ground floor","level_type":"ground"},"spaces":[{"space_name":"entryway","space_type":"entryway"},{"space_type":"living-room","space_name":"living room"},{"space_type":"garage","space_name":"garage"}]}', + tool_call_id='tool_insert_level_with_spaces_Dp2OGkGjAwsmqUUwHwmj', + ) + ] + ) From ce16c0b9515ce8a201922decc049ff06861fd68c Mon Sep 17 00:00:00 2001 From: David Sanchez <64162682+dsfaccini@users.noreply.github.com> Date: Wed, 3 Dec 2025 12:18:43 -0500 Subject: [PATCH 5/6] more coverage --- tests/providers/test_openrouter.py | 37 ++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/providers/test_openrouter.py b/tests/providers/test_openrouter.py index c484cbb0c8..8aa30327bb 100644 --- a/tests/providers/test_openrouter.py +++ b/tests/providers/test_openrouter.py @@ -173,3 +173,40 @@ def test_openrouter_provider_model_profile(mocker: MockerFixture): unknown_profile = provider.model_profile('unknown/model') assert unknown_profile is not None assert unknown_profile.json_schema_transformer == OpenAIJsonSchemaTransformer + + +def test_openrouter_google_json_schema_transformer(): + """Test _OpenRouterGoogleJsonSchemaTransformer covers all transformation cases.""" + schema = { + '$schema': 'http://json-schema.org/draft-07/schema#', + 'title': 'TestSchema', + 'type': 'object', + 'properties': { + 'status': {'const': 'active'}, + 'category': {'oneOf': [{'type': 'string'}, {'type': 'integer'}]}, + 'email': {'type': 'string', 'format': 'email', 'description': 'User email'}, + 'date': {'type': 'string', 'format': 'date'}, + }, + } + + transformer = _OpenRouterGoogleJsonSchemaTransformer(schema) + result = transformer.walk() + + # const -> enum conversion (line 59) + assert result['properties']['status'] == {'enum': ['active'], 'type': 'string'} + + # oneOf -> anyOf conversion (line 68) + assert 'anyOf' in result['properties']['category'] + assert 'oneOf' not in result['properties']['category'] + + # format -> description with existing description (lines 73-75) + assert result['properties']['email']['description'] == 'User email (format: email)' + assert 'format' not in result['properties']['email'] + + # format -> description without existing description (lines 76-77) + assert result['properties']['date']['description'] == 'Format: date' + assert 'format' not in result['properties']['date'] + + # Removed fields + assert '$schema' not in result + assert 'title' not in result From 735386a10bcb1e37e136cfa346619c4a515a605d Mon Sep 17 00:00:00 2001 From: David Sanchez <64162682+dsfaccini@users.noreply.github.com> Date: Wed, 3 Dec 2025 14:59:07 -0500 Subject: [PATCH 6/6] remove lines and add tool test --- .../test_openrouter_google_nested_schema.yaml | 219 ++++++++++++++++-- tests/models/test_openrouter.py | 45 ++-- tests/providers/test_openrouter.py | 8 +- 3 files changed, 228 insertions(+), 44 deletions(-) diff --git a/tests/models/cassettes/test_openrouter/test_openrouter_google_nested_schema.yaml b/tests/models/cassettes/test_openrouter/test_openrouter_google_nested_schema.yaml index bed734f33a..a30ec9d550 100644 --- a/tests/models/cassettes/test_openrouter/test_openrouter_google_nested_schema.yaml +++ b/tests/models/cassettes/test_openrouter/test_openrouter_google_nested_schema.yaml @@ -8,7 +8,7 @@ interactions: connection: - keep-alive content-length: - - '914' + - '1241' content-type: - application/json host: @@ -26,10 +26,9 @@ interactions: description: Insert a level with its spaces. name: insert_level_with_spaces parameters: - description: Insert a level with its spaces. + additionalProperties: false properties: level: - default: null nullable: true properties: level_name: @@ -65,10 +64,34 @@ interactions: type: object type: array required: + - level - spaces type: object strict: true type: function + - function: + description: Result of inserting a level. + name: final_result + parameters: + properties: + level_name: + type: string + level_type: + enum: + - ground + - basement + - floor + - attic + type: string + space_count: + type: integer + required: + - level_name + - level_type + - space_count + type: object + strict: true + type: function uri: https://openrouter.ai/api/v1/chat/completions response: headers: @@ -77,7 +100,7 @@ interactions: connection: - keep-alive content-length: - - '1163' + - '1164' content-type: - application/json permissions-policy: @@ -101,35 +124,201 @@ interactions: role: assistant tool_calls: - function: - arguments: '{"level":{"level_name":"ground floor","level_type":"ground"},"spaces":[{"space_name":"entryway","space_type":"entryway"},{"space_type":"living-room","space_name":"living - room"},{"space_type":"garage","space_name":"garage"}]}' + arguments: '{"spaces":[{"space_type":"entryway","space_name":"entryway"},{"space_name":"living_room","space_type":"living-room"},{"space_name":"garage","space_type":"garage"}],"level":{"level_type":"ground","level_name":"ground_floor"}}' name: insert_level_with_spaces - id: tool_insert_level_with_spaces_Dp2OGkGjAwsmqUUwHwmj + id: tool_insert_level_with_spaces_3ZiChYzj8xER8HixJe7W + index: 0 + type: function + native_finish_reason: STOP + created: 1764791728 + id: gen-1764791728-YEVpGoInRszZx8oZ508T + model: google/gemini-2.5-flash + object: chat.completion + provider: Google AI Studio + usage: + completion_tokens: 91 + completion_tokens_details: + image_tokens: 0 + reasoning_tokens: 0 + cost: 0 + cost_details: + upstream_inference_completions_cost: 0.0002275 + upstream_inference_cost: 0.0003253 + upstream_inference_prompt_cost: 9.78e-05 + is_byok: true + prompt_tokens: 326 + prompt_tokens_details: + audio_tokens: 0 + cached_tokens: 0 + video_tokens: 0 + total_tokens: 417 + status: + code: 200 + message: OK +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '1883' + content-type: + - application/json + host: + - openrouter.ai + method: POST + parsed_body: + messages: + - content: It's a house with a ground floor that has an entryway, a living room and a garage. + role: user + - content: null + role: assistant + tool_calls: + - function: + arguments: '{"spaces":[{"space_type":"entryway","space_name":"entryway"},{"space_name":"living_room","space_type":"living-room"},{"space_name":"garage","space_type":"garage"}],"level":{"level_type":"ground","level_name":"ground_floor"}}' + name: insert_level_with_spaces + id: tool_insert_level_with_spaces_3ZiChYzj8xER8HixJe7W + type: function + - content: 'Inserted level level_name=''ground_floor'' level_type= with 3 spaces' + role: tool + tool_call_id: tool_insert_level_with_spaces_3ZiChYzj8xER8HixJe7W + model: google/gemini-2.5-flash + stream: false + tool_choice: required + tools: + - function: + description: Insert a level with its spaces. + name: insert_level_with_spaces + parameters: + additionalProperties: false + properties: + level: + nullable: true + properties: + level_name: + type: string + level_type: + enum: + - ground + - basement + - floor + - attic + type: string + required: + - level_name + - level_type + type: object + spaces: + items: + properties: + space_name: + type: string + space_type: + enum: + - entryway + - living-room + - kitchen + - bedroom + - bathroom + - garage + type: string + required: + - space_name + - space_type + type: object + type: array + required: + - level + - spaces + type: object + strict: true + type: function + - function: + description: Result of inserting a level. + name: final_result + parameters: + properties: + level_name: + type: string + level_type: + enum: + - ground + - basement + - floor + - attic + type: string + space_count: + type: integer + required: + - level_name + - level_type + - space_count + type: object + strict: true + type: function + uri: https://openrouter.ai/api/v1/chat/completions + response: + headers: + access-control-allow-origin: + - '*' + connection: + - keep-alive + content-length: + - '945' + content-type: + - application/json + permissions-policy: + - payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com" "https://js.stripe.com" "https://*.js.stripe.com" + "https://hooks.stripe.com") + referrer-policy: + - no-referrer, strict-origin-when-cross-origin + transfer-encoding: + - chunked + vary: + - Accept-Encoding + parsed_body: + choices: + - finish_reason: tool_calls + index: 0 + logprobs: null + message: + content: '' + reasoning: null + refusal: null + role: assistant + tool_calls: + - function: + arguments: '{"level_type":"ground","level_name":"ground_floor","space_count":3}' + name: final_result + id: tool_final_result_HesCvwqQXZaVlFW3buU8 index: 0 type: function native_finish_reason: STOP - created: 1764781178 - id: gen-1764781178-SLI9BlpnJoXSOwuii48G + created: 1764791730 + id: gen-1764791730-nym6GYDkejhRdrWtv8l3 model: google/gemini-2.5-flash object: chat.completion provider: Google AI Studio usage: - completion_tokens: 89 + completion_tokens: 33 completion_tokens_details: image_tokens: 0 reasoning_tokens: 0 cost: 0 cost_details: - upstream_inference_completions_cost: 0.0002225 - upstream_inference_cost: 0.000296 - upstream_inference_prompt_cost: 7.35e-05 + upstream_inference_completions_cost: 8.25e-05 + upstream_inference_cost: 0.0002265 + upstream_inference_prompt_cost: 0.000144 is_byok: true - prompt_tokens: 245 + prompt_tokens: 480 prompt_tokens_details: audio_tokens: 0 cached_tokens: 0 video_tokens: 0 - total_tokens: 334 + total_tokens: 513 status: code: 200 message: OK diff --git a/tests/models/test_openrouter.py b/tests/models/test_openrouter.py index 1b8e9c1112..d439fee026 100644 --- a/tests/models/test_openrouter.py +++ b/tests/models/test_openrouter.py @@ -440,38 +440,33 @@ class SpaceArg(BaseModel): space_name: str space_type: SpaceType - class InsertLevelWithSpaces(BaseModel): - """Insert a level with its spaces.""" + class InsertedLevel(BaseModel): + """Result of inserting a level.""" - level: InsertLevelArg | None = None - spaces: list[SpaceArg] + level_name: str + level_type: LevelType + space_count: int model = OpenRouterModel('google/gemini-2.5-flash', provider=provider) - response = await model_request( - model, - [ - ModelRequest.user_text_prompt( - "It's a house with a ground floor that has an entryway, a living room and a garage." - ) - ], - model_request_parameters=ModelRequestParameters( - function_tools=[ - ToolDefinition( - name='insert_level_with_spaces', - description=InsertLevelWithSpaces.__doc__ or '', - parameters_json_schema=InsertLevelWithSpaces.model_json_schema(), - ) - ], - allow_text_output=False, - ), - ) + agent: Agent[None, InsertedLevel] = Agent(model, output_type=InsertedLevel) - assert response.parts == snapshot( + @agent.tool_plain + def insert_level_with_spaces(level: InsertLevelArg | None, spaces: list[SpaceArg]) -> str: + """Insert a level with its spaces.""" + return f'Inserted level {level} with {len(spaces)} spaces' + + result = await agent.run("It's a house with a ground floor that has an entryway, a living room and a garage.") + + tool_call_message = result.all_messages()[1] + assert tool_call_message.parts == snapshot( [ ToolCallPart( tool_name='insert_level_with_spaces', - args='{"level":{"level_name":"ground floor","level_type":"ground"},"spaces":[{"space_name":"entryway","space_type":"entryway"},{"space_type":"living-room","space_name":"living room"},{"space_type":"garage","space_name":"garage"}]}', - tool_call_id='tool_insert_level_with_spaces_Dp2OGkGjAwsmqUUwHwmj', + args='{"spaces":[{"space_type":"entryway","space_name":"entryway"},{"space_name":"living_room","space_type":"living-room"},{"space_name":"garage","space_type":"garage"}],"level":{"level_type":"ground","level_name":"ground_floor"}}', + tool_call_id='tool_insert_level_with_spaces_3ZiChYzj8xER8HixJe7W', ) ] ) + + assert result.output.level_type == LevelType.ground + assert result.output.space_count == 3 diff --git a/tests/providers/test_openrouter.py b/tests/providers/test_openrouter.py index 8aa30327bb..8ee94bce7f 100644 --- a/tests/providers/test_openrouter.py +++ b/tests/providers/test_openrouter.py @@ -192,18 +192,18 @@ def test_openrouter_google_json_schema_transformer(): transformer = _OpenRouterGoogleJsonSchemaTransformer(schema) result = transformer.walk() - # const -> enum conversion (line 59) + # const -> enum conversion assert result['properties']['status'] == {'enum': ['active'], 'type': 'string'} - # oneOf -> anyOf conversion (line 68) + # oneOf -> anyOf conversion assert 'anyOf' in result['properties']['category'] assert 'oneOf' not in result['properties']['category'] - # format -> description with existing description (lines 73-75) + # format -> description with existing description assert result['properties']['email']['description'] == 'User email (format: email)' assert 'format' not in result['properties']['email'] - # format -> description without existing description (lines 76-77) + # format -> description without existing description assert result['properties']['date']['description'] == 'Format: date' assert 'format' not in result['properties']['date']