From 4ba0a5cfbd3d997e243bbff9d8854b49bea1fb47 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Sun, 4 Oct 2026 19:30:51 +0530 Subject: [PATCH 1/9] Python: fix(core): serialize Pydantic tool results in JSON mode Co-Authored-By: Claude Opus 5.5 (1M context) --- python/packages/core/agent_framework/_tools.py | 3 ++- python/packages/core/tests/core/test_types.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index 602da9dc4cb..81d9354a16b 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -1256,7 +1256,8 @@ def _make_dumpable(value: Any) -> Any: if isinstance(value, Content): return value.to_dict(exclude={"raw_representation", "additional_properties"}) if isinstance(value, BaseModel): - return value.model_dump() + # JSON mode, so enums, datetimes and UUIDs come out as Pydantic serializes them. + return value.model_dump(mode="json") if hasattr(value, "to_dict"): return value.to_dict() if hasattr(value, "text") and isinstance(value.text, str): diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 05d474687f4..32c123b80d6 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -3821,6 +3821,22 @@ def test_parse_result_nested_pydantic_model(): assert "18.0" in parsed[0].text or "18" in parsed[0].text # type: ignore[operator] # pyrefly: ignore[not-iterable] # ty: ignore[unsupported-operator] +def test_parse_result_pydantic_model_uses_json_mode(): + """Enum and datetime fields are serialized the way Pydantic writes them to JSON.""" + from enum import Enum + + class Status(Enum): + OPEN = "open" + + class Ticket(BaseModel): + status: Status + created: datetime + + parsed = FunctionTool.parse_result(Ticket(status=Status.OPEN, created=datetime(2024, 1, 1, 12))) + + assert json.loads(parsed[0].text) == {"status": "open", "created": "2024-01-01T12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + + # region FunctionTool.parse_result with MCP TextContent-like objects From c9a68d954caf15293b3840b5c537a2d35f83a6e6 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Mon, 5 Oct 2026 22:35:20 +0530 Subject: [PATCH 2/9] Python: fall back to the python dump when a model has no JSON serializer --- python/packages/core/agent_framework/_tools.py | 7 ++++++- python/packages/core/tests/core/test_types.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index 81d9354a16b..f52dde564d9 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -45,6 +45,7 @@ from opentelemetry import trace from opentelemetry.metrics import Histogram, NoOpHistogram from pydantic import BaseModel, Field, ValidationError, create_model +from pydantic_core import PydanticSerializationError from ._serialization import SerializationMixin from .exceptions import ResponseInvalidatedException, ToolException, UserInputRequiredException @@ -1257,7 +1258,11 @@ def _make_dumpable(value: Any) -> Any: return value.to_dict(exclude={"raw_representation", "additional_properties"}) if isinstance(value, BaseModel): # JSON mode, so enums, datetimes and UUIDs come out as Pydantic serializes them. - return value.model_dump(mode="json") + # Arbitrary types have no JSON serializer, so those models keep the python dump. + try: + return value.model_dump(mode="json") + except PydanticSerializationError: + return value.model_dump() if hasattr(value, "to_dict"): return value.to_dict() if hasattr(value, "text") and isinstance(value.text, str): diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 32c123b80d6..c941ae5be88 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -3837,6 +3837,24 @@ class Ticket(BaseModel): assert json.loads(parsed[0].text) == {"status": "open", "created": "2024-01-01T12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] +def test_parse_result_pydantic_model_with_arbitrary_type(): + """A field Pydantic cannot serialize to JSON falls back to str, as before.""" + + class Handle: + def __str__(self) -> str: + return "handle-1" + + class Result(BaseModel): + model_config = {"arbitrary_types_allowed": True} + + handle: Handle + created: datetime + + parsed = FunctionTool.parse_result(Result(handle=Handle(), created=datetime(2024, 1, 1, 12))) + + assert json.loads(parsed[0].text) == {"handle": "handle-1", "created": "2024-01-01 12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + + # region FunctionTool.parse_result with MCP TextContent-like objects From ee3a266d0ddf2c58f03eb3f92a4d3a591cec0455 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Wed, 7 Oct 2026 14:18:32 +0530 Subject: [PATCH 3/9] Python: fall back to str per value instead of dumping the whole model in python mode --- python/packages/core/agent_framework/_tools.py | 10 +++------- python/packages/core/tests/core/test_types.py | 12 +++++++++--- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index f52dde564d9..44fa646b4b5 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -45,7 +45,6 @@ from opentelemetry import trace from opentelemetry.metrics import Histogram, NoOpHistogram from pydantic import BaseModel, Field, ValidationError, create_model -from pydantic_core import PydanticSerializationError from ._serialization import SerializationMixin from .exceptions import ResponseInvalidatedException, ToolException, UserInputRequiredException @@ -1257,12 +1256,9 @@ def _make_dumpable(value: Any) -> Any: if isinstance(value, Content): return value.to_dict(exclude={"raw_representation", "additional_properties"}) if isinstance(value, BaseModel): - # JSON mode, so enums, datetimes and UUIDs come out as Pydantic serializes them. - # Arbitrary types have no JSON serializer, so those models keep the python dump. - try: - return value.model_dump(mode="json") - except PydanticSerializationError: - return value.model_dump() + # JSON mode, so enums, datetimes, UUIDs and JSON-only field serializers apply. Values + # Pydantic cannot serialize to JSON (arbitrary types) fall back to str individually. + return value.model_dump(mode="json", fallback=str) if hasattr(value, "to_dict"): return value.to_dict() if hasattr(value, "text") and isinstance(value.text, str): diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index c941ae5be88..18805dfd3a0 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -3838,7 +3838,8 @@ class Ticket(BaseModel): def test_parse_result_pydantic_model_with_arbitrary_type(): - """A field Pydantic cannot serialize to JSON falls back to str, as before.""" + """Only the field Pydantic cannot serialize to JSON falls back to str; the others stay in JSON mode.""" + from pydantic import field_serializer class Handle: def __str__(self) -> str: @@ -3849,10 +3850,15 @@ class Result(BaseModel): handle: Handle created: datetime + token: str - parsed = FunctionTool.parse_result(Result(handle=Handle(), created=datetime(2024, 1, 1, 12))) + @field_serializer("token", when_used="json") + def _redact(self, value: str) -> str: + return "***" - assert json.loads(parsed[0].text) == {"handle": "handle-1", "created": "2024-01-01 12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + parsed = FunctionTool.parse_result(Result(handle=Handle(), created=datetime(2024, 1, 1, 12), token="secret")) + + assert json.loads(parsed[0].text) == {"handle": "handle-1", "created": "2024-01-01T12:00:00", "token": "***"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] # region FunctionTool.parse_result with MCP TextContent-like objects From 603ec936b646a3b6fed0bbccf8cb3760e94d4d04 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Tue, 13 Oct 2026 14:00:27 +0530 Subject: [PATCH 4/9] Fall back to the python-mode dump when JSON mode can't encode a value --- python/packages/core/agent_framework/_tools.py | 7 ++++++- python/packages/core/tests/core/test_types.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index 44fa646b4b5..e1e6136e9af 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -1258,7 +1258,12 @@ def _make_dumpable(value: Any) -> Any: if isinstance(value, BaseModel): # JSON mode, so enums, datetimes, UUIDs and JSON-only field serializers apply. Values # Pydantic cannot serialize to JSON (arbitrary types) fall back to str individually. - return value.model_dump(mode="json", fallback=str) + try: + return value.model_dump(mode="json", fallback=str) + except ValueError: + # e.g. bytes that aren't valid UTF-8. The python-mode dump still drops excluded + # fields, unlike the str(result) fallback in invoke. + return value.model_dump() if hasattr(value, "to_dict"): return value.to_dict() if hasattr(value, "text") and isinstance(value.text, str): diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 18805dfd3a0..744bf46b9db 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -3861,6 +3861,20 @@ def _redact(self, value: str) -> str: assert json.loads(parsed[0].text) == {"handle": "handle-1", "created": "2024-01-01T12:00:00", "token": "***"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] +def test_parse_result_pydantic_model_with_non_utf8_bytes_keeps_excluded_fields_out(): + """Bytes that JSON mode cannot encode don't fail the whole result, and excluded fields stay out.""" + from pydantic import Field + + class Result(BaseModel): + data: bytes + api_key: str = Field(exclude=True) + + parsed = FunctionTool.parse_result(Result(data=b"\xff\xfe", api_key="secret")) + + assert "secret" not in parsed[0].text # type: ignore[operator] # pyrefly: ignore[not-iterable] # ty: ignore[unsupported-operator] + assert set(json.loads(parsed[0].text)) == {"data"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + + # region FunctionTool.parse_result with MCP TextContent-like objects From ffd46edfc341ba9528a2741942708fba3472826e Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Thu, 15 Oct 2026 03:20:00 +0530 Subject: [PATCH 5/9] Fall back per field when JSON mode can't encode a value, and only pass fallback where Pydantic supports it --- .../packages/core/agent_framework/_tools.py | 41 ++++++++++++---- python/packages/core/tests/core/test_types.py | 48 ++++++++++++++++--- 2 files changed, 75 insertions(+), 14 deletions(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index e1e6136e9af..da3fb68868e 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -411,6 +411,10 @@ def _annotation_includes_function_invocation_context(annotation: Any) -> bool: _ToolDescriptionFormat: TypeAlias = _ToolParameterFormat | Mapping[str, _ToolParameterFormat] _NormalizedToolDescriptionFormat: TypeAlias = _ToolParameterFormat | dict[str, _ToolParameterFormat] _TOOL_DESCRIPTION_FORMAT_ERROR = "tool_description_format must be 'compact', 'json', or a tool-name mapping." +# model_dump's fallback (str for values JSON mode doesn't know) only exists on newer Pydantic 2 versions. +_MODEL_DUMP_JSON_FALLBACK: dict[str, Any] = ( + {"fallback": str} if "fallback" in inspect.signature(BaseModel.model_dump).parameters else {} +) def _normalize_tool_description_format( # pyright: ignore[reportUnusedFunction] @@ -1256,20 +1260,41 @@ def _make_dumpable(value: Any) -> Any: if isinstance(value, Content): return value.to_dict(exclude={"raw_representation", "additional_properties"}) if isinstance(value, BaseModel): - # JSON mode, so enums, datetimes, UUIDs and JSON-only field serializers apply. Values - # Pydantic cannot serialize to JSON (arbitrary types) fall back to str individually. - try: - return value.model_dump(mode="json", fallback=str) - except ValueError: - # e.g. bytes that aren't valid UTF-8. The python-mode dump still drops excluded - # fields, unlike the str(result) fallback in invoke. - return value.model_dump() + return FunctionTool._dump_model(value) if hasattr(value, "to_dict"): return value.to_dict() if hasattr(value, "text") and isinstance(value.text, str): return value.text return value + @staticmethod + def _dump_model(model: BaseModel) -> Any: + """Dump a Pydantic model in JSON mode, falling back to python mode only for the fields that need it. + + JSON mode applies enum, datetime and UUID encoding and JSON-only field serializers. A field + JSON mode cannot encode (bytes that aren't valid UTF-8, or an arbitrary type on Pydantic + versions without ``fallback``) is dumped in python mode on its own, so the other fields keep + their JSON serializers. Excluded fields stay out either way, unlike the str(result) fallback + in invoke. + """ + try: + return model.model_dump(mode="json", **_MODEL_DUMP_JSON_FALLBACK) + except ValueError: + pass + model_type = type(model) + names = [*model_type.model_fields, *model_type.model_computed_fields, *(model.model_extra or {})] + dumped: dict[str, Any] = {} + for name in names: + try: + dumped.update(model.model_dump(mode="json", include={name}, **_MODEL_DUMP_JSON_FALLBACK)) + except ValueError: + field_dump = model.model_dump(include={name}) + field_value = getattr(model, name, None) + if isinstance(field_value, BaseModel): + field_dump = {key: FunctionTool._dump_model(field_value) for key in field_dump} + dumped.update(field_dump) + return dumped + @staticmethod def parse_result(result: Any) -> list[Content]: """Convert a raw function return value to a list of Content items. diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 744bf46b9db..2ab53d42b52 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -3861,18 +3861,54 @@ def _redact(self, value: str) -> str: assert json.loads(parsed[0].text) == {"handle": "handle-1", "created": "2024-01-01T12:00:00", "token": "***"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] -def test_parse_result_pydantic_model_with_non_utf8_bytes_keeps_excluded_fields_out(): - """Bytes that JSON mode cannot encode don't fail the whole result, and excluded fields stay out.""" - from pydantic import Field +def test_parse_result_pydantic_model_with_non_utf8_bytes_falls_back_per_field(): + """Only the bytes fall back; JSON-only serializers still run and excluded fields stay out.""" + from pydantic import Field, field_serializer class Result(BaseModel): data: bytes + created: datetime + token: str api_key: str = Field(exclude=True) - parsed = FunctionTool.parse_result(Result(data=b"\xff\xfe", api_key="secret")) + @field_serializer("token", when_used="json") + def _redact(self, value: str) -> str: + return "***" + + result = Result(data=b"\xff\xfe", created=datetime(2024, 1, 1, 12), token="secret", api_key="key") + parsed = FunctionTool.parse_result(result) + + assert json.loads(parsed[0].text) == { # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + "data": str(b"\xff\xfe"), + "created": "2024-01-01T12:00:00", + "token": "***", + } + + +def test_parse_result_nested_pydantic_model_with_non_utf8_bytes(): + """A nested model with bytes JSON mode cannot encode keeps its JSON-only serializers.""" + from pydantic import field_serializer + + class Inner(BaseModel): + data: bytes + token: str + + @field_serializer("token", when_used="json") + def _redact(self, value: str) -> str: + return "***" + + class Outer(BaseModel): + inner: Inner + created: datetime + + parsed = FunctionTool.parse_result( + Outer(inner=Inner(data=b"\xff\xfe", token="secret"), created=datetime(2024, 1, 1, 12)) + ) - assert "secret" not in parsed[0].text # type: ignore[operator] # pyrefly: ignore[not-iterable] # ty: ignore[unsupported-operator] - assert set(json.loads(parsed[0].text)) == {"data"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + assert json.loads(parsed[0].text) == { # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + "inner": {"data": str(b"\xff\xfe"), "token": "***"}, + "created": "2024-01-01T12:00:00", + } # region FunctionTool.parse_result with MCP TextContent-like objects From 4295bcd3ad9e7f7a9c08b1e8bbbd5b6068137ad9 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Thu, 15 Oct 2026 16:43:15 +0530 Subject: [PATCH 6/9] Take fallback field names from the python dump and keep JSON mode for models in lists and dicts --- .../packages/core/agent_framework/_tools.py | 69 +++++++++++++++---- python/packages/core/tests/core/test_types.py | 43 ++++++++++++ 2 files changed, 97 insertions(+), 15 deletions(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index da3fb68868e..4b7b1c725d3 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -44,7 +44,8 @@ from opentelemetry import trace from opentelemetry.metrics import Histogram, NoOpHistogram -from pydantic import BaseModel, Field, ValidationError, create_model +from pydantic import BaseModel, Field, PlainSerializer, ValidationError, WrapSerializer, create_model +from pydantic_core import to_jsonable_python from ._serialization import SerializationMixin from .exceptions import ResponseInvalidatedException, ToolException, UserInputRequiredException @@ -417,6 +418,17 @@ def _annotation_includes_function_invocation_context(annotation: Any) -> bool: ) +def _has_field_serializer(model_type: type[BaseModel], name: str) -> bool: + """Whether a model field has its own serializer, from a decorator or an annotation.""" + decorators = model_type.__pydantic_decorators__.field_serializers.values() + if any(name in decorator.info.fields or "*" in decorator.info.fields for decorator in decorators): + return True + field = model_type.model_fields.get(name) + return field is not None and any( + isinstance(metadata, (PlainSerializer, WrapSerializer)) for metadata in field.metadata + ) + + def _normalize_tool_description_format( # pyright: ignore[reportUnusedFunction] value: object, ) -> _NormalizedToolDescriptionFormat: @@ -1269,32 +1281,59 @@ def _make_dumpable(value: Any) -> Any: @staticmethod def _dump_model(model: BaseModel) -> Any: - """Dump a Pydantic model in JSON mode, falling back to python mode only for the fields that need it. - - JSON mode applies enum, datetime and UUID encoding and JSON-only field serializers. A field - JSON mode cannot encode (bytes that aren't valid UTF-8, or an arbitrary type on Pydantic - versions without ``fallback``) is dumped in python mode on its own, so the other fields keep - their JSON serializers. Excluded fields stay out either way, unlike the str(result) fallback - in invoke. + """Dump a Pydantic model in JSON mode, falling back to python mode only for the values that need it. + + JSON mode applies enum, datetime and UUID encoding and JSON-only field serializers. When it + fails (bytes that aren't valid UTF-8, or an arbitrary type on Pydantic versions without + ``fallback``), each field is dumped in JSON mode on its own and only a failing field falls + back. Models inside a failing field, directly or in a list or dict, are dumped the same way, + so their JSON serializers and excluded fields still apply. Excluded fields stay out either + way, unlike the str(result) fallback in invoke. """ try: return model.model_dump(mode="json", **_MODEL_DUMP_JSON_FALLBACK) except ValueError: pass + # The python-mode dump gives the serialized field names without the excluded ones. The class's + # model_computed_fields can't be used for that, it's a property on Pydantic 2.0. + python_dump = model.model_dump(by_alias=False) + if not isinstance(python_dump, dict): + return model.model_dump() model_type = type(model) - names = [*model_type.model_fields, *model_type.model_computed_fields, *(model.model_extra or {})] dumped: dict[str, Any] = {} - for name in names: + for name in python_dump: try: dumped.update(model.model_dump(mode="json", include={name}, **_MODEL_DUMP_JSON_FALLBACK)) + continue except ValueError: - field_dump = model.model_dump(include={name}) - field_value = getattr(model, name, None) - if isinstance(field_value, BaseModel): - field_dump = {key: FunctionTool._dump_model(field_value) for key in field_dump} - dumped.update(field_dump) + pass + for key, python_value in model.model_dump(include={name}).items(): + value = getattr(model, name, None) + if isinstance(value, (BaseModel, dict, list, tuple, set, frozenset)) and not _has_field_serializer( + model_type, name + ): + dumped[key] = FunctionTool._dump_fallback_value(value) + else: + dumped[key] = python_value return dumped + @staticmethod + def _dump_fallback_value(value: Any) -> Any: + """Dump a field value JSON mode could not encode, keeping JSON mode for what inside it can be.""" + if isinstance(value, BaseModel): + return FunctionTool._dump_model(value) + if isinstance(value, dict): + dict_value = cast(dict[object, object], value) + return {key: FunctionTool._dump_fallback_value(item) for key, item in dict_value.items()} + if isinstance(value, (list, tuple, set, frozenset)): + items = cast(Iterable[object], value) + return [FunctionTool._dump_fallback_value(item) for item in items] + try: + return to_jsonable_python(value) + except ValueError: + # parse_result's json.dumps(default=str) turns it into a string + return value + @staticmethod def parse_result(result: Any) -> list[Content]: """Convert a raw function return value to a list of Content items. diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 2ab53d42b52..b30cf55ce23 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -3911,6 +3911,49 @@ class Outer(BaseModel): } +def test_parse_result_pydantic_model_with_non_utf8_bytes_keeps_computed_fields(): + """The per-field fallback includes computed fields and leaves excluded fields out.""" + from pydantic import Field, computed_field + + class Result(BaseModel): + data: bytes + api_key: str = Field(exclude=True) + + @computed_field # type: ignore[prop-decorator] + @property + def size(self) -> int: + return len(self.data) + + parsed = FunctionTool.parse_result(Result(data=b"\xff\xfe", api_key="key")) + + assert json.loads(parsed[0].text) == {"data": str(b"\xff\xfe"), "size": 2} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + + +def test_parse_result_pydantic_models_in_list_and_dict_with_non_utf8_bytes(): + """Models inside a list or dict field that falls back keep their JSON-only serializers and exclusions.""" + from pydantic import Field, field_serializer + + class Inner(BaseModel): + data: bytes + created: datetime + token: str + api_key: str = Field(exclude=True) + + @field_serializer("token", when_used="json") + def _redact(self, value: str) -> str: + return "***" + + class Outer(BaseModel): + items: list[Inner] + by_name: dict[str, Inner] + + inner = Inner(data=b"\xff\xfe", created=datetime(2024, 1, 1, 12), token="secret", api_key="key") + parsed = FunctionTool.parse_result(Outer(items=[inner], by_name={"a": inner})) + + expected_inner = {"data": str(b"\xff\xfe"), "created": "2024-01-01T12:00:00", "token": "***"} + assert json.loads(parsed[0].text) == {"items": [expected_inner], "by_name": {"a": expected_inner}} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + + # region FunctionTool.parse_result with MCP TextContent-like objects From 0a73949d4420daaaf807bf1181337fdf81c204ca Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Thu, 22 Oct 2026 23:53:56 +0530 Subject: [PATCH 7/9] Keep JSON mode for a field with its own serializer when its output has invalid UTF-8 bytes --- .../packages/core/agent_framework/_tools.py | 40 ++++++++++++++++--- python/packages/core/tests/core/test_types.py | 38 ++++++++++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index 7e1d48d3564..d11417e7cfa 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -517,6 +517,24 @@ def _has_field_serializer(model_type: type[BaseModel], name: str) -> bool: ) +def _escape_invalid_utf8(value: Any) -> Any: + """Replace bytes that aren't valid UTF-8 with their str() form, inside models, dicts and sequences too.""" + if isinstance(value, bytes): + try: + value.decode("utf-8") + except UnicodeDecodeError: + return str(value).encode() + return value + if isinstance(value, BaseModel): + return value.model_copy(update={name: _escape_invalid_utf8(item) for name, item in value.__dict__.items()}) + if isinstance(value, dict): + dict_value = cast(dict[object, object], value) + return {key: _escape_invalid_utf8(item) for key, item in dict_value.items()} + if type(value) in (list, tuple, set, frozenset): + return type(value)(_escape_invalid_utf8(item) for item in value) + return value + + def _normalize_tool_description_format( # pyright: ignore[reportUnusedFunction] value: object, ) -> _NormalizedToolDescriptionFormat: @@ -1402,7 +1420,9 @@ def _dump_model(model: BaseModel) -> Any: fails (bytes that aren't valid UTF-8, or an arbitrary type on Pydantic versions without ``fallback``), each field is dumped in JSON mode on its own and only a failing field falls back. Models inside a failing field, directly or in a list or dict, are dumped the same way, - so their JSON serializers and excluded fields still apply. Excluded fields stay out either + so their JSON serializers and excluded fields still apply. A failing field with its own + serializer is dumped in JSON mode again with its invalid UTF-8 bytes escaped, so the + serializer still runs, and is left out if that fails too. Excluded fields stay out either way, unlike the str(result) fallback in invoke. """ try: @@ -1422,11 +1442,21 @@ def _dump_model(model: BaseModel) -> Any: continue except ValueError: pass + value = getattr(model, name, None) + if _has_field_serializer(model_type, name): + # The python-mode value would skip the field's serializer, and with it any redaction it does. + escaped = model.model_copy(update={name: _escape_invalid_utf8(value)}) + try: + dumped.update(escaped.model_dump(mode="json", include={name}, **_MODEL_DUMP_JSON_FALLBACK)) + except ValueError: + logger.warning( + "Leaving %s.%s out of the tool result: its serializer's output can't be encoded as JSON.", + model_type.__name__, + name, + ) + continue for key, python_value in model.model_dump(include={name}).items(): - value = getattr(model, name, None) - if isinstance(value, (BaseModel, dict, list, tuple, set, frozenset)) and not _has_field_serializer( - model_type, name - ): + if isinstance(value, (BaseModel, dict, list, tuple, set, frozenset)): dumped[key] = FunctionTool._dump_fallback_value(value) else: dumped[key] = python_value diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 1da72d71082..1d417e529c0 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -4660,6 +4660,44 @@ class Outer(BaseModel): assert json.loads(parsed[0].text) == {"items": [expected_inner], "by_name": {"a": expected_inner}} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] +def test_parse_result_pydantic_field_serializer_output_with_non_utf8_bytes(): + """A JSON-only serializer whose output has bytes JSON mode cannot encode still runs; only the bytes fall back.""" + from pydantic import field_serializer + + class Result(BaseModel): + info: dict[str, Any] + created: datetime + + @field_serializer("info", when_used="json") + def _redact(self, value: dict[str, Any]) -> dict[str, Any]: + return {**value, "token": "***"} + + result = Result(info={"token": "secret", "data": b"\xff\xfe"}, created=datetime(2024, 1, 1, 12)) + parsed = FunctionTool.parse_result(result) + + assert json.loads(parsed[0].text) == { # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + "info": {"token": "***", "data": str(b"\xff\xfe")}, + "created": "2024-01-01T12:00:00", + } + + +def test_parse_result_pydantic_field_serializer_output_that_cannot_be_encoded_is_left_out(): + """A field whose serializer output JSON mode cannot encode is left out instead of sent without the serializer.""" + from pydantic import field_serializer + + class Result(BaseModel): + token: str + created: datetime + + @field_serializer("token", when_used="json") + def _redact(self, value: str) -> bytes: + return b"\xff***" + + parsed = FunctionTool.parse_result(Result(token="secret", created=datetime(2024, 1, 1, 12))) + + assert json.loads(parsed[0].text) == {"created": "2024-01-01T12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + + # region FunctionTool.parse_result with MCP TextContent-like objects From 317e9bb918ac623f2922cff38dfbef3592f73c91 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Wed, 4 Nov 2026 19:32:10 +0530 Subject: [PATCH 8/9] Keep the declared field types when the JSON dump falls back The fallback dumped raw attribute values by their own class, so a subclass in a field declared as its base sent its subclass-only fields. Escape the values JSON mode can't encode in a copy of the model and let Pydantic dump the copy instead. --- .../packages/core/agent_framework/_tools.py | 95 +++++++------------ python/packages/core/tests/core/test_types.py | 25 +++++ 2 files changed, 57 insertions(+), 63 deletions(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index d11417e7cfa..169d8c1c514 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -48,7 +48,7 @@ from opentelemetry import trace from opentelemetry.metrics import Histogram, NoOpHistogram from opentelemetry.util.types import AttributeValue -from pydantic import BaseModel, Field, PlainSerializer, TypeAdapter, ValidationError, WrapSerializer, create_model +from pydantic import BaseModel, Field, TypeAdapter, ValidationError, create_model from pydantic_core import to_jsonable_python from ._serialization import SerializationMixin @@ -506,19 +506,13 @@ def _annotation_includes_function_invocation_context(annotation: Any) -> bool: ) -def _has_field_serializer(model_type: type[BaseModel], name: str) -> bool: - """Whether a model field has its own serializer, from a decorator or an annotation.""" - decorators = model_type.__pydantic_decorators__.field_serializers.values() - if any(name in decorator.info.fields or "*" in decorator.info.fields for decorator in decorators): - return True - field = model_type.model_fields.get(name) - return field is not None and any( - isinstance(metadata, (PlainSerializer, WrapSerializer)) for metadata in field.metadata - ) - +def _escape_unencodable(value: Any) -> Any: + """Copy a value with the parts JSON mode can't encode replaced, inside models, dicts and sequences too. -def _escape_invalid_utf8(value: Any) -> Any: - """Replace bytes that aren't valid UTF-8 with their str() form, inside models, dicts and sequences too.""" + Bytes that aren't valid UTF-8 become their str() form. On Pydantic versions without ``fallback``, + values JSON mode doesn't know become str() too. Models are copied, not rebuilt, so their class stays + the same and the parent's declared field types still decide how they are serialized. + """ if isinstance(value, bytes): try: value.decode("utf-8") @@ -526,12 +520,17 @@ def _escape_invalid_utf8(value: Any) -> Any: return str(value).encode() return value if isinstance(value, BaseModel): - return value.model_copy(update={name: _escape_invalid_utf8(item) for name, item in value.__dict__.items()}) + return value.model_copy(update={name: _escape_unencodable(item) for name, item in value.__dict__.items()}) if isinstance(value, dict): dict_value = cast(dict[object, object], value) - return {key: _escape_invalid_utf8(item) for key, item in dict_value.items()} + return {key: _escape_unencodable(item) for key, item in dict_value.items()} if type(value) in (list, tuple, set, frozenset): - return type(value)(_escape_invalid_utf8(item) for item in value) + return type(value)(_escape_unencodable(item) for item in value) + if not _MODEL_DUMP_JSON_FALLBACK: + try: + to_jsonable_python(value) + except ValueError: + return str(value) return value @@ -1414,71 +1413,41 @@ def _make_dumpable(value: Any) -> Any: @staticmethod def _dump_model(model: BaseModel) -> Any: - """Dump a Pydantic model in JSON mode, falling back to python mode only for the values that need it. + """Dump a Pydantic model in JSON mode. JSON mode applies enum, datetime and UUID encoding and JSON-only field serializers. When it fails (bytes that aren't valid UTF-8, or an arbitrary type on Pydantic versions without - ``fallback``), each field is dumped in JSON mode on its own and only a failing field falls - back. Models inside a failing field, directly or in a list or dict, are dumped the same way, - so their JSON serializers and excluded fields still apply. A failing field with its own - serializer is dumped in JSON mode again with its invalid UTF-8 bytes escaped, so the - serializer still runs, and is left out if that fails too. Excluded fields stay out either - way, unlike the str(result) fallback in invoke. + ``fallback``), those values are escaped in a copy of the model and Pydantic dumps the copy, so + the declared field types, serializers and exclusions still decide what is written. Computed + fields are dumped from the original model. A field that still can't be encoded is left out. + Excluded fields stay out either way, unlike the str(result) fallback in invoke. """ try: return model.model_dump(mode="json", **_MODEL_DUMP_JSON_FALLBACK) except ValueError: pass + escaped = _escape_unencodable(model) # The python-mode dump gives the serialized field names without the excluded ones. The class's # model_computed_fields can't be used for that, it's a property on Pydantic 2.0. python_dump = model.model_dump(by_alias=False) if not isinstance(python_dump, dict): - return model.model_dump() - model_type = type(model) + try: + return escaped.model_dump(mode="json", **_MODEL_DUMP_JSON_FALLBACK) + except ValueError: + logger.warning("Leaving %s out of the tool result: it can't be encoded as JSON.", type(model).__name__) + return None + fields = type(model).model_fields dumped: dict[str, Any] = {} for name in python_dump: + source = escaped if name in fields else model try: - dumped.update(model.model_dump(mode="json", include={name}, **_MODEL_DUMP_JSON_FALLBACK)) - continue + dumped.update(source.model_dump(mode="json", include={name}, **_MODEL_DUMP_JSON_FALLBACK)) except ValueError: - pass - value = getattr(model, name, None) - if _has_field_serializer(model_type, name): - # The python-mode value would skip the field's serializer, and with it any redaction it does. - escaped = model.model_copy(update={name: _escape_invalid_utf8(value)}) - try: - dumped.update(escaped.model_dump(mode="json", include={name}, **_MODEL_DUMP_JSON_FALLBACK)) - except ValueError: - logger.warning( - "Leaving %s.%s out of the tool result: its serializer's output can't be encoded as JSON.", - model_type.__name__, - name, - ) - continue - for key, python_value in model.model_dump(include={name}).items(): - if isinstance(value, (BaseModel, dict, list, tuple, set, frozenset)): - dumped[key] = FunctionTool._dump_fallback_value(value) - else: - dumped[key] = python_value + logger.warning( + "Leaving %s.%s out of the tool result: it can't be encoded as JSON.", type(model).__name__, name + ) return dumped - @staticmethod - def _dump_fallback_value(value: Any) -> Any: - """Dump a field value JSON mode could not encode, keeping JSON mode for what inside it can be.""" - if isinstance(value, BaseModel): - return FunctionTool._dump_model(value) - if isinstance(value, dict): - dict_value = cast(dict[object, object], value) - return {key: FunctionTool._dump_fallback_value(item) for key, item in dict_value.items()} - if isinstance(value, (list, tuple, set, frozenset)): - items = cast(Iterable[object], value) - return [FunctionTool._dump_fallback_value(item) for item in items] - try: - return to_jsonable_python(value) - except ValueError: - # parse_result's json.dumps(default=str) turns it into a string - return value - @staticmethod def parse_result(result: Any) -> list[Content]: """Convert a raw function return value to a list of Content items. diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 1d417e529c0..8616dbb8572 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -4698,6 +4698,31 @@ def _redact(self, value: str) -> bytes: assert json.loads(parsed[0].text) == {"created": "2024-01-01T12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] +def test_parse_result_pydantic_fallback_keeps_declared_field_types(): + """A subclass in a field declared as its base keeps the subclass-only fields out when the fallback is used.""" + + class Public(BaseModel): + data: bytes + + class Secret(Public): + api_key: str + + class Envelope(BaseModel): + item: Public + items: list[Public] + by_name: dict[str, Public] + + secret = Secret(data=b"\xff\xfe", api_key="key") + parsed = FunctionTool.parse_result(Envelope(item=secret, items=[secret], by_name={"a": secret})) + + expected_item = {"data": str(b"\xff\xfe")} + assert json.loads(parsed[0].text) == { # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + "item": expected_item, + "items": [expected_item], + "by_name": {"a": expected_item}, + } + + # region FunctionTool.parse_result with MCP TextContent-like objects From b40e2dc1c07b4602563610f604394af2617754e0 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Tue, 10 Nov 2026 02:36:21 +0530 Subject: [PATCH 9/9] Let Pydantic dump the whole model, with a placeholder when it can't --- .../packages/core/agent_framework/_tools.py | 77 ++---- python/packages/core/tests/core/test_types.py | 244 ++++++++++-------- 2 files changed, 153 insertions(+), 168 deletions(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index 491a0074740..9a1f142cfb7 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -514,37 +514,7 @@ def _annotation_includes_function_invocation_context(annotation: Any) -> bool: _NormalizedToolDescriptionFormat: TypeAlias = _ToolParameterFormat | dict[str, _ToolParameterFormat] _TOOL_DESCRIPTION_FORMAT_ERROR = "tool_description_format must be 'compact', 'json', or a tool-name mapping." # model_dump's fallback (str for values JSON mode doesn't know) only exists on newer Pydantic 2 versions. -_MODEL_DUMP_JSON_FALLBACK: dict[str, Any] = ( - {"fallback": str} if "fallback" in inspect.signature(BaseModel.model_dump).parameters else {} -) - - -def _escape_unencodable(value: Any) -> Any: - """Copy a value with the parts JSON mode can't encode replaced, inside models, dicts and sequences too. - - Bytes that aren't valid UTF-8 become their str() form. On Pydantic versions without ``fallback``, - values JSON mode doesn't know become str() too. Models are copied, not rebuilt, so their class stays - the same and the parent's declared field types still decide how they are serialized. - """ - if isinstance(value, bytes): - try: - value.decode("utf-8") - except UnicodeDecodeError: - return str(value).encode() - return value - if isinstance(value, BaseModel): - return value.model_copy(update={name: _escape_unencodable(item) for name, item in value.__dict__.items()}) - if isinstance(value, dict): - dict_value = cast(dict[object, object], value) - return {key: _escape_unencodable(item) for key, item in dict_value.items()} - if type(value) in (list, tuple, set, frozenset): - return type(value)(_escape_unencodable(item) for item in value) - if not _MODEL_DUMP_JSON_FALLBACK: - try: - to_jsonable_python(value) - except ValueError: - return str(value) - return value +_MODEL_DUMP_HAS_FALLBACK = "fallback" in inspect.signature(BaseModel.model_dump).parameters def _normalize_tool_description_format( # pyright: ignore[reportUnusedFunction] @@ -1473,38 +1443,23 @@ def _make_dumpable(value: Any) -> Any: def _dump_model(model: BaseModel) -> Any: """Dump a Pydantic model in JSON mode. - JSON mode applies enum, datetime and UUID encoding and JSON-only field serializers. When it - fails (bytes that aren't valid UTF-8, or an arbitrary type on Pydantic versions without - ``fallback``), those values are escaped in a copy of the model and Pydantic dumps the copy, so - the declared field types, serializers and exclusions still decide what is written. Computed - fields are dumped from the original model. A field that still can't be encoded is left out. - Excluded fields stay out either way, unlike the str(result) fallback in invoke. + JSON mode applies enum, datetime and UUID encoding and JSON-only serializers, and values it + doesn't know become their str() form. Pydantic versions without ``model_dump(..., fallback=...)`` + get the same output from ``to_jsonable_python(..., serialize_unknown=True)``. If Pydantic still + can't dump the model (bytes that aren't valid UTF-8, a reference cycle), a placeholder is returned, + so no part of the model is sent without its serializers and exclusions. """ try: - return model.model_dump(mode="json", **_MODEL_DUMP_JSON_FALLBACK) - except ValueError: - pass - escaped = _escape_unencodable(model) - # The python-mode dump gives the serialized field names without the excluded ones. The class's - # model_computed_fields can't be used for that, it's a property on Pydantic 2.0. - python_dump = model.model_dump(by_alias=False) - if not isinstance(python_dump, dict): - try: - return escaped.model_dump(mode="json", **_MODEL_DUMP_JSON_FALLBACK) - except ValueError: - logger.warning("Leaving %s out of the tool result: it can't be encoded as JSON.", type(model).__name__) - return None - fields = type(model).model_fields - dumped: dict[str, Any] = {} - for name in python_dump: - source = escaped if name in fields else model - try: - dumped.update(source.model_dump(mode="json", include={name}, **_MODEL_DUMP_JSON_FALLBACK)) - except ValueError: - logger.warning( - "Leaving %s.%s out of the tool result: it can't be encoded as JSON.", type(model).__name__, name - ) - return dumped + if _MODEL_DUMP_HAS_FALLBACK: + return model.model_dump(mode="json", fallback=str) + return to_jsonable_python(model, by_alias=False, serialize_unknown=True) + except Exception as exception: + logger.warning( + "Replacing %s in the tool result with a placeholder: it can't be serialized as JSON (%s).", + type(model).__name__, + type(exception).__name__, + ) + return f"[{type(model).__name__} could not be serialized as JSON]" @staticmethod def parse_result(result: Any) -> list[Content]: diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 25dee0f15ff..854840ffeb1 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -4721,36 +4721,38 @@ class Ticket(BaseModel): assert json.loads(parsed[0].text) == {"status": "open", "created": "2024-01-01T12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] -def test_parse_result_pydantic_model_with_arbitrary_type(): - """Only the field Pydantic cannot serialize to JSON falls back to str; the others stay in JSON mode.""" - from pydantic import field_serializer +class _UnknownValue: + """A value Pydantic JSON mode doesn't know how to encode.""" - class Handle: - def __str__(self) -> str: - return "handle-1" + def __str__(self) -> str: + return "unknown-1" - class Result(BaseModel): - model_config = {"arbitrary_types_allowed": True} - handle: Handle - created: datetime - token: str +@fixture(params=["model_dump", "to_jsonable_python"]) +def pydantic_json_dump(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: + """Run a test with model_dump(fallback=str) and with the path for Pydantic versions without it.""" + from agent_framework import _tools - @field_serializer("token", when_used="json") - def _redact(self, value: str) -> str: - return "***" + if request.param == "model_dump" and not _tools._MODEL_DUMP_HAS_FALLBACK: + pytest.skip("model_dump has no fallback parameter on this Pydantic version") + monkeypatch.setattr(_tools, "_MODEL_DUMP_HAS_FALLBACK", request.param == "model_dump") - parsed = FunctionTool.parse_result(Result(handle=Handle(), created=datetime(2024, 1, 1, 12), token="secret")) - assert json.loads(parsed[0].text) == {"handle": "handle-1", "created": "2024-01-01T12:00:00", "token": "***"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] +def _parse_result_json(result: Any) -> Any: + text = FunctionTool.parse_result(result)[0].text + assert text is not None + return json.loads(text) -def test_parse_result_pydantic_model_with_non_utf8_bytes_falls_back_per_field(): - """Only the bytes fall back; JSON-only serializers still run and excluded fields stay out.""" - from pydantic import Field, field_serializer +@mark.usefixtures("pydantic_json_dump") +def test_parse_result_pydantic_model_with_arbitrary_type(): + """Only the value Pydantic cannot serialize to JSON becomes str; the rest stays in JSON mode.""" + from pydantic import field_serializer class Result(BaseModel): - data: bytes + model_config = {"arbitrary_types_allowed": True} + + handle: _UnknownValue created: datetime token: str api_key: str = Field(exclude=True) @@ -4759,128 +4761,118 @@ class Result(BaseModel): def _redact(self, value: str) -> str: return "***" - result = Result(data=b"\xff\xfe", created=datetime(2024, 1, 1, 12), token="secret", api_key="key") - parsed = FunctionTool.parse_result(result) + result = Result(handle=_UnknownValue(), created=datetime(2024, 1, 1, 12), token="secret", api_key="key") - assert json.loads(parsed[0].text) == { # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] - "data": str(b"\xff\xfe"), - "created": "2024-01-01T12:00:00", - "token": "***", - } + assert _parse_result_json(result) == {"handle": "unknown-1", "created": "2024-01-01T12:00:00", "token": "***"} -def test_parse_result_nested_pydantic_model_with_non_utf8_bytes(): - """A nested model with bytes JSON mode cannot encode keeps its JSON-only serializers.""" - from pydantic import field_serializer +@mark.usefixtures("pydantic_json_dump") +def test_parse_result_pydantic_model_serializer_output_is_kept(): + """A wrapping model serializer keeps its structure when a field needs the str fallback.""" + from pydantic import model_serializer - class Inner(BaseModel): - data: bytes - token: str - - @field_serializer("token", when_used="json") - def _redact(self, value: str) -> str: - return "***" + class Result(BaseModel): + model_config = {"arbitrary_types_allowed": True} - class Outer(BaseModel): - inner: Inner - created: datetime + count: int + handle: _UnknownValue - parsed = FunctionTool.parse_result( - Outer(inner=Inner(data=b"\xff\xfe", token="secret"), created=datetime(2024, 1, 1, 12)) - ) + @model_serializer(mode="wrap") + def _wrap(self, handler: Any) -> dict[str, Any]: + return {"payload": handler(self)} - assert json.loads(parsed[0].text) == { # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] - "inner": {"data": str(b"\xff\xfe"), "token": "***"}, - "created": "2024-01-01T12:00:00", + assert _parse_result_json(Result(count=1, handle=_UnknownValue())) == { + "payload": {"count": 1, "handle": "unknown-1"} } -def test_parse_result_pydantic_model_with_non_utf8_bytes_keeps_computed_fields(): - """The per-field fallback includes computed fields and leaves excluded fields out.""" - from pydantic import Field, computed_field +@mark.usefixtures("pydantic_json_dump") +def test_parse_result_nested_pydantic_computed_field_uses_original_values(): + """A nested computed field reads the original value, not its str() fallback.""" + from pydantic import computed_field - class Result(BaseModel): - data: bytes - api_key: str = Field(exclude=True) + class Inner(BaseModel): + model_config = {"arbitrary_types_allowed": True} + + handle: _UnknownValue @computed_field # type: ignore[prop-decorator] @property - def size(self) -> int: - return len(self.data) - - parsed = FunctionTool.parse_result(Result(data=b"\xff\xfe", api_key="key")) + def handle_type(self) -> str: + return type(self.handle).__name__ - assert json.loads(parsed[0].text) == {"data": str(b"\xff\xfe"), "size": 2} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + class Outer(BaseModel): + inner: Inner + assert _parse_result_json(Outer(inner=Inner(handle=_UnknownValue()))) == { + "inner": {"handle": "unknown-1", "handle_type": "_UnknownValue"} + } -def test_parse_result_pydantic_models_in_list_and_dict_with_non_utf8_bytes(): - """Models inside a list or dict field that falls back keep their JSON-only serializers and exclusions.""" - from pydantic import Field, field_serializer - class Inner(BaseModel): - data: bytes - created: datetime - token: str - api_key: str = Field(exclude=True) +@mark.usefixtures("pydantic_json_dump") +def test_parse_result_pydantic_bytes_field_serializer_gets_original_bytes(): + """A bytes field serializer receives the original bytes when a sibling needs the str fallback.""" + from pydantic import field_serializer - @field_serializer("token", when_used="json") - def _redact(self, value: str) -> str: - return "***" + class Result(BaseModel): + model_config = {"arbitrary_types_allowed": True} - class Outer(BaseModel): - items: list[Inner] - by_name: dict[str, Inner] + raw: bytes + handle: _UnknownValue - inner = Inner(data=b"\xff\xfe", created=datetime(2024, 1, 1, 12), token="secret", api_key="key") - parsed = FunctionTool.parse_result(Outer(items=[inner], by_name={"a": inner})) + @field_serializer("raw") + def _hex(self, value: bytes) -> str: + return value.hex() - expected_inner = {"data": str(b"\xff\xfe"), "created": "2024-01-01T12:00:00", "token": "***"} - assert json.loads(parsed[0].text) == {"items": [expected_inner], "by_name": {"a": expected_inner}} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + assert _parse_result_json(Result(raw=b"\x01\xff", handle=_UnknownValue())) == { + "raw": "01ff", + "handle": "unknown-1", + } -def test_parse_result_pydantic_field_serializer_output_with_non_utf8_bytes(): - """A JSON-only serializer whose output has bytes JSON mode cannot encode still runs; only the bytes fall back.""" - from pydantic import field_serializer +@mark.usefixtures("pydantic_json_dump") +def test_parse_result_pydantic_excluded_cyclic_field_is_not_visited(): + """Excluded fields, including one holding a reference cycle, stay out of the tool result.""" class Result(BaseModel): - info: dict[str, Any] - created: datetime + model_config = {"arbitrary_types_allowed": True} - @field_serializer("info", when_used="json") - def _redact(self, value: dict[str, Any]) -> dict[str, Any]: - return {**value, "token": "***"} + handle: _UnknownValue + cache: Any = Field(default=None, exclude=True) + api_key: str = Field(exclude=True) - result = Result(info={"token": "secret", "data": b"\xff\xfe"}, created=datetime(2024, 1, 1, 12)) - parsed = FunctionTool.parse_result(result) + result = Result(handle=_UnknownValue(), api_key="key") + result.cache = result - assert json.loads(parsed[0].text) == { # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] - "info": {"token": "***", "data": str(b"\xff\xfe")}, - "created": "2024-01-01T12:00:00", - } + assert _parse_result_json(result) == {"handle": "unknown-1"} -def test_parse_result_pydantic_field_serializer_output_that_cannot_be_encoded_is_left_out(): - """A field whose serializer output JSON mode cannot encode is left out instead of sent without the serializer.""" - from pydantic import field_serializer +@mark.usefixtures("pydantic_json_dump") +def test_parse_result_nested_pydantic_dataclass_keeps_exclusions(): + """A nested Pydantic dataclass keeps its exclusions; only the unknown value becomes str.""" + from pydantic.dataclasses import dataclass as pydantic_dataclass - class Result(BaseModel): - token: str - created: datetime + @pydantic_dataclass(config={"arbitrary_types_allowed": True}) + class Inner: + handle: _UnknownValue + api_key: str = Field(default="key", exclude=True) - @field_serializer("token", when_used="json") - def _redact(self, value: str) -> bytes: - return b"\xff***" + class Outer(BaseModel): + model_config = {"arbitrary_types_allowed": True} - parsed = FunctionTool.parse_result(Result(token="secret", created=datetime(2024, 1, 1, 12))) + inner: Inner - assert json.loads(parsed[0].text) == {"created": "2024-01-01T12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + assert _parse_result_json(Outer(inner=Inner(handle=_UnknownValue()))) == {"inner": {"handle": "unknown-1"}} -def test_parse_result_pydantic_fallback_keeps_declared_field_types(): - """A subclass in a field declared as its base keeps the subclass-only fields out when the fallback is used.""" +@mark.usefixtures("pydantic_json_dump") +def test_parse_result_pydantic_declared_field_types_are_kept(): + """A subclass in a field declared as its base keeps its subclass-only fields out.""" class Public(BaseModel): - data: bytes + model_config = {"arbitrary_types_allowed": True} + + handle: _UnknownValue class Secret(Public): api_key: str @@ -4890,17 +4882,55 @@ class Envelope(BaseModel): items: list[Public] by_name: dict[str, Public] - secret = Secret(data=b"\xff\xfe", api_key="key") - parsed = FunctionTool.parse_result(Envelope(item=secret, items=[secret], by_name={"a": secret})) + secret = Secret(handle=_UnknownValue(), api_key="key") + expected_item = {"handle": "unknown-1"} - expected_item = {"data": str(b"\xff\xfe")} - assert json.loads(parsed[0].text) == { # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + assert _parse_result_json(Envelope(item=secret, items=[secret], by_name={"a": secret})) == { "item": expected_item, "items": [expected_item], "by_name": {"a": expected_item}, } +@mark.usefixtures("pydantic_json_dump") +def test_parse_result_pydantic_model_with_non_utf8_bytes_is_replaced(): + """A model with bytes that aren't valid UTF-8 is replaced by a placeholder, so nothing in it leaks.""" + from pydantic import field_serializer + + class Inner(BaseModel): + data: bytes + token: str + api_key: str = Field(exclude=True) + + @field_serializer("token", when_used="json") + def _redact(self, value: str) -> str: + return "***" + + class Outer(BaseModel): + inner: Inner + + inner = Inner(data=b"\xff\xfe", token="secret", api_key="key") + placeholder = "[Outer could not be serialized as JSON]" + + assert FunctionTool.parse_result(Outer(inner=inner))[0].text == placeholder + assert _parse_result_json([Outer(inner=inner), {"ok": 1}]) == [placeholder, {"ok": 1}] + + +@mark.usefixtures("pydantic_json_dump") +def test_parse_result_pydantic_model_with_reference_cycle_is_replaced(): + """A model with a reference cycle in a serialized field is replaced by a placeholder.""" + + class Node(BaseModel): + name: str + next: Any = None + api_key: str = Field(default="key", exclude=True) + + node = Node(name="a") + node.next = node + + assert FunctionTool.parse_result(node)[0].text == "[Node could not be serialized as JSON]" + + # region FunctionTool.parse_result with MCP TextContent-like objects