From 18470bd887642e6df1414e000db568612bacfcc5 Mon Sep 17 00:00:00 2001 From: sophia-ramsey Date: Tue, 10 Nov 2026 15:24:29 -0800 Subject: [PATCH 1/6] keep citations as annotation types --- .../_toolbox.py | 127 ++++++ .../foundry_hosting/tests/test_toolbox.py | 381 ++++++++++++++++++ 2 files changed, 508 insertions(+) diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py index d9a02501c56..dd1ec19bcb1 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py @@ -6,6 +6,7 @@ import inspect import logging import os +from collections import Counter from collections.abc import Callable, Mapping, Sequence from contextlib import _AsyncGeneratorContextManager # pyright: ignore[reportPrivateUsage] from contextvars import ContextVar @@ -14,6 +15,7 @@ import httpx2 as httpx from agent_framework import ( + Annotation, CachingSkillsSource, DeduplicatingSkillsSource, MCPSkillsSource, @@ -25,6 +27,7 @@ from agent_framework._telemetry import mark_feature_used from agent_framework.exceptions import ToolExecutionException from azure.ai.agentserver.core import get_request_context +from mcp import types as mcp_types from mcp.types import Tool from typing_extensions import override @@ -222,6 +225,61 @@ def _toolbox_approval_mode(item: Tool | Prompt) -> Literal["always_require", "ne return "never_require" +def _parse_toolbox_url_citation( + value: Any, text_utf16: bytes | None = None +) -> tuple[Annotation, tuple[str, str | None, int | None, int | None]] | None: + """Validate a Toolbox URL citation and convert valid UTF-16 spans to Python offsets.""" + if not isinstance(value, Mapping): + return None + raw = cast(Mapping[str, Any], value) + url = raw.get("url") + if raw.get("type") != "url_citation" or not isinstance(url, str) or not url.strip(): + return None + + raw_title = raw.get("title") + title = raw_title if isinstance(raw_title, str) else None + raw_start, raw_end = raw.get("start_index"), raw.get("end_index") + start = raw_start if type(raw_start) is int else None + end = raw_end if type(raw_end) is int else None + citation = Annotation(type="citation", url=url) + if title is not None: + citation["title"] = title + if text_utf16 is not None and start is not None and end is not None and 0 <= start <= end <= len(text_utf16) // 2: + # Toolbox offsets use UTF-16 code units; framework spans use Python text offsets. + boundaries = (start, end) + if all( + offset == 0 + or offset == len(text_utf16) // 2 + or not ( + 0xD800 <= int.from_bytes(text_utf16[(offset - 1) * 2 : offset * 2], "little") <= 0xDBFF + and 0xDC00 <= int.from_bytes(text_utf16[offset * 2 : (offset + 1) * 2], "little") <= 0xDFFF + ) + for offset in boundaries + ): + citation["annotated_regions"] = [ + { + "type": "text_span", + "start_index": len(text_utf16[: start * 2].decode("utf-16-le", errors="surrogatepass")), + "end_index": len(text_utf16[: end * 2].decode("utf-16-le", errors="surrogatepass")), + } + ] + return citation, (url, title, start, end) + + +def _toolbox_block_annotations(item: Any) -> list[Any]: + """Collect annotation metadata from an MCP content block and its embedded resource.""" + metadata: list[Any] = [getattr(item, "meta", None)] + if isinstance(item, mcp_types.EmbeddedResource): + metadata.append(item.resource.meta) + annotations: list[Any] = [] + for entry in metadata: + if isinstance(entry, Mapping): + values = cast(Mapping[str, Any], entry).get("annotations") + if isinstance(values, list): + annotations.extend(cast(list[Any], values)) + return annotations + + class FoundryToolbox(MCPStreamableHTTPTool): """A Foundry toolbox exposed as an MCP tool, with hosting wired in. @@ -235,6 +293,14 @@ class FoundryToolbox(MCPStreamableHTTPTool): - forwards the platform per-request call-id (``x-agent-foundry-call-id``) so the Foundry MCP proxy can resolve the caller context server-side. + By default, web-search results with URL citation metadata retain their content blocks + and expose the citations as framework annotations. Structured content remains + model-visible, and aggregate citations without an identified content block remain + attached to that structured content without spans. Citation spans refer to the + search result, not a later assistant answer. An explicit ``tool_result_content`` + or ``parse_tool_results`` option replaces this Toolbox-specific projection with + the selected MCP behavior. + Platform headers are resolved at each operation boundary before work crosses into the MCP connection's writer task. The complete effective header set is bound to the MCP session; when the caller context changes, the Toolbox @@ -363,6 +429,7 @@ def __init__( self._credential = credential self._token_scope = token_scope self._timeout = timeout + self._use_toolbox_result_projection = "tool_result_content" not in kwargs caller_header_provider = cast( Callable[[dict[str, Any]], dict[str, str]] | None, kwargs.pop("header_provider", None), @@ -396,6 +463,66 @@ def toolbox_header_provider(runtime_kwargs: dict[str, Any]) -> dict[str, str]: **kwargs, ) + @override + def _parse_tool_result_from_mcp(self, mcp_type: mcp_types.CallToolResult) -> list[Content]: + """Preserve Toolbox search text and project recognized citations onto the returned content.""" + if ( + not self._use_toolbox_result_projection + or self.tool_result_content != "structured_first" + or mcp_type.is_error + or not mcp_type.content + ): + return super()._parse_tool_result_from_mcp(mcp_type) + + structured: Any = mcp_type.structured_content + structured_mapping = cast(Mapping[str, Any], structured) if isinstance(structured, Mapping) else None + structured_values = structured_mapping.get("annotations") if structured_mapping is not None else None + structured_annotations = cast(list[Any], structured_values) if isinstance(structured_values, list) else [] + block_annotations = [_toolbox_block_annotations(item) for item in mcp_type.content] + if not any( + _parse_toolbox_url_citation(value) is not None + for value in [*structured_annotations, *(value for values in block_annotations for value in values)] + ): + return super()._parse_tool_result_from_mcp(mcp_type) + + result = super()._parse_tool_result_from_mcp(mcp_type.model_copy(update={"structured_content": None})) + text_items = [item for item in result if item.type == "text" and item.text is not None] + if not any(item.text for item in text_items): + return super()._parse_tool_result_from_mcp(mcp_type) + + # Match aggregate and block-owned occurrences without collapsing repeated citations. + block_citations: Counter[tuple[str, str | None, int | None, int | None]] = Counter() + for item, values in zip(result, block_annotations, strict=True): + text = item.text + text_utf16 = ( + text.encode("utf-16-le", errors="surrogatepass") if item.type == "text" and text is not None else None + ) + item_citations = list(item.annotations or []) + item_seen: set[tuple[str, str | None, int | None, int | None]] = set() + for value in values: + parsed = _parse_toolbox_url_citation(value, text_utf16) + if parsed is not None and parsed[1] not in item_seen: + item_citations.append(parsed[0]) + item_seen.add(parsed[1]) + block_citations[parsed[1]] += 1 + if item_citations: + item.annotations = item_citations + + unbound: list[Annotation] = [] + for value in structured_annotations: + parsed = _parse_toolbox_url_citation(value) + if parsed is not None: + if block_citations[parsed[1]]: + block_citations[parsed[1]] -= 1 + else: + unbound.append(parsed[0]) + if structured is not None: + structured_result = super()._parse_tool_result_from_mcp(mcp_type.model_copy(update={"content": []})) + if unbound: + structured_result[0].annotations = unbound + result.extend(structured_result) + return result + @override async def connect(self, *, reset: bool = False) -> None: """Connect to the toolbox and mark its first meaningful activation.""" diff --git a/python/packages/foundry_hosting/tests/test_toolbox.py b/python/packages/foundry_hosting/tests/test_toolbox.py index 6396df53df1..4d66462fd47 100644 --- a/python/packages/foundry_hosting/tests/test_toolbox.py +++ b/python/packages/foundry_hosting/tests/test_toolbox.py @@ -18,6 +18,8 @@ import pytest from agent_framework import ( Agent, + AgentSession, + Content, FunctionInvocationContext, MCPStreamableHTTPTool, Message, @@ -26,6 +28,7 @@ SupportsAgentRun, ) from agent_framework.exceptions import ToolException, ToolExecutionException +from agent_framework.openai import OpenAIChatClient from azure.ai.agentserver.core import ( FoundryAgentRequestContext, reset_request_context, @@ -52,6 +55,22 @@ ) +class _ToolResultServer(ToolboxApprovalServer): + """Offline Toolbox server whose ``tools/call`` response is configurable per test.""" + + def __init__(self) -> None: + super().__init__() + self.call_result: dict[str, Any] = {} + + async def handle(self, request: httpx.Request) -> httpx.Response: + if request.method == "POST": + payload = json.loads(request.content) + if payload.get("method") == "tools/call" and "id" in payload: + self.calls.append(payload["params"]) + return httpx.Response(200, json={"jsonrpc": "2.0", "id": payload["id"], "result": self.call_result}) + return await super().handle(request) + + class _StubAgent: """Minimal stand-in for a ``SupportsAgentRun`` used to build a source context.""" @@ -386,6 +405,368 @@ def test_init_forwards_additional_tool_arguments_and_parent_kwargs() -> None: assert toolbox._tool_extra_arg_names == {"search": {"thread"}} +@pytest.mark.parametrize( + "metadata_location", ["structured", "block", "resource", "block_and_structured", "resource_and_structured"] +) +async def test_web_search_preserves_prose_and_citation_metadata(metadata_location: str) -> None: + citation = { + "type": "url_citation", + "url": "https://example.test/source", + "title": "Source", + "start_index": 0, + "end_index": 4, + } + resource: dict[str, Any] = {"uri": "about:web-search", "text": "Mars is red."} + block: dict[str, Any] = {"type": "resource", "resource": resource} + if metadata_location.startswith("block"): + block["_meta"] = {"annotations": [citation]} + elif metadata_location.startswith("resource"): + resource["_meta"] = {"annotations": [citation]} + structured: dict[str, Any] = {"query": "Mars"} + if "structured" in metadata_location: + structured["annotations"] = [citation] + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = {"content": [block], "structuredContent": structured} + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert [item.text for item in result] == ["Mars is red.", json.dumps(structured)] + expected: dict[str, Any] = { + "type": "citation", + "url": "https://example.test/source", + "title": "Source", + } + if metadata_location != "structured": + expected["annotated_regions"] = [{"type": "text_span", "start_index": 0, "end_index": 4}] + citation_item = result[1] if metadata_location == "structured" else result[0] + assert citation_item.annotations == [expected] + + +async def test_web_search_keeps_aggregate_citations_unbound_when_content_has_multiple_blocks() -> None: + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [{"type": "text", "text": "First"}, {"type": "text", "text": "Second"}], + "structuredContent": { + "annotations": [ + { + "type": "url_citation", + "url": "https://example.test/source", + "start_index": 0, + "end_index": 5, + } + ] + }, + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert [item.text for item in result] == ["First", "Second", json.dumps(server.call_result["structuredContent"])] + assert result[0].annotations is None + assert result[1].annotations is None + assert result[2].annotations == [{"type": "citation", "url": "https://example.test/source"}] + + +async def test_web_search_keeps_block_citations_with_their_owning_text() -> None: + first = {"type": "url_citation", "url": "https://example.test/first", "start_index": 0, "end_index": 5} + second = {"type": "url_citation", "url": "https://example.test/second", "start_index": 0, "end_index": 6} + unbound = {"type": "url_citation", "url": "https://example.test/other", "start_index": 0, "end_index": 5} + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [ + {"type": "text", "text": "First", "_meta": {"annotations": [first]}}, + {"type": "text", "text": "Second", "_meta": {"annotations": [second]}}, + ], + "structuredContent": {"annotations": [first, second, unbound]}, + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert [item.text for item in result] == ["First", "Second", json.dumps(server.call_result["structuredContent"])] + assert result[0].annotations == [ + { + "type": "citation", + "url": "https://example.test/first", + "annotated_regions": [{"type": "text_span", "start_index": 0, "end_index": 5}], + } + ] + assert result[1].annotations == [ + { + "type": "citation", + "url": "https://example.test/second", + "annotated_regions": [{"type": "text_span", "start_index": 0, "end_index": 6}], + } + ] + assert result[2].annotations == [{"type": "citation", "url": "https://example.test/other"}] + + +async def test_web_search_keeps_repeated_aggregate_citations_not_matched_to_a_block() -> None: + citation = {"type": "url_citation", "url": "https://example.test/source", "start_index": 0, "end_index": 5} + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [ + {"type": "text", "text": "First", "_meta": {"annotations": [citation]}}, + {"type": "text", "text": "Second"}, + ], + "structuredContent": {"annotations": [citation, citation]}, + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert result[0].annotations == [ + { + "type": "citation", + "url": "https://example.test/source", + "annotated_regions": [{"type": "text_span", "start_index": 0, "end_index": 5}], + } + ] + assert result[1].annotations is None + assert result[2].annotations == [{"type": "citation", "url": "https://example.test/source"}] + + +async def test_web_search_keeps_citations_on_non_text_blocks_without_a_span() -> None: + citation = {"type": "url_citation", "url": "https://example.test/image", "start_index": 0, "end_index": 5} + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [ + {"type": "image", "data": "eA==", "mimeType": "image/png", "_meta": {"annotations": [citation]}}, + {"type": "text", "text": "Search prose"}, + ], + "structuredContent": {"annotations": [citation]}, + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert result[0].type == "data" + assert result[0].annotations == [{"type": "citation", "url": "https://example.test/image"}] + assert result[1].text == "Search prose" + assert result[2].text == json.dumps(server.call_result["structuredContent"]) + + +@pytest.mark.parametrize( + ("start", "end", "expected_region"), + [ + (1, 3, {"type": "text_span", "start_index": 1, "end_index": 2}), + (2, 3, None), + (True, 3, None), + (-1, 3, None), + (1, 6, None), + ], +) +async def test_web_search_ignores_invalid_utf16_citation_spans( + start: Any, end: Any, expected_region: dict[str, Any] | None +) -> None: + citation = {"type": "url_citation", "url": "https://example.test/source", "start_index": start, "end_index": end} + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [{"type": "text", "text": "A\U0001f30eBC", "_meta": {"annotations": [citation]}}], + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert result[0].annotations is not None + assert result[0].annotations[0].get("annotated_regions") == ( + [expected_region] if expected_region is not None else None + ) + if expected_region is not None: + assert result[0].text is not None + assert result[0].text[expected_region["start_index"] : expected_region["end_index"]] == "\U0001f30e" + + +async def test_web_search_preserves_unrecognized_structured_entries() -> None: + unrecognized = [ + {"type": "url_citation", "title": "missing URL"}, + {"type": "url_citation", "url": " "}, + {"type": "other", "value": 7}, + "invalid", + ] + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [{"type": "text", "text": "Search prose"}], + "structuredContent": { + "annotations": [{"type": "url_citation", "url": "https://example.test"}, *unrecognized], + "query": "Mars", + }, + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert result[0].text == "Search prose" + assert result[1].text is not None + assert result[1].annotations == [{"type": "citation", "url": "https://example.test"}] + assert json.loads(result[1].text) == server.call_result["structuredContent"] + + +@pytest.mark.parametrize("content", [[], [{"type": "text", "text": ""}]]) +async def test_web_search_without_prose_keeps_structured_result(content: list[dict[str, Any]]) -> None: + structured = {"annotations": [{"type": "url_citation", "url": "https://example.test"}]} + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = {"content": content, "structuredContent": structured} + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert result[0].text is not None + assert json.loads(result[0].text) == structured + + +@pytest.mark.parametrize( + ("mode", "expected"), + [ + ("structured_first", ['{"annotations": [{"type": "url_citation", "url": "https://example.test"}]}']), + ("content_first", ["Search prose"]), + ("content_only", ["Search prose"]), + ("structured_only", ['{"annotations": [{"type": "url_citation", "url": "https://example.test"}]}']), + ("both", ["Search prose", '{"annotations": [{"type": "url_citation", "url": "https://example.test"}]}']), + ], +) +async def test_explicit_mcp_content_mode_overrides_toolbox_projection(mode: str, expected: list[str]) -> None: + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [{"type": "text", "text": "Search prose"}], + "structuredContent": {"annotations": [{"type": "url_citation", "url": "https://example.test"}]}, + } + async with toolbox_connection(server, tool_result_content=mode) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert [item.text for item in result] == expected + assert all(item.annotations is None for item in result) + + +async def test_custom_mcp_result_parser_overrides_toolbox_projection() -> None: + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [{"type": "text", "text": "Search prose"}], + "structuredContent": {"annotations": [{"type": "url_citation", "url": "https://example.test"}]}, + } + async with toolbox_connection(server, parse_tool_results=lambda _: "Custom result") as toolbox: + assert await toolbox.call_tool("learn___microsoft_docs_search") == "Custom result" + + +async def test_non_web_search_result_keeps_default_mcp_selection() -> None: + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [{"type": "text", "text": "Summary"}], + "structuredContent": {"value": 42}, + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert [item.text for item in result] == ['{"value": 42}'] + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_web_search_provenance_survives_function_invocation(stream: bool) -> None: + citation = { + "type": "url_citation", + "url": "https://example.test/source", + "start_index": 0, + "end_index": 4, + } + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [ + { + "type": "resource", + "resource": { + "uri": "about:web-search", + "text": "Mars is red.", + "_meta": {"annotations": [citation]}, + }, + } + ], + "structuredContent": {"annotations": [citation]}, + "_meta": {"source": "server"}, + } + client = ToolboxApprovalClient("learn___microsoft_docs_search") + async with toolbox_connection(server) as toolbox: + agent = Agent(client=client, tools=toolbox) + session = agent.create_session() + if stream: + response_stream = agent.run("Search", session=session, stream=True) + updates = [update async for update in response_stream] + response = await response_stream.get_final_response() + assert sum(content.type == "function_result" for update in updates for content in update.contents) == 1 + else: + response = await agent.run("Search", session=session) + + restored_session = AgentSession.from_dict(json.loads(json.dumps(session.to_dict()))) + if stream: + continued_stream = agent.run("Continue", session=restored_session, stream=True) + _ = [update async for update in continued_stream] + await continued_stream.get_final_response() + else: + await agent.run("Continue", session=restored_session) + + results = [ + content for message in response.messages for content in message.contents if content.type == "function_result" + ] + assert len(results) == 1 + assert results[0].call_id == "call-search" + assert results[0].items is not None + assert results[0].items[0].text == "Mars is red." + assert results[0].items[0].annotations == [ + { + "type": "citation", + "url": "https://example.test/source", + "annotated_regions": [{"type": "text_span", "start_index": 0, "end_index": 4}], + } + ] + assert results[0].result == "Mars is red.\n" + json.dumps(server.call_result["structuredContent"]) + model_results = [ + content for message in client.calls[1] for content in message.contents if content.type == "function_result" + ] + assert len(model_results) == 1 + assert model_results[0].call_id == "call-search" + assert model_results[0].items is not None + assert model_results[0].items[0].annotations == results[0].items[0].annotations + replayed = [content for message in client.calls[2] for content in message.contents] + assert sum(content.type == "function_call" and content.call_id == "call-search" for content in replayed) == 1 + replayed_results = [content for content in replayed if content.type == "function_result"] + assert len(replayed_results) == 1 + assert replayed_results[0].call_id == "call-search" + assert replayed_results[0].items is not None + assert replayed_results[0].items[0].annotations == results[0].items[0].annotations + provider = OpenAIChatClient(model="test-model", api_key="test-key") + try: + assert provider._prepare_content_for_openai("tool", model_results[0]) == { + "type": "function_call_output", + "call_id": "call-search", + "output": results[0].result, + } + finally: + await provider.client.close() + assert results[0].items[0].additional_properties["_meta"] == {"source": "server"} + restored = Content.from_dict(results[0].to_dict()) + assert restored.items is not None + assert restored.items[0].annotations == results[0].items[0].annotations + host_payload = restored.additional_properties["_mcp_tool_result_host_payload"] + assert host_payload["content"] == server.call_result["content"] + assert host_payload["structuredContent"] == server.call_result["structuredContent"] + assert host_payload["_meta"] == {"source": "server"} + assert len(server.calls) == 1 + + def test_toolbox_owns_feature_index_53() -> None: assert FeatureIndex.FOUNDRY_TOOLBOX == 53 From 30a0084945873669c490c75ab7d37ab600e9b73a Mon Sep 17 00:00:00 2001 From: sophia-ramsey Date: Wed, 11 Nov 2026 15:35:27 -0800 Subject: [PATCH 2/6] use all retrieved sources for citations --- .../_responses.py | 112 +++++++++- .../foundry_hosting/tests/test_responses.py | 192 ++++++++++++++++++ 2 files changed, 293 insertions(+), 11 deletions(-) diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py index f9853ed0548..be3dabebd40 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py @@ -152,6 +152,7 @@ _HOSTED_PROVIDER_OUTPUT_COUNT_KEY = "_foundry_provider_output_count" _HOSTED_PROVIDER_USAGE_KEY = "_foundry_provider_usage" _PENDING_CONTAINER_FILE_CITATIONS_KEY = "_foundry_pending_container_file_citations" +_PENDING_URL_CITATIONS_KEY = "_foundry_pending_url_citations" _CONTAINER_FILE_CITATIONS_KEY = "container_file_citations" _PUBLIC_STREAM_FAILURE_MESSAGE = "An internal server error occurred." @@ -282,6 +283,33 @@ def _container_file_citations_from_function_result(content: Content) -> list[_Co return citations +def _url_citations_from_function_result(content: Content) -> dict[str, str]: + """Collect the web sources cited by a function result, keyed by URL in first-seen order with their titles.""" + candidates: list[Any] = list(content.annotations or []) + for item in content.items or []: + candidates.extend(item.annotations or []) + + citations: dict[str, str] = {} + for annotation in candidates: + if not isinstance(annotation, Mapping): + continue + annotation_mapping = cast(Mapping[str, Any], annotation) + if annotation_mapping.get("type") != "citation" or annotation_mapping.get("file_id"): + continue + url = annotation_mapping.get("url") + if not isinstance(url, str) or url in citations: + continue + try: + scheme = urlparse(url).scheme.lower() + except ValueError: + continue + if scheme not in {"http", "https"}: + continue + title = annotation_mapping.get("title") + citations[url] = title if isinstance(title, str) else "" + return citations + + def _annotations_to_output_text( annotations: Sequence[FrameworkAnnotation] | None, ) -> list[ResponsesAnnotation]: @@ -1528,7 +1556,7 @@ async def _handle_prepared_response( # below so its partial output is not misreported as a failure. return - for event in tracker.close(): + for event in tracker.close(final_answer=True): yield event if cancellation_signal.is_set() and context.client_cancelled: @@ -1539,7 +1567,7 @@ async def _handle_prepared_response( # selecting the terminal event. Same ``client_cancelled`` gate as above. return - tracker.discard_pending_container_file_citations() + tracker.discard_pending_citations() incomplete_reason = tracker.incomplete_reason if tracker.oauth_consent_requested or incomplete_reason is not None: yield response_event_stream.emit_incomplete(reason=incomplete_reason, usage=tracker.usage) @@ -1918,7 +1946,9 @@ async def _handle_inner_agent( yield event if context.shutdown.is_set() or cancellation_signal.is_set(): continue - for event in tracker.close(): + # Unfinished provider outputs end at a function result, so text is still + # active here only when it ends the finished answer. + for event in tracker.close(final_answer=True): yield event private_metadata = _hosted_internal_metadata(response_event_stream) private_metadata[_HOSTED_PROVIDER_OUTPUT_COUNT_KEY] = output_count @@ -2514,7 +2544,7 @@ def _emit_failure( yield from tracker.close() except Exception as close_error: _log_response_failure("failed tracker cleanup", response_id, close_error) - tracker.discard_pending_container_file_citations() + tracker.discard_pending_citations() if public_message is not None: message = public_message elif isinstance(ex, _PublicResponseFailure): @@ -2576,6 +2606,7 @@ def __init__( self._outstanding_computer_calls: set[str] = set() self._oauth_consent_requests: set[tuple[str, str]] = set() self._pending_container_file_citations: dict[str, _ContainerFileCitation] = {} + self._pending_url_citations: dict[str, str] = {} self._text_annotations: list[ResponsesAnnotation] = [] self._message_text_annotations: dict[int, list[ResponsesAnnotation]] = {} # Set when an agent update reports the model stopped early (content filter, token @@ -2615,6 +2646,21 @@ def __init__( file_id=file_id, filename=filename, ) + persisted_url_citations = _hosted_internal_metadata(stream).get(_PENDING_URL_CITATIONS_KEY) + if persisted_url_citations is not None: + if not isinstance(persisted_url_citations, Sequence) or isinstance( + persisted_url_citations, (str, bytes, bytearray) + ): + raise RuntimeError("The persisted URL citations are invalid.") + for raw_url_citation in cast(Sequence[Any], persisted_url_citations): + if not isinstance(raw_url_citation, Mapping): + raise RuntimeError("The persisted URL citations are invalid.") + url_citation = cast(Mapping[str, Any], raw_url_citation) + url = url_citation.get("url") + title = url_citation.get("title") + if not isinstance(url, str) or not url or not isinstance(title, str): + raise RuntimeError("The persisted URL citations are invalid.") + self._pending_url_citations.setdefault(url, title) for item in stream.response.get("output", []): if not isinstance(item, Mapping): continue @@ -2810,6 +2856,11 @@ async def handle( for citation in citations: self._pending_container_file_citations[citation.filename] = citation self._persist_pending_container_file_citations() + url_citations = _url_citations_from_function_result(content) + if url_citations: + for url, title in url_citations.items(): + self._pending_url_citations.setdefault(url, title) + self._persist_pending_url_citations() async for event in self._stream.output_item_function_call_output( content.call_id, # type: ignore[arg-type] _json_safe_to_str(content.result), @@ -3063,14 +3114,20 @@ async def handle( # response stream isn't broken by one unsupported content item. logger.warning(f"Content type '{content.type}' is not supported yet. This is usually safe to ignore.") - def close(self) -> Generator[ResponseStreamEvent]: - """Flush any remaining active builder without discarding recoverable state.""" - yield from self._close() + def close(self, *, final_answer: bool = False) -> Generator[ResponseStreamEvent]: + """Flush any remaining active builder without discarding recoverable state. - def discard_pending_container_file_citations(self) -> None: + Set ``final_answer`` only when the active text ends the agent's answer, so pending tool-result + sources attach there instead of to interim text. + """ + yield from self._close(attach_sources=final_answer) + + def discard_pending_citations(self) -> None: """Discard unmatched citations immediately before a terminal response event.""" self._pending_container_file_citations.clear() self._persist_pending_container_file_citations() + self._pending_url_citations.clear() + self._persist_pending_url_citations() # -- Private open/close helpers -- @@ -3159,9 +3216,9 @@ def _open_mcp_call(self, content: Content) -> Generator[ResponseStreamEvent]: self._active_id = content.call_id or f"{content.server_name or 'default'}::{content.tool_name}" yield self._mcp_builder.emit_added() - def _close(self) -> Generator[ResponseStreamEvent]: + def _close(self, *, attach_sources: bool = False) -> Generator[ResponseStreamEvent]: if self._active_type in {"text", "refusal"}: - yield from self._close_message_content() + yield from self._close_message_content(attach_sources=attach_sources) if self._message_item is not None: message_done = self._message_item.emit_done() if self._message_text_annotations: @@ -3219,13 +3276,15 @@ def _close(self) -> Generator[ResponseStreamEvent]: self._active_message_id = None self._accumulated.clear() - def _close_message_content(self) -> Generator[ResponseStreamEvent]: + def _close_message_content(self, *, attach_sources: bool = False) -> Generator[ResponseStreamEvent]: accumulated = "".join(self._accumulated) if self._active_type == "text" and self._text_content is not None: annotations = list(self._text_annotations) for annotation in self._container_file_annotations_for_text(accumulated): if annotation not in annotations: annotations.append(annotation) + if attach_sources: + annotations.extend(self._source_url_annotations_for_text(accumulated, annotations)) yield self._text_content.emit_text_done(accumulated) for annotation in annotations: yield self._text_content.emit_annotation_added(annotation) @@ -3291,6 +3350,37 @@ def _container_file_annotations_for_text(self, text: str) -> list[ContainerFileC self._persist_pending_container_file_citations() return annotations + def _source_url_annotations_for_text( + self, + text: str, + existing: Sequence[ResponsesAnnotation], + ) -> list[UrlCitationBody]: + """Cite pending tool-result sources at the end of the final answer text, without claiming a supported span.""" + if not text or not self._pending_url_citations: + return [] + cited_urls = { + cast(Mapping[str, Any], annotation).get("url") + for annotation in existing + if cast(Mapping[str, Any], annotation).get("type") == "url_citation" + } + end_index = len(text) + annotations = [ + UrlCitationBody(type="url_citation", url=url, title=title, start_index=end_index, end_index=end_index) + for url, title in self._pending_url_citations.items() + if url not in cited_urls + ] + self._pending_url_citations.clear() + self._persist_pending_url_citations() + return annotations + + def _persist_pending_url_citations(self) -> None: + if not self._pending_url_citations: + _hosted_internal_metadata(self._stream).pop(_PENDING_URL_CITATIONS_KEY, None) + return + _hosted_internal_metadata(self._stream)[_PENDING_URL_CITATIONS_KEY] = [ + {"url": url, "title": title} for url, title in self._pending_url_citations.items() + ] + def _persist_pending_container_file_citations(self) -> None: if not self._pending_container_file_citations: _hosted_internal_metadata(self._stream).pop(_PENDING_CONTAINER_FILE_CITATIONS_KEY, None) diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index 6022b6c9467..9565c6e6e5c 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -247,6 +247,51 @@ def _expected_container_file_citation( } +_PYTHON_SOURCE_URL = "https://www.python.org/downloads/release/python-31315/" +_DOCS_SOURCE_URL = "https://docs.python.org/3/whatsnew/3.13.html" + + +def _web_search_result(call_id: str) -> Content: + """Build a search tool result whose citations repeat a URL and include ignored non-web citations.""" + return Content.from_function_result( + call_id, + result=[ + Content.from_text( + "Python 3.13.15 is the latest release.", + annotations=[_url_citation(), Annotation(type="citation", url=_DOCS_SOURCE_URL, title="What's New")], + ), + Content.from_text( + '{"annotations": []}', + annotations=[ + Annotation(type="citation", url=_PYTHON_SOURCE_URL, title="Duplicate title"), + Annotation(type="citation", url="javascript:alert(1)", title="Unsafe"), + Annotation(type="citation", url="notes.txt", file_id="file_123", title="File"), + ], + ), + ], + ) + + +def _expected_source_citations(text: str) -> list[dict[str, Any]]: + """Return the zero-length end-of-text citations expected for ``_web_search_result``.""" + return [ + _expected_url_citation(start_index=len(text), end_index=len(text)), + _expected_url_citation(url=_DOCS_SOURCE_URL, title="What's New", start_index=len(text), end_index=len(text)), + ] + + +def _web_search_citation_updates(answer: str) -> list[AgentResponseUpdate]: + return [ + AgentResponseUpdate( + contents=[Content.from_function_call("call_1", "web_search", arguments={})], + role="assistant", + message_id="msg_call", + ), + AgentResponseUpdate(contents=[_web_search_result("call_1")], role="tool", message_id="msg_result"), + AgentResponseUpdate(contents=[Content.from_text(answer)], role="assistant", message_id="msg_answer"), + ] + + def _url_citation( *, url: str = "https://www.python.org/downloads/release/python-31315/", @@ -2517,6 +2562,56 @@ async def run(messages: Any = None, *, session: AgentSession, **kwargs: Any) -> assert output_done["item"] == message assert response.to_dict() == original_response + async def test_provider_background_attaches_tool_result_sources_to_final_answer( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + answer = "Python 3.13.15 is out." + response = AgentResponse( + messages=[ + Message( + role="assistant", + contents=[ + Content.from_text("Let me search."), + Content.from_function_call("call_1", "web_search", arguments={}), + ], + ), + Message(role="tool", contents=[_web_search_result("call_1")]), + Message(role="assistant", contents=[Content.from_text(answer)]), + ], + finish_reason="stop", + ) + agent = Agent(client=_ServiceStorageRecordingClient()) + + async def run(messages: Any = None, *, session: AgentSession, **kwargs: Any) -> AgentResponse: + del messages, kwargs + session.service_session_id = "private-service-conversation" + return response + + monkeypatch.setattr(agent, "run", MagicMock(side_effect=run)) + server = _make_server( + agent, + session_store=SessionStore(), + history_source="service", + background_source="provider", + options=ResponsesServerOptions(resilient_background=True), + response_store=FileResponseStore(storage_dir=tmp_path), + ) + request = CreateResponse(input="hello", store=True, background=True) + context = ResponseContext(response_id="outer-response", mode_flags=MagicMock()) + + with patch.object(ResponseContext, "get_input_items", new=AsyncMock(return_value=[])): + events = [ + event + async for event in server._handle_response(request, context, asyncio.Event()) + if isinstance(event, Mapping) + ] + + completed = next(event["response"] for event in events if event["type"] == "response.completed") + messages = [item for item in completed["output"] if item["type"] == "message"] + assert [message["content"][0]["text"] for message in messages] == ["Let me search.", answer] + assert messages[0]["content"][0]["annotations"] == [] + assert messages[1]["content"][0]["annotations"] == _expected_source_citations(answer) + async def test_provider_background_keeps_private_token_under_outer_response( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -4485,6 +4580,56 @@ async def test_framework_url_and_file_citations_are_added_to_output_text(self) - _expected_file_citation(), ] + async def test_function_result_url_citations_wait_for_final_output_text(self) -> None: + answer = "Python 3.13.15 is out." + releases_url = "https://www.python.org/downloads/" + second_search = Content.from_function_result( + "call_2", + result=[ + Content.from_text( + "Release list.", annotations=[Annotation(type="citation", url=releases_url, title="Releases")] + ) + ], + ) + updates = [ + *_web_search_citation_updates("Let me search more.")[:-1], + AgentResponseUpdate( + contents=[Content.from_text("Let me search more.")], role="assistant", message_id="msg_interim" + ), + AgentResponseUpdate( + contents=[Content.from_function_call("call_2", "web_search", arguments={})], + role="assistant", + message_id="msg_call_2", + ), + AgentResponseUpdate(contents=[second_search], role="tool", message_id="msg_result_2"), + AgentResponseUpdate( + contents=[Content.from_text(answer, annotations=[_url_citation()])], + role="assistant", + message_id="msg_answer", + ), + ] + + resp = await _post(_make_server(_make_agent(stream_updates=updates)), stream=False) + + assert resp.status_code == 200 + output = resp.json()["output"] + assert [item["type"] for item in output] == [ + "function_call", + "function_call_output", + "message", + "function_call", + "function_call_output", + "message", + ] + assert output[2]["content"][0]["annotations"] == [] + assert output[5]["content"][0]["annotations"] == [ + _expected_url_citation(), + _expected_url_citation( + url=_DOCS_SOURCE_URL, title="What's New", start_index=len(answer), end_index=len(answer) + ), + _expected_url_citation(url=releases_url, title="Releases", start_index=len(answer), end_index=len(answer)), + ] + async def test_latest_pending_container_file_citation_replaces_same_filename(self) -> None: agent = _make_agent( stream_updates=[ @@ -5362,6 +5507,53 @@ async def test_framework_url_citation_is_consistent_across_stream_events(self) - final_message = next(item for item in completed["output"] if item["type"] == "message") assert final_message["content"][0]["annotations"] == [expected_annotation] + async def test_function_result_url_citations_are_consistent_across_stream_events(self) -> None: + answer = "Python 3.13.15 is out." + resp = await _post( + _make_server(_make_agent(stream_updates=_web_search_citation_updates(answer))), + stream=True, + ) + + assert resp.status_code == 200 + events = _parse_sse_events(resp.text) + expected = _expected_source_citations(answer) + annotation_events = [event for event in events if event["event"] == "response.output_text.annotation.added"] + assert [event["data"]["annotation"] for event in annotation_events] == expected + content_done = next(event for event in events if event["event"] == "response.content_part.done") + assert content_done["data"]["part"]["annotations"] == expected + completed = next(event["data"]["response"] for event in events if event["event"] == "response.completed") + final_message = next(item for item in completed["output"] if item["type"] == "message") + assert final_message["content"][0]["annotations"] == expected + + async def test_pending_url_citations_survive_recovery_and_are_discarded_at_completion(self) -> None: + stream = ResponseEventStream(response_id="resp_url_citations") + stream.emit_created() + stream.emit_in_progress() + tracker = _OutputItemTracker(stream) + _ = [event async for event in tracker.handle(Content.from_function_call("call_1", "web_search", arguments={}))] + _ = [event async for event in tracker.handle(_web_search_result("call_1"))] + _ = list(tracker.close()) + _ = stream.checkpoint() + + recovered = _OutputItemTracker(stream) + interim = [event async for event in recovered.handle(Content.from_text("Checking."), message_id="msg_interim")] + interim.extend(recovered.close()) + assert not any(event["type"] == "response.output_text.annotation.added" for event in interim) + + events = [event async for event in recovered.handle(Content.from_text("Done."), message_id="msg_final")] + events.extend(recovered.close(final_answer=True)) + annotations = [ + event["annotation"] for event in events if event["type"] == "response.output_text.annotation.added" + ] + assert annotations == _expected_source_citations("Done.") + + _ = [event async for event in recovered.handle(_web_search_result("call_2"))] + recovered.discard_pending_citations() + discarded = _OutputItemTracker(stream) + events = [event async for event in discarded.handle(Content.from_text("Later."), message_id="msg_later")] + events.extend(discarded.close(final_answer=True)) + assert not any(event["type"] == "response.output_text.annotation.added" for event in events) + async def test_pending_container_file_citation_survives_tracker_recovery(self) -> None: stream = ResponseEventStream(response_id="resp_container_file") stream.emit_created() From bd305cd2d728bbffd9d87ae75571f816dcae0aa3 Mon Sep 17 00:00:00 2001 From: sophia-ramsey Date: Thu, 12 Nov 2026 15:36:29 -0800 Subject: [PATCH 3/6] extend citation fix to core --- python/packages/core/agent_framework/_mcp.py | 129 ++++++++++++++++- python/packages/core/tests/core/test_mcp.py | 45 ++++++ python/packages/foundry_hosting/README.md | 13 ++ .../_toolbox.py | 132 +----------------- .../foundry_hosting/tests/test_toolbox.py | 3 +- 5 files changed, 192 insertions(+), 130 deletions(-) diff --git a/python/packages/core/agent_framework/_mcp.py b/python/packages/core/agent_framework/_mcp.py index 41e37a0d9ff..895c307f718 100644 --- a/python/packages/core/agent_framework/_mcp.py +++ b/python/packages/core/agent_framework/_mcp.py @@ -46,6 +46,7 @@ _FunctionResultPayloadBudget, # pyright: ignore[reportPrivateUsage] ) from ._types import ( + Annotation, ChatOptions, Content, Message, @@ -194,7 +195,10 @@ class MCPSpecificApproval(TypedDict, total=False): MCP servers disagree on whether the two fields are duplicates or complementary (#7866). Callers pick an explicit policy: -- ``structured_first`` (default): use ``structuredContent`` when present, else ``content`` +- ``structured_first`` (default): use ``structuredContent`` when present, else ``content``. When the result + has text content and ``url_citation`` entries in ``structuredContent.annotations`` or a content block's + ``_meta.annotations`` (for example, Foundry Toolbox web search), the text content is kept before + ``structuredContent`` and the citations are attached as framework annotations. - ``content_first``: use ``content`` when non-empty, else ``structuredContent`` - ``content_only``: ignore ``structuredContent`` for the model-visible result - ``structured_only``: ignore ``content`` for the model-visible result @@ -204,6 +208,124 @@ class MCPSpecificApproval(TypedDict, total=False): Custom ``parse_tool_results`` overrides this policy entirely. """ +_MCPCitationKey: TypeAlias = tuple[str, str | None, int | None, int | None] + + +def _parse_mcp_url_citation(value: Any, text_utf16: bytes | None = None) -> tuple[Annotation, _MCPCitationKey] | None: + """Validate an MCP ``url_citation`` entry and convert valid UTF-16 spans to Python text offsets.""" + if not isinstance(value, Mapping): + return None + raw = cast(Mapping[str, Any], value) + url = raw.get("url") + if raw.get("type") != "url_citation" or not isinstance(url, str) or not url.strip(): + return None + + raw_title = raw.get("title") + title = raw_title if isinstance(raw_title, str) else None + raw_start, raw_end = raw.get("start_index"), raw.get("end_index") + start = raw_start if type(raw_start) is int else None + end = raw_end if type(raw_end) is int else None + citation = Annotation(type="citation", url=url) + if title is not None: + citation["title"] = title + if text_utf16 is not None and start is not None and end is not None and 0 <= start <= end <= len(text_utf16) // 2: + # Citation offsets use UTF-16 code units; framework spans use Python text offsets. + boundaries = (start, end) + if all( + offset == 0 + or offset == len(text_utf16) // 2 + or not ( + 0xD800 <= int.from_bytes(text_utf16[(offset - 1) * 2 : offset * 2], "little") <= 0xDBFF + and 0xDC00 <= int.from_bytes(text_utf16[offset * 2 : (offset + 1) * 2], "little") <= 0xDFFF + ) + for offset in boundaries + ): + citation["annotated_regions"] = [ + { + "type": "text_span", + "start_index": len(text_utf16[: start * 2].decode("utf-16-le", errors="surrogatepass")), + "end_index": len(text_utf16[: end * 2].decode("utf-16-le", errors="surrogatepass")), + } + ] + return citation, (url, title, start, end) + + +def _mcp_block_annotations(item: Any) -> list[Any]: + """Collect annotation metadata from an MCP content block and its embedded resource.""" + metadata: list[Any] = [getattr(item, "meta", None)] + resource = getattr(item, "resource", None) + if resource is not None: + metadata.append(getattr(resource, "meta", None)) + annotations: list[Any] = [] + for entry in metadata: + if isinstance(entry, Mapping): + values = cast(Mapping[str, Any], entry).get("annotations") + if isinstance(values, list): + annotations.extend(cast(list[Any], values)) + return annotations + + +def _project_mcp_url_citations( + mcp_type: types.CallToolResult, + content_items: list[Content], + structured_block: Content | None, +) -> list[Content] | None: + """Keep cited text content before structured content and attach its URL citations as annotations. + + Returns ``None`` when the result is an error, has no recognized ``url_citation`` entries, or has no text content. + ``content_items`` must hold exactly one parsed item per MCP content block. + """ + if mcp_type.is_error or not mcp_type.content: + return None + structured: Any = mcp_type.structured_content + structured_values = ( + cast(Mapping[str, Any], structured).get("annotations") if isinstance(structured, Mapping) else None + ) + structured_annotations = cast(list[Any], structured_values) if isinstance(structured_values, list) else [] + block_annotations = [_mcp_block_annotations(item) for item in mcp_type.content] + if not any( + _parse_mcp_url_citation(value) is not None + for value in [*structured_annotations, *(value for values in block_annotations for value in values)] + ): + return None + if not any(item.type == "text" and item.text for item in content_items): + return None + + # Match aggregate and block-owned occurrences without collapsing repeated citations. + block_citations: dict[_MCPCitationKey, int] = {} + for item, values in zip(content_items, block_annotations, strict=True): + text_utf16 = ( + item.text.encode("utf-16-le", errors="surrogatepass") + if item.type == "text" and item.text is not None + else None + ) + item_citations = list(item.annotations or []) + item_seen: set[_MCPCitationKey] = set() + for value in values: + parsed = _parse_mcp_url_citation(value, text_utf16) + if parsed is not None and parsed[1] not in item_seen: + item_citations.append(parsed[0]) + item_seen.add(parsed[1]) + block_citations[parsed[1]] = block_citations.get(parsed[1], 0) + 1 + if item_citations: + item.annotations = item_citations + + unbound: list[Annotation] = [] + for value in structured_annotations: + parsed = _parse_mcp_url_citation(value) + if parsed is None: + continue + if block_citations.get(parsed[1]): + block_citations[parsed[1]] -= 1 + else: + unbound.append(parsed[0]) + if structured_block is None: + return content_items + if unbound: + structured_block.annotations = [*(structured_block.annotations or []), *unbound] + return [*content_items, structured_block] + + _MCP_REMOTE_NAME_KEY = "_mcp_remote_name" _MCP_IS_TOOL_KEY = "_mcp_is_tool" _MCP_NORMALIZED_NAME_KEY = "_mcp_normalized_name" @@ -1521,7 +1643,10 @@ def _parse_tool_result_from_mcp( if not result and structured_block is not None: result.append(structured_block) elif mode == "structured_first": - if structured_block is not None: + cited = _project_mcp_url_citations(mcp_type, result, structured_block) + if cited is not None: + result = cited + elif structured_block is not None: result = [structured_block] else: # pragma: no cover - validated in __init__ raise ValueError(f"Unknown tool_result_content mode: {mode!r}") diff --git a/python/packages/core/tests/core/test_mcp.py b/python/packages/core/tests/core/test_mcp.py index 9362851adb1..e6d8b7c31bd 100644 --- a/python/packages/core/tests/core/test_mcp.py +++ b/python/packages/core/tests/core/test_mcp.py @@ -1412,6 +1412,51 @@ def test_parse_tool_result_content_modes_for_complementary_and_duplicate_payload assert content_only._parse_tool_result_from_mcp(empty)[0].text == "null" +def test_structured_first_keeps_cited_content_and_attaches_url_citations(): + """Default MCP parsing keeps search text when url_citation metadata is present (for example, Foundry Toolbox).""" + block_citation = { + "type": "url_citation", + "url": "https://example.test/block", + "title": "Block", + "start_index": 0, + "end_index": 4, + } + aggregate_citation = {"type": "url_citation", "url": "https://example.test/aggregate"} + structured = {"annotations": [block_citation, aggregate_citation, {"type": "other"}]} + mcp_result = types.CallToolResult( + content=[types.TextContent(type="text", text="Mars is red.", _meta={"annotations": [block_citation]})], + structured_content=structured, + _meta={"source": "server"}, + ) + + result = _HELPER_MCP_TOOL._parse_tool_result_from_mcp(mcp_result) + + assert [item.text for item in result] == ["Mars is red.", json.dumps(structured)] + assert result[0].annotations == [ + { + "type": "citation", + "url": "https://example.test/block", + "title": "Block", + "annotated_regions": [{"type": "text_span", "start_index": 0, "end_index": 4}], + } + ] + assert result[1].annotations == [{"type": "citation", "url": "https://example.test/aggregate"}] + assert all(item.additional_properties["_meta"] == {"source": "server"} for item in result) + + +def test_structured_first_does_not_attach_url_citations_to_error_results(): + mcp_result = types.CallToolResult( + content=[types.TextContent(type="text", text="Search failed")], + structured_content={"annotations": [{"type": "url_citation", "url": "https://example.test"}]}, + is_error=True, + ) + + result = _HELPER_MCP_TOOL._parse_tool_result_from_mcp(mcp_result) + + assert len(result) == 1 + assert result[0].annotations is None + + async def test_generated_mcp_tool_preserves_complete_host_payload_once() -> None: """The generated FunctionTool path retains one complete, persistent Host payload.""" file_meta = { diff --git a/python/packages/foundry_hosting/README.md b/python/packages/foundry_hosting/README.md index 068188eb61f..6bda7c4fe34 100644 --- a/python/packages/foundry_hosting/README.md +++ b/python/packages/foundry_hosting/README.md @@ -25,6 +25,13 @@ supplied through their configured Toolbox. The private metadata parser is intend Toolbox; reusing it elsewhere provides no security or compatibility guarantees. It is not a general-purpose MCP approval policy. +## Toolbox search citations + +`FoundryToolbox` and generic MCP tools use the default `structured_first` result handling for Toolbox search results. +When a result contains text content and `url_citation` annotations, the text stays visible to the model before the +structured JSON, and the citations are attached to the tool result. Another `tool_result_content` mode or a custom +`parse_tool_results` replaces this handling, and hosted answers then receive no Toolbox citations. + ## Agent instances and factories `ResponsesHostServer` and `InvocationsHostServer` accept an agent instance or a zero-argument callable through the @@ -311,6 +318,12 @@ Completed provider responses retain separate text content parts, keeping citatio their original part. Ordinary streaming deltas still merge within their existing text occurrence, without shifting citation offsets. +Web URL citations on function results, such as `FoundryToolbox` search results, are listed as sources on the +final answer text when the response completes. These citations do not identify which part of the answer each +source supports, so the host emits each one as a zero-length `url_citation` at the end of that text. Sources are +deduplicated by URL, limited to `http` and `https`, skipped when the answer already cites the same URL, and not +attached to interim text. Plain `agent.run()` responses do not include these sources. + ## State store ### Local persistence diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py index dd1ec19bcb1..eefa5c5ba2b 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py @@ -6,7 +6,6 @@ import inspect import logging import os -from collections import Counter from collections.abc import Callable, Mapping, Sequence from contextlib import _AsyncGeneratorContextManager # pyright: ignore[reportPrivateUsage] from contextvars import ContextVar @@ -15,7 +14,6 @@ import httpx2 as httpx from agent_framework import ( - Annotation, CachingSkillsSource, DeduplicatingSkillsSource, MCPSkillsSource, @@ -27,7 +25,6 @@ from agent_framework._telemetry import mark_feature_used from agent_framework.exceptions import ToolExecutionException from azure.ai.agentserver.core import get_request_context -from mcp import types as mcp_types from mcp.types import Tool from typing_extensions import override @@ -225,61 +222,6 @@ def _toolbox_approval_mode(item: Tool | Prompt) -> Literal["always_require", "ne return "never_require" -def _parse_toolbox_url_citation( - value: Any, text_utf16: bytes | None = None -) -> tuple[Annotation, tuple[str, str | None, int | None, int | None]] | None: - """Validate a Toolbox URL citation and convert valid UTF-16 spans to Python offsets.""" - if not isinstance(value, Mapping): - return None - raw = cast(Mapping[str, Any], value) - url = raw.get("url") - if raw.get("type") != "url_citation" or not isinstance(url, str) or not url.strip(): - return None - - raw_title = raw.get("title") - title = raw_title if isinstance(raw_title, str) else None - raw_start, raw_end = raw.get("start_index"), raw.get("end_index") - start = raw_start if type(raw_start) is int else None - end = raw_end if type(raw_end) is int else None - citation = Annotation(type="citation", url=url) - if title is not None: - citation["title"] = title - if text_utf16 is not None and start is not None and end is not None and 0 <= start <= end <= len(text_utf16) // 2: - # Toolbox offsets use UTF-16 code units; framework spans use Python text offsets. - boundaries = (start, end) - if all( - offset == 0 - or offset == len(text_utf16) // 2 - or not ( - 0xD800 <= int.from_bytes(text_utf16[(offset - 1) * 2 : offset * 2], "little") <= 0xDBFF - and 0xDC00 <= int.from_bytes(text_utf16[offset * 2 : (offset + 1) * 2], "little") <= 0xDFFF - ) - for offset in boundaries - ): - citation["annotated_regions"] = [ - { - "type": "text_span", - "start_index": len(text_utf16[: start * 2].decode("utf-16-le", errors="surrogatepass")), - "end_index": len(text_utf16[: end * 2].decode("utf-16-le", errors="surrogatepass")), - } - ] - return citation, (url, title, start, end) - - -def _toolbox_block_annotations(item: Any) -> list[Any]: - """Collect annotation metadata from an MCP content block and its embedded resource.""" - metadata: list[Any] = [getattr(item, "meta", None)] - if isinstance(item, mcp_types.EmbeddedResource): - metadata.append(item.resource.meta) - annotations: list[Any] = [] - for entry in metadata: - if isinstance(entry, Mapping): - values = cast(Mapping[str, Any], entry).get("annotations") - if isinstance(values, list): - annotations.extend(cast(list[Any], values)) - return annotations - - class FoundryToolbox(MCPStreamableHTTPTool): """A Foundry toolbox exposed as an MCP tool, with hosting wired in. @@ -293,13 +235,12 @@ class FoundryToolbox(MCPStreamableHTTPTool): - forwards the platform per-request call-id (``x-agent-foundry-call-id``) so the Foundry MCP proxy can resolve the caller context server-side. - By default, web-search results with URL citation metadata retain their content blocks - and expose the citations as framework annotations. Structured content remains - model-visible, and aggregate citations without an identified content block remain - attached to that structured content without spans. Citation spans refer to the - search result, not a later assistant answer. An explicit ``tool_result_content`` - or ``parse_tool_results`` option replaces this Toolbox-specific projection with - the selected MCP behavior. + Web-search results with URL citation metadata use the core MCP ``structured_first`` + handling: content blocks stay model-visible before the structured content, and the + citations become framework annotations. Aggregate citations without an identified + content block remain attached to the structured content without spans. Citation spans + refer to the search result, not a later assistant answer. Another ``tool_result_content`` + mode or a custom ``parse_tool_results`` replaces this handling. Platform headers are resolved at each operation boundary before work crosses into the MCP connection's writer task. The complete effective header set is @@ -429,7 +370,6 @@ def __init__( self._credential = credential self._token_scope = token_scope self._timeout = timeout - self._use_toolbox_result_projection = "tool_result_content" not in kwargs caller_header_provider = cast( Callable[[dict[str, Any]], dict[str, str]] | None, kwargs.pop("header_provider", None), @@ -463,66 +403,6 @@ def toolbox_header_provider(runtime_kwargs: dict[str, Any]) -> dict[str, str]: **kwargs, ) - @override - def _parse_tool_result_from_mcp(self, mcp_type: mcp_types.CallToolResult) -> list[Content]: - """Preserve Toolbox search text and project recognized citations onto the returned content.""" - if ( - not self._use_toolbox_result_projection - or self.tool_result_content != "structured_first" - or mcp_type.is_error - or not mcp_type.content - ): - return super()._parse_tool_result_from_mcp(mcp_type) - - structured: Any = mcp_type.structured_content - structured_mapping = cast(Mapping[str, Any], structured) if isinstance(structured, Mapping) else None - structured_values = structured_mapping.get("annotations") if structured_mapping is not None else None - structured_annotations = cast(list[Any], structured_values) if isinstance(structured_values, list) else [] - block_annotations = [_toolbox_block_annotations(item) for item in mcp_type.content] - if not any( - _parse_toolbox_url_citation(value) is not None - for value in [*structured_annotations, *(value for values in block_annotations for value in values)] - ): - return super()._parse_tool_result_from_mcp(mcp_type) - - result = super()._parse_tool_result_from_mcp(mcp_type.model_copy(update={"structured_content": None})) - text_items = [item for item in result if item.type == "text" and item.text is not None] - if not any(item.text for item in text_items): - return super()._parse_tool_result_from_mcp(mcp_type) - - # Match aggregate and block-owned occurrences without collapsing repeated citations. - block_citations: Counter[tuple[str, str | None, int | None, int | None]] = Counter() - for item, values in zip(result, block_annotations, strict=True): - text = item.text - text_utf16 = ( - text.encode("utf-16-le", errors="surrogatepass") if item.type == "text" and text is not None else None - ) - item_citations = list(item.annotations or []) - item_seen: set[tuple[str, str | None, int | None, int | None]] = set() - for value in values: - parsed = _parse_toolbox_url_citation(value, text_utf16) - if parsed is not None and parsed[1] not in item_seen: - item_citations.append(parsed[0]) - item_seen.add(parsed[1]) - block_citations[parsed[1]] += 1 - if item_citations: - item.annotations = item_citations - - unbound: list[Annotation] = [] - for value in structured_annotations: - parsed = _parse_toolbox_url_citation(value) - if parsed is not None: - if block_citations[parsed[1]]: - block_citations[parsed[1]] -= 1 - else: - unbound.append(parsed[0]) - if structured is not None: - structured_result = super()._parse_tool_result_from_mcp(mcp_type.model_copy(update={"content": []})) - if unbound: - structured_result[0].annotations = unbound - result.extend(structured_result) - return result - @override async def connect(self, *, reset: bool = False) -> None: """Connect to the toolbox and mark its first meaningful activation.""" diff --git a/python/packages/foundry_hosting/tests/test_toolbox.py b/python/packages/foundry_hosting/tests/test_toolbox.py index 4d66462fd47..bf17ac48b95 100644 --- a/python/packages/foundry_hosting/tests/test_toolbox.py +++ b/python/packages/foundry_hosting/tests/test_toolbox.py @@ -627,14 +627,13 @@ async def test_web_search_without_prose_keeps_structured_result(content: list[di @pytest.mark.parametrize( ("mode", "expected"), [ - ("structured_first", ['{"annotations": [{"type": "url_citation", "url": "https://example.test"}]}']), ("content_first", ["Search prose"]), ("content_only", ["Search prose"]), ("structured_only", ['{"annotations": [{"type": "url_citation", "url": "https://example.test"}]}']), ("both", ["Search prose", '{"annotations": [{"type": "url_citation", "url": "https://example.test"}]}']), ], ) -async def test_explicit_mcp_content_mode_overrides_toolbox_projection(mode: str, expected: list[str]) -> None: +async def test_other_mcp_content_modes_replace_citation_handling(mode: str, expected: list[str]) -> None: server = _ToolResultServer() server.tools = [toolbox_tool("never")] server.call_result = { From 98fc4a83d5fdee6222f4367b24ad2d096a2ce7fb Mon Sep 17 00:00:00 2001 From: sophia-ramsey Date: Thu, 12 Nov 2026 17:12:04 -0800 Subject: [PATCH 4/6] update test helper functions --- python/packages/core/agent_framework/_mcp.py | 28 +++- python/packages/core/tests/core/test_mcp.py | 19 +++ python/packages/foundry_hosting/README.md | 3 +- .../_responses.py | 16 +- .../web_search_test_helpers.py | 107 +++++++++++++ .../foundry_hosting/tests/test_responses.py | 148 ++++++++---------- .../foundry_hosting/tests/test_toolbox.py | 20 +-- 7 files changed, 233 insertions(+), 108 deletions(-) create mode 100644 python/packages/foundry_hosting/tests/foundry_hosting/web_search_test_helpers.py diff --git a/python/packages/core/agent_framework/_mcp.py b/python/packages/core/agent_framework/_mcp.py index 895c307f718..2df8f075f85 100644 --- a/python/packages/core/agent_framework/_mcp.py +++ b/python/packages/core/agent_framework/_mcp.py @@ -198,7 +198,8 @@ class MCPSpecificApproval(TypedDict, total=False): - ``structured_first`` (default): use ``structuredContent`` when present, else ``content``. When the result has text content and ``url_citation`` entries in ``structuredContent.annotations`` or a content block's ``_meta.annotations`` (for example, Foundry Toolbox web search), the text content is kept before - ``structuredContent`` and the citations are attached as framework annotations. + ``structuredContent`` and the citations are attached as framework annotations. Without text content, citations + are attached without spans to the selected structured result, when present. - ``content_first``: use ``content`` when non-empty, else ``structuredContent`` - ``content_only``: ignore ``structuredContent`` for the model-visible result - ``structured_only``: ignore ``content`` for the model-visible result @@ -272,10 +273,11 @@ def _project_mcp_url_citations( ) -> list[Content] | None: """Keep cited text content before structured content and attach its URL citations as annotations. - Returns ``None`` when the result is an error, has no recognized ``url_citation`` entries, or has no text content. - ``content_items`` must hold exactly one parsed item per MCP content block. + When there is no text content, attach recognized citations without spans to the structured block, if present. + Returns ``None`` when the result is an error or has no recognized ``url_citation`` entries. ``content_items`` + must hold exactly one parsed item per MCP content block. """ - if mcp_type.is_error or not mcp_type.content: + if mcp_type.is_error: return None structured: Any = mcp_type.structured_content structured_values = ( @@ -288,8 +290,22 @@ def _project_mcp_url_citations( for value in [*structured_annotations, *(value for values in block_annotations for value in values)] ): return None - if not any(item.type == "text" and item.text for item in content_items): - return None + has_text_content = any(item.type == "text" and item.text for item in content_items) + if not has_text_content and structured_block is not None: + citations: list[Annotation] = [] + seen: set[_MCPCitationKey] = set() + all_annotations = [ + *structured_annotations, + *(value for values in block_annotations for value in values), + ] + for value in all_annotations: + parsed = _parse_mcp_url_citation(value) + if parsed is not None and parsed[1] not in seen: + citations.append(parsed[0]) + seen.add(parsed[1]) + if citations: + structured_block.annotations = [*(structured_block.annotations or []), *citations] + return [structured_block] # Match aggregate and block-owned occurrences without collapsing repeated citations. block_citations: dict[_MCPCitationKey, int] = {} diff --git a/python/packages/core/tests/core/test_mcp.py b/python/packages/core/tests/core/test_mcp.py index e6d8b7c31bd..71773eea796 100644 --- a/python/packages/core/tests/core/test_mcp.py +++ b/python/packages/core/tests/core/test_mcp.py @@ -1457,6 +1457,25 @@ def test_structured_first_does_not_attach_url_citations_to_error_results(): assert result[0].annotations is None +@pytest.mark.parametrize( + "content", + [ + [], + [types.TextContent(type="text", text="")], + ], +) +def test_structured_first_attaches_unspanned_citations_without_search_text(content): + citation = {"type": "url_citation", "url": "https://example.test/source", "title": "Source"} + structured = {"annotations": [citation]} + mcp_result = types.CallToolResult(content=content, structured_content=structured) + + result = _HELPER_MCP_TOOL._parse_tool_result_from_mcp(mcp_result) + + assert len(result) == 1 + assert result[0].text == json.dumps(structured) + assert result[0].annotations == [{"type": "citation", "url": "https://example.test/source", "title": "Source"}] + + async def test_generated_mcp_tool_preserves_complete_host_payload_once() -> None: """The generated FunctionTool path retains one complete, persistent Host payload.""" file_meta = { diff --git a/python/packages/foundry_hosting/README.md b/python/packages/foundry_hosting/README.md index 6bda7c4fe34..1f830aac17e 100644 --- a/python/packages/foundry_hosting/README.md +++ b/python/packages/foundry_hosting/README.md @@ -29,7 +29,8 @@ general-purpose MCP approval policy. `FoundryToolbox` and generic MCP tools use the default `structured_first` result handling for Toolbox search results. When a result contains text content and `url_citation` annotations, the text stays visible to the model before the -structured JSON, and the citations are attached to the tool result. Another `tool_result_content` mode or a custom +structured JSON, and the citations are attached to the tool result. If the result has citations but no text content, +they are attached without spans to the selected structured result. Another `tool_result_content` mode or a custom `parse_tool_results` replaces this handling, and hosted answers then receive no Toolbox citations. ## Agent instances and factories diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py index be3dabebd40..a7a7335f28f 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py @@ -2441,14 +2441,20 @@ async def latest_checkpoint_id() -> str | None: def snapshot_response( checkpoint_id: str | None, + *, + final_answer: bool = False, ) -> Generator[ResponseStreamEvent | ResponseCheckpointEvent]: # Pair the response output emitted so far with the workflow checkpoint it corresponds # to, so recovery from that checkpoint replays exactly the updates that came after it. - if checkpoint_id is None or checkpoint_id == _hosted_internal_metadata(response_event_stream).get( - _LATEST_CHECKPOINT_ID_KEY - ): + if checkpoint_id is None: return - yield from tracker.close() + previous_checkpoint_id = _hosted_internal_metadata(response_event_stream).get(_LATEST_CHECKPOINT_ID_KEY) + if checkpoint_id == previous_checkpoint_id and not final_answer: + return + events = list(tracker.close(final_answer=final_answer)) + if checkpoint_id == previous_checkpoint_id and not events: + return + yield from events _hosted_internal_metadata(response_event_stream)[_LATEST_CHECKPOINT_ID_KEY] = checkpoint_id yield response_event_stream.checkpoint() @@ -2482,7 +2488,7 @@ def snapshot_response( elif self._resilient_background and not main_iter.signalled: # The workflow ran to completion: pair its final checkpoint with the full output, so # recovery after this point does not replay the last superstep. - for event in snapshot_response(await latest_checkpoint_id()): + for event in snapshot_response(await latest_checkpoint_id(), final_answer=True): yield event except Exception as exc: _log_response_failure("workflow execution", context.response_id, exc) diff --git a/python/packages/foundry_hosting/tests/foundry_hosting/web_search_test_helpers.py b/python/packages/foundry_hosting/tests/foundry_hosting/web_search_test_helpers.py new file mode 100644 index 00000000000..f97003e1650 --- /dev/null +++ b/python/packages/foundry_hosting/tests/foundry_hosting/web_search_test_helpers.py @@ -0,0 +1,107 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Reusable offline test helpers for web-search citations.""" + +from __future__ import annotations + +import json +from typing import Any + +import httpx2 as httpx +from agent_framework import AgentResponseUpdate, Annotation, Content + +from .toolbox_approval_helpers import ToolboxApprovalServer + +_PYTHON_SOURCE_URL = "https://www.python.org/downloads/release/python-31315/" +_DOCS_SOURCE_URL = "https://docs.python.org/3/whatsnew/3.13.html" + + +class _ToolResultServer(ToolboxApprovalServer): + """Offline Toolbox server whose ``tools/call`` response is configurable per test.""" + + def __init__(self) -> None: + super().__init__() + self.call_result: dict[str, Any] = {} + + async def handle(self, request: httpx.Request) -> httpx.Response: + if request.method == "POST": + payload = json.loads(request.content) + if payload.get("method") == "tools/call" and "id" in payload: + self.calls.append(payload["params"]) + return httpx.Response(200, json={"jsonrpc": "2.0", "id": payload["id"], "result": self.call_result}) + return await super().handle(request) + + +def _web_search_result(call_id: str) -> Content: + """Build a search result with duplicate URLs and non-web citations for response-host tests.""" + return Content.from_function_result( + call_id, + result=[ + Content.from_text( + "Python 3.13.15 is the latest release.", + annotations=[ + _url_citation(), + Annotation(type="citation", url=_DOCS_SOURCE_URL, title="What's New"), + ], + ), + Content.from_text( + '{"annotations": []}', + annotations=[ + Annotation(type="citation", url=_PYTHON_SOURCE_URL, title="Duplicate title"), + Annotation(type="citation", url="javascript:alert(1)", title="Unsafe"), + Annotation(type="citation", url="notes.txt", file_id="file_123", title="File"), + ], + ), + ], + ) + + +def _expected_source_citations(text: str) -> list[dict[str, Any]]: + """Return the zero-length end-of-text citations expected for ``_web_search_result``.""" + return [ + _expected_url_citation(start_index=len(text), end_index=len(text)), + _expected_url_citation(url=_DOCS_SOURCE_URL, title="What's New", start_index=len(text), end_index=len(text)), + ] + + +def _web_search_citation_updates(answer: str) -> list[AgentResponseUpdate]: + return [ + AgentResponseUpdate( + contents=[Content.from_function_call("call_1", "web_search", arguments={})], + role="assistant", + message_id="msg_call", + ), + AgentResponseUpdate(contents=[_web_search_result("call_1")], role="tool", message_id="msg_result"), + AgentResponseUpdate(contents=[Content.from_text(answer)], role="assistant", message_id="msg_answer"), + ] + + +def _url_citation( + *, + url: str = _PYTHON_SOURCE_URL, + title: str = "Python 3.13.15", + start_index: int = 0, + end_index: int = 6, +) -> Annotation: + return Annotation( + type="citation", + url=url, + title=title, + annotated_regions=[{"type": "text_span", "start_index": start_index, "end_index": end_index}], + ) + + +def _expected_url_citation( + *, + url: str = _PYTHON_SOURCE_URL, + title: str = "Python 3.13.15", + start_index: int = 0, + end_index: int = 6, +) -> dict[str, Any]: + return { + "type": "url_citation", + "url": url, + "title": title, + "start_index": start_index, + "end_index": end_index, + } diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index 9565c6e6e5c..71d52e1e9dc 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -120,6 +120,15 @@ FunctionApprovalStoreProvider, ) +from .foundry_hosting.web_search_test_helpers import ( # pyrefly: ignore[missing-import] - pytest's importlib namespace + _DOCS_SOURCE_URL, + _expected_source_citations, + _expected_url_citation, + _url_citation, + _web_search_citation_updates, + _web_search_result, +) + _OPENAI_HTTPX = cast(Any, import_module(DefaultAsyncHttpxClient.__mro__[1].__module__.partition(".")[0])) _PRIVATE_ERROR_DETAIL = "test-token-value at /srv/private/tool.py" _SESSION_STORAGE_ERROR_DETAIL = "session-storage-secret at C:\\sensitive\\session_store.py" @@ -247,82 +256,6 @@ def _expected_container_file_citation( } -_PYTHON_SOURCE_URL = "https://www.python.org/downloads/release/python-31315/" -_DOCS_SOURCE_URL = "https://docs.python.org/3/whatsnew/3.13.html" - - -def _web_search_result(call_id: str) -> Content: - """Build a search tool result whose citations repeat a URL and include ignored non-web citations.""" - return Content.from_function_result( - call_id, - result=[ - Content.from_text( - "Python 3.13.15 is the latest release.", - annotations=[_url_citation(), Annotation(type="citation", url=_DOCS_SOURCE_URL, title="What's New")], - ), - Content.from_text( - '{"annotations": []}', - annotations=[ - Annotation(type="citation", url=_PYTHON_SOURCE_URL, title="Duplicate title"), - Annotation(type="citation", url="javascript:alert(1)", title="Unsafe"), - Annotation(type="citation", url="notes.txt", file_id="file_123", title="File"), - ], - ), - ], - ) - - -def _expected_source_citations(text: str) -> list[dict[str, Any]]: - """Return the zero-length end-of-text citations expected for ``_web_search_result``.""" - return [ - _expected_url_citation(start_index=len(text), end_index=len(text)), - _expected_url_citation(url=_DOCS_SOURCE_URL, title="What's New", start_index=len(text), end_index=len(text)), - ] - - -def _web_search_citation_updates(answer: str) -> list[AgentResponseUpdate]: - return [ - AgentResponseUpdate( - contents=[Content.from_function_call("call_1", "web_search", arguments={})], - role="assistant", - message_id="msg_call", - ), - AgentResponseUpdate(contents=[_web_search_result("call_1")], role="tool", message_id="msg_result"), - AgentResponseUpdate(contents=[Content.from_text(answer)], role="assistant", message_id="msg_answer"), - ] - - -def _url_citation( - *, - url: str = "https://www.python.org/downloads/release/python-31315/", - title: str = "Python 3.13.15", - start_index: int = 0, - end_index: int = 6, -) -> Annotation: - return Annotation( - type="citation", - url=url, - title=title, - annotated_regions=[{"type": "text_span", "start_index": start_index, "end_index": end_index}], - ) - - -def _expected_url_citation( - *, - url: str = "https://www.python.org/downloads/release/python-31315/", - title: str = "Python 3.13.15", - start_index: int = 0, - end_index: int = 6, -) -> dict[str, Any]: - return { - "type": "url_citation", - "url": url, - "title": title, - "start_index": start_index, - "end_index": end_index, - } - - def _file_citation( *, file_id: str = "file_123", @@ -2608,9 +2541,11 @@ async def run(messages: Any = None, *, session: AgentSession, **kwargs: Any) -> completed = next(event["response"] for event in events if event["type"] == "response.completed") messages = [item for item in completed["output"] if item["type"] == "message"] - assert [message["content"][0]["text"] for message in messages] == ["Let me search.", answer] - assert messages[0]["content"][0]["annotations"] == [] - assert messages[1]["content"][0]["annotations"] == _expected_source_citations(answer) + parts = [part for message in messages for part in message["content"] if part["type"] == "output_text"] + assert len(parts) == len(messages) + assert [part["text"] for part in parts] == ["Let me search.", answer] + assert parts[0]["annotations"] == [] + assert parts[1]["annotations"] == _expected_source_citations(answer) async def test_provider_background_keeps_private_token_under_outer_response( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -9947,7 +9882,11 @@ async def _iter() -> AsyncIterator[AgentResponseUpdate]: return ResponseStream(_iter(), finalizer=AgentResponse.from_updates) -def _build_text_workflow_agent(text: str, *, finish_reason: FinishReasonLiteral | None = None) -> WorkflowAgent: +def _build_text_workflow_agent( + text: str, + *, + finish_reason: FinishReasonLiteral | None = None, +) -> WorkflowAgent: """Build a minimal ``WorkflowAgent`` whose inner agent emits a fixed text.""" class _TextAgent(SupportsAgentRun): @@ -10767,6 +10706,55 @@ async def test_snapshots_pair_output_with_the_checkpoint_it_follows(self, tmp_pa assert final_response["internal_metadata"][_INCOMPLETE_REASON_KEY] == "content_filter" assert final_response["status"] == "in_progress" + async def test_final_checkpoint_attaches_pending_tool_sources( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + workflow_agent = _build_text_workflow_agent("Final answer") + server = _make_server( + workflow_agent, + response_store=FileResponseStore(storage_dir=tmp_path), + options=ResponsesServerOptions(resilient_background=True), + ) + checkpoint_storage = MagicMock() + checkpoint_storage.get_latest = AsyncMock(side_effect=[None, MagicMock(checkpoint_id="checkpoint-final")]) + monkeypatch.setattr( + server._checkpoint_storage_provider, "get_store", MagicMock(return_value=checkpoint_storage) + ) + monkeypatch.setattr(server._function_approval_storage_provider, "get_store", MagicMock(return_value=None)) + + async def no_updates() -> AsyncIterator[AgentResponseUpdate]: + if False: + yield AgentResponseUpdate(contents=[], role="assistant") + + monkeypatch.setattr(workflow_agent, "run", MagicMock(return_value=no_updates())) + stream = ResponseEventStream(response_id="response-final-citations") + stream.emit_created() + stream.emit_in_progress() + tracker = _OutputItemTracker(stream) + _ = [event async for event in tracker.handle(_web_search_result("call_web_search"))] + _ = [event async for event in tracker.handle(Content.from_text("Final answer"), message_id="msg_final")] + + request = CreateResponse(model="m", input="hi", background=True, stream=True, store=True) + context = ResponseContext(response_id="response-final-citations", mode_flags=MagicMock()) + with patch.object(ResponseContext, "get_input_items", new=AsyncMock(return_value=[])): + events = [ + event + async for event in server._handle_inner_workflow( + request, + context, + stream, + tracker, + asyncio.Event(), + workflow_agent, + ) + ] + + checkpoints = [event for event in events if isinstance(event, ResponseCheckpointEvent)] + assert len(checkpoints) == 1 + final_message = next(item for item in checkpoints[0].response["output"] if item["type"] == "message") + final_content = cast(dict[str, Any], final_message["content"][0]) + assert final_content["annotations"] == _expected_source_citations("Final answer") + # endregion diff --git a/python/packages/foundry_hosting/tests/test_toolbox.py b/python/packages/foundry_hosting/tests/test_toolbox.py index bf17ac48b95..32fe7a9c8c8 100644 --- a/python/packages/foundry_hosting/tests/test_toolbox.py +++ b/python/packages/foundry_hosting/tests/test_toolbox.py @@ -53,22 +53,9 @@ toolbox_connection, toolbox_tool, ) - - -class _ToolResultServer(ToolboxApprovalServer): - """Offline Toolbox server whose ``tools/call`` response is configurable per test.""" - - def __init__(self) -> None: - super().__init__() - self.call_result: dict[str, Any] = {} - - async def handle(self, request: httpx.Request) -> httpx.Response: - if request.method == "POST": - payload = json.loads(request.content) - if payload.get("method") == "tools/call" and "id" in payload: - self.calls.append(payload["params"]) - return httpx.Response(200, json={"jsonrpc": "2.0", "id": payload["id"], "result": self.call_result}) - return await super().handle(request) +from .foundry_hosting.web_search_test_helpers import ( # pyrefly: ignore[missing-import] - pytest's importlib namespace + _ToolResultServer, +) class _StubAgent: @@ -622,6 +609,7 @@ async def test_web_search_without_prose_keeps_structured_result(content: list[di assert isinstance(result, list) assert result[0].text is not None assert json.loads(result[0].text) == structured + assert result[0].annotations == [{"type": "citation", "url": "https://example.test"}] @pytest.mark.parametrize( From ade86492f1669c25f670daa315e03051358caff1 Mon Sep 17 00:00:00 2001 From: sophia-ramsey Date: Thu, 12 Nov 2026 19:42:47 -0800 Subject: [PATCH 5/6] offset improvements and source quantity limitations --- python/packages/core/agent_framework/_mcp.py | 64 ++++-- python/packages/core/tests/core/test_mcp.py | 28 +++ python/packages/foundry_hosting/README.md | 3 +- .../_responses.py | 62 ++++-- .../_workflow_responses.py | 35 +++- .../foundry_hosting/test_native_responses.py | 144 ++++++++++++++ .../web_search_test_helpers.py | 80 +++++++- .../foundry_hosting/tests/test_responses.py | 185 ++++++++++++++++++ 8 files changed, 554 insertions(+), 47 deletions(-) diff --git a/python/packages/core/agent_framework/_mcp.py b/python/packages/core/agent_framework/_mcp.py index 2df8f075f85..edd6141fcfa 100644 --- a/python/packages/core/agent_framework/_mcp.py +++ b/python/packages/core/agent_framework/_mcp.py @@ -10,6 +10,8 @@ import sys import warnings from abc import abstractmethod +from array import array +from bisect import bisect_left from collections.abc import AsyncGenerator, Awaitable, Callable, Collection, Coroutine, Generator, Mapping, Sequence from contextlib import AsyncExitStack, _AsyncGeneratorContextManager, asynccontextmanager, contextmanager # type: ignore from contextvars import ContextVar @@ -212,8 +214,27 @@ class MCPSpecificApproval(TypedDict, total=False): _MCPCitationKey: TypeAlias = tuple[str, str | None, int | None, int | None] -def _parse_mcp_url_citation(value: Any, text_utf16: bytes | None = None) -> tuple[Annotation, _MCPCitationKey] | None: - """Validate an MCP ``url_citation`` entry and convert valid UTF-16 spans to Python text offsets.""" +def _utf16_surrogate_offsets(text: str) -> tuple[int, array[int]]: + """Return UTF-16 length and low-surrogate offsets belonging to pairs.""" + offset = 0 + previous_high = False + low_offsets = array("Q") + for character in text: + code = ord(character) + if code > 0xFFFF: + low_offsets.append(offset + 1) + offset += 2 + previous_high = False + else: + if previous_high and 0xDC00 <= code <= 0xDFFF: + low_offsets.append(offset) + previous_high = 0xD800 <= code <= 0xDBFF + offset += 1 + return offset, low_offsets + + +def _parse_mcp_url_citation(value: Any) -> tuple[Annotation, _MCPCitationKey] | None: + """Validate an MCP ``url_citation`` entry.""" if not isinstance(value, Mapping): return None raw = cast(Mapping[str, Any], value) @@ -229,26 +250,27 @@ def _parse_mcp_url_citation(value: Any, text_utf16: bytes | None = None) -> tupl citation = Annotation(type="citation", url=url) if title is not None: citation["title"] = title - if text_utf16 is not None and start is not None and end is not None and 0 <= start <= end <= len(text_utf16) // 2: + return citation, (url, title, start, end) + + +def _attach_mcp_citation_span( + citation: Annotation, key: _MCPCitationKey, text_utf16_length: int, low_offsets: Sequence[int] +) -> None: + _, _, start, end = key + if start is not None and end is not None and 0 <= start <= end <= text_utf16_length: # Citation offsets use UTF-16 code units; framework spans use Python text offsets. - boundaries = (start, end) - if all( - offset == 0 - or offset == len(text_utf16) // 2 - or not ( - 0xD800 <= int.from_bytes(text_utf16[(offset - 1) * 2 : offset * 2], "little") <= 0xDBFF - and 0xDC00 <= int.from_bytes(text_utf16[offset * 2 : (offset + 1) * 2], "little") <= 0xDFFF - ) - for offset in boundaries + start_pairs = bisect_left(low_offsets, start) + end_pairs = bisect_left(low_offsets, end) + if (start_pairs == len(low_offsets) or low_offsets[start_pairs] != start) and ( + end_pairs == len(low_offsets) or low_offsets[end_pairs] != end ): citation["annotated_regions"] = [ { "type": "text_span", - "start_index": len(text_utf16[: start * 2].decode("utf-16-le", errors="surrogatepass")), - "end_index": len(text_utf16[: end * 2].decode("utf-16-le", errors="surrogatepass")), + "start_index": start - start_pairs, + "end_index": end - end_pairs, } ] - return citation, (url, title, start, end) def _mcp_block_annotations(item: Any) -> list[Any]: @@ -310,16 +332,18 @@ def _project_mcp_url_citations( # Match aggregate and block-owned occurrences without collapsing repeated citations. block_citations: dict[_MCPCitationKey, int] = {} for item, values in zip(content_items, block_annotations, strict=True): - text_utf16 = ( - item.text.encode("utf-16-le", errors="surrogatepass") - if item.type == "text" and item.text is not None - else None + text_utf16_length, low_offsets = ( + _utf16_surrogate_offsets(item.text) + if values and item.type == "text" and item.text is not None + else (None, array("Q")) ) item_citations = list(item.annotations or []) item_seen: set[_MCPCitationKey] = set() for value in values: - parsed = _parse_mcp_url_citation(value, text_utf16) + parsed = _parse_mcp_url_citation(value) if parsed is not None and parsed[1] not in item_seen: + if text_utf16_length is not None: + _attach_mcp_citation_span(parsed[0], parsed[1], text_utf16_length, low_offsets) item_citations.append(parsed[0]) item_seen.add(parsed[1]) block_citations[parsed[1]] = block_citations.get(parsed[1], 0) + 1 diff --git a/python/packages/core/tests/core/test_mcp.py b/python/packages/core/tests/core/test_mcp.py index 71773eea796..485548b327f 100644 --- a/python/packages/core/tests/core/test_mcp.py +++ b/python/packages/core/tests/core/test_mcp.py @@ -1444,6 +1444,34 @@ def test_structured_first_keeps_cited_content_and_attaches_url_citations(): assert all(item.additional_properties["_meta"] == {"source": "server"} for item in result) +def test_cited_text_converts_multiple_utf16_spans_and_deduplicates(): + text = "A\U0001f30eB\U0001f680C" + citations = [ + {"type": "url_citation", "url": "https://example.test/earth", "start_index": 1, "end_index": 3}, + {"type": "url_citation", "url": "https://example.test/rocket", "start_index": 4, "end_index": 6}, + {"type": "url_citation", "url": "https://example.test/invalid", "start_index": 2, "end_index": 6}, + ] + mcp_result = types.CallToolResult( + content=[types.TextContent(type="text", text=text, _meta={"annotations": [*citations, *citations]})] + ) + + result = _HELPER_MCP_TOOL._parse_tool_result_from_mcp(mcp_result) + + assert result[0].annotations == [ + { + "type": "citation", + "url": "https://example.test/earth", + "annotated_regions": [{"type": "text_span", "start_index": 1, "end_index": 2}], + }, + { + "type": "citation", + "url": "https://example.test/rocket", + "annotated_regions": [{"type": "text_span", "start_index": 3, "end_index": 4}], + }, + {"type": "citation", "url": "https://example.test/invalid"}, + ] + + def test_structured_first_does_not_attach_url_citations_to_error_results(): mcp_result = types.CallToolResult( content=[types.TextContent(type="text", text="Search failed")], diff --git a/python/packages/foundry_hosting/README.md b/python/packages/foundry_hosting/README.md index 1f830aac17e..49450a7591f 100644 --- a/python/packages/foundry_hosting/README.md +++ b/python/packages/foundry_hosting/README.md @@ -323,7 +323,8 @@ Web URL citations on function results, such as `FoundryToolbox` search results, final answer text when the response completes. These citations do not identify which part of the answer each source supports, so the host emits each one as a zero-length `url_citation` at the end of that text. Sources are deduplicated by URL, limited to `http` and `https`, skipped when the answer already cites the same URL, and not -attached to interim text. Plain `agent.run()` responses do not include these sources. +attached to interim text. A response retains at most 100 sources and 64 KiB of UTF-8 URL and title data; +additional or oversized sources are skipped with a warning. Plain `agent.run()` responses do not include these sources. ## State store diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py index a7a7335f28f..e0a4ca999f5 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py @@ -18,12 +18,14 @@ Awaitable, Callable, Generator, + Iterator, Mapping, Sequence, ) from contextlib import AbstractAsyncContextManager, AsyncExitStack, aclosing, suppress from copy import copy from dataclasses import asdict, dataclass, is_dataclass +from itertools import chain from typing import TYPE_CHECKING, Any, Generic, Literal, TypeGuard, TypeVar, cast from urllib.parse import urlparse @@ -153,6 +155,8 @@ _HOSTED_PROVIDER_USAGE_KEY = "_foundry_provider_usage" _PENDING_CONTAINER_FILE_CITATIONS_KEY = "_foundry_pending_container_file_citations" _PENDING_URL_CITATIONS_KEY = "_foundry_pending_url_citations" +_MAX_PENDING_URL_CITATIONS = 100 +_MAX_PENDING_URL_CITATION_BYTES = 64 * 1024 _CONTAINER_FILE_CITATIONS_KEY = "container_file_citations" _PUBLIC_STREAM_FAILURE_MESSAGE = "An internal server error occurred." @@ -283,13 +287,11 @@ def _container_file_citations_from_function_result(content: Content) -> list[_Co return citations -def _url_citations_from_function_result(content: Content) -> dict[str, str]: - """Collect the web sources cited by a function result, keyed by URL in first-seen order with their titles.""" - candidates: list[Any] = list(content.annotations or []) - for item in content.items or []: - candidates.extend(item.annotations or []) - - citations: dict[str, str] = {} +def _url_citations_from_function_result(content: Content) -> Iterator[tuple[str, str]]: + """Yield web sources cited by a function result in their original order.""" + candidates = chain( + content.annotations or [], chain.from_iterable(item.annotations or [] for item in content.items or []) + ) for annotation in candidates: if not isinstance(annotation, Mapping): continue @@ -297,7 +299,7 @@ def _url_citations_from_function_result(content: Content) -> dict[str, str]: if annotation_mapping.get("type") != "citation" or annotation_mapping.get("file_id"): continue url = annotation_mapping.get("url") - if not isinstance(url, str) or url in citations: + if not isinstance(url, str): continue try: scheme = urlparse(url).scheme.lower() @@ -306,8 +308,7 @@ def _url_citations_from_function_result(content: Content) -> dict[str, str]: if scheme not in {"http", "https"}: continue title = annotation_mapping.get("title") - citations[url] = title if isinstance(title, str) else "" - return citations + yield url, title if isinstance(title, str) else "" def _annotations_to_output_text( @@ -2613,6 +2614,7 @@ def __init__( self._oauth_consent_requests: set[tuple[str, str]] = set() self._pending_container_file_citations: dict[str, _ContainerFileCitation] = {} self._pending_url_citations: dict[str, str] = {} + self._pending_url_citation_bytes = 0 self._text_annotations: list[ResponsesAnnotation] = [] self._message_text_annotations: dict[int, list[ResponsesAnnotation]] = {} # Set when an agent update reports the model stopped early (content filter, token @@ -2658,6 +2660,7 @@ def __init__( persisted_url_citations, (str, bytes, bytearray) ): raise RuntimeError("The persisted URL citations are invalid.") + skipped_url_citation = False for raw_url_citation in cast(Sequence[Any], persisted_url_citations): if not isinstance(raw_url_citation, Mapping): raise RuntimeError("The persisted URL citations are invalid.") @@ -2666,7 +2669,10 @@ def __init__( title = url_citation.get("title") if not isinstance(url, str) or not url or not isinstance(title, str): raise RuntimeError("The persisted URL citations are invalid.") - self._pending_url_citations.setdefault(url, title) + skipped_url_citation |= not self._retain_pending_url_citation(url, title) + if skipped_url_citation: + logger.warning("Pending URL citations exceeded the hosted source limits; excess sources were skipped.") + self._persist_pending_url_citations() for item in stream.response.get("output", []): if not isinstance(item, Mapping): continue @@ -2862,10 +2868,14 @@ async def handle( for citation in citations: self._pending_container_file_citations[citation.filename] = citation self._persist_pending_container_file_citations() - url_citations = _url_citations_from_function_result(content) - if url_citations: - for url, title in url_citations.items(): - self._pending_url_citations.setdefault(url, title) + skipped_url_citation = False + had_url_citations = False + for url, title in _url_citations_from_function_result(content): + had_url_citations = True + skipped_url_citation |= not self._retain_pending_url_citation(url, title) + if skipped_url_citation: + logger.warning("Pending URL citations exceeded the hosted source limits; excess sources were skipped.") + if had_url_citations: self._persist_pending_url_citations() async for event in self._stream.output_item_function_call_output( content.call_id, # type: ignore[arg-type] @@ -3133,6 +3143,7 @@ def discard_pending_citations(self) -> None: self._pending_container_file_citations.clear() self._persist_pending_container_file_citations() self._pending_url_citations.clear() + self._pending_url_citation_bytes = 0 self._persist_pending_url_citations() # -- Private open/close helpers -- @@ -3290,7 +3301,10 @@ def _close_message_content(self, *, attach_sources: bool = False) -> Generator[R if annotation not in annotations: annotations.append(annotation) if attach_sources: - annotations.extend(self._source_url_annotations_for_text(accumulated, annotations)) + earlier_annotations = chain.from_iterable(self._message_text_annotations.values()) + annotations.extend( + self._source_url_annotations_for_text(accumulated, [*earlier_annotations, *annotations]) + ) yield self._text_content.emit_text_done(accumulated) for annotation in annotations: yield self._text_content.emit_annotation_added(annotation) @@ -3376,9 +3390,25 @@ def _source_url_annotations_for_text( if url not in cited_urls ] self._pending_url_citations.clear() + self._pending_url_citation_bytes = 0 self._persist_pending_url_citations() return annotations + def _retain_pending_url_citation(self, url: str, title: str) -> bool: + if url in self._pending_url_citations: + return True + if ( + len(self._pending_url_citations) >= _MAX_PENDING_URL_CITATIONS + or len(url) + len(title) > _MAX_PENDING_URL_CITATION_BYTES + ): + return False + size = len(url.encode("utf-8", errors="surrogatepass")) + len(title.encode("utf-8", errors="surrogatepass")) + if self._pending_url_citation_bytes + size > _MAX_PENDING_URL_CITATION_BYTES: + return False + self._pending_url_citations[url] = title + self._pending_url_citation_bytes += size + return True + def _persist_pending_url_citations(self) -> None: if not self._pending_url_citations: _hosted_internal_metadata(self._stream).pop(_PENDING_URL_CITATIONS_KEY, None) diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_workflow_responses.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_workflow_responses.py index 2df89e5ecb7..306eb9e5b54 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_workflow_responses.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_workflow_responses.py @@ -504,7 +504,7 @@ async def abort() -> bool: buffered: list[WorkflowEvent[Any]] = [] tracker = _OutputItemTracker(stream, self.allowed_origins) - async def flush(checkpoint_id: str | None) -> list[ResponseStreamEvent]: + async def flush(checkpoint_id: str | None, *, final_answer: bool = False) -> list[ResponseStreamEvent]: nonlocal stream, tracker, aliases checkpoint = await run.get_checkpoint(checkpoint_id) if run.stored else None pending = checkpoint.pending_request_info_events if checkpoint is not None else {} @@ -519,7 +519,10 @@ async def flush(checkpoint_id: str | None) -> list[ResponseStreamEvent]: events: list[ResponseStreamEvent] = [] for event in buffered: events.extend(await _project(event, draft, next_tracker, approval_projection)) - events.extend(next_tracker.close()) + is_final_output = final_answer and bool(buffered) and buffered[-1].type == "output" + events.extend(next_tracker.close(final_answer=is_final_output)) + if final_answer: + next_tracker.discard_pending_citations() _hosted_internal_metadata(draft)[_HOSTED_PROVIDER_USAGE_KEY] = next_tracker.usage_details await run.stage(_snapshot(draft), approvals=next_aliases, checkpoint_id=checkpoint_id) stream, tracker, aliases = draft, next_tracker, next_aliases @@ -531,6 +534,8 @@ async def checkpoint_stamp() -> str | None: # ruff: ignore[unused-async] phase = "execution" if not run.has_completed_output: + pending_flush = False + pending_checkpoint: str | None = None iterator = _SignalledIterator( run.events(turn, client_kwargs=client_kwargs, function_invocation_kwargs=function_kwargs), context.shutdown, @@ -539,6 +544,18 @@ async def checkpoint_stamp() -> str | None: # ruff: ignore[unused-async] ) async with aclosing(iterator): async for event in iterator: + if pending_flush and event.type in ( + "superstep_started", + "request_info", + "output", + "intermediate", + "data", + ): + for output_event in await flush(pending_checkpoint): + yield wire(output_event) + if resilient: + yield stream.checkpoint() + pending_flush = False if event.type == "request_info" and event.request_id not in represented_pending: if not self.resolver.is_factory: raise _WorkflowRequestError( @@ -552,10 +569,8 @@ async def checkpoint_stamp() -> str | None: # ruff: ignore[unused-async] if resilient: yield stream.checkpoint() elif event.type == "superstep_completed": - for output_event in await flush(iterator.stamp): - yield wire(output_event) - if resilient: - yield stream.checkpoint() + pending_checkpoint = iterator.stamp + pending_flush = True if iterator.signalled: if context.shutdown.is_set() and resilient and run.snapshot is not None: preserve_for_recovery = True @@ -564,13 +579,19 @@ async def checkpoint_stamp() -> str | None: # ruff: ignore[unused-async] await abort() return raise _WorkflowRequestError("The workflow stopped without a recoverable background pair.") + if pending_flush: + for output_event in await flush(pending_checkpoint, final_answer=True): + yield wire(output_event) + if resilient: + yield stream.checkpoint() if cancellation_signal.is_set(): await abort() return phase = "final output persistence" if buffered: - for event in await flush(run.checkpoint_id): + for event in await flush(run.checkpoint_id, final_answer=True): yield wire(event) + tracker.discard_pending_citations() terminal = ( stream.emit_incomplete(reason=tracker.incomplete_reason, usage=tracker.usage) if tracker.oauth_consent_requested or tracker.incomplete_reason is not None diff --git a/python/packages/foundry_hosting/tests/foundry_hosting/test_native_responses.py b/python/packages/foundry_hosting/tests/foundry_hosting/test_native_responses.py index bd90420d6a0..92fb84aef8a 100644 --- a/python/packages/foundry_hosting/tests/foundry_hosting/test_native_responses.py +++ b/python/packages/foundry_hosting/tests/foundry_hosting/test_native_responses.py @@ -49,6 +49,11 @@ ) from agent_framework_foundry_hosting._workflow_state import FoundryWorkflowBindingStore, HostedWorkflowRun +from .web_search_test_helpers import ( # pyrefly: ignore[missing-import] - pytest's importlib namespace + _CitationWorkflowExecutor, + _expected_source_citations, +) + class _Ticket(BaseModel): text: str @@ -604,6 +609,145 @@ async def start(self, message: str, ctx: WorkflowContext[str, Any]) -> None: assert message["content"][0]["annotations"] == [_expected_url_citation(start_index=4, end_index=10)] +@pytest.mark.parametrize("stored", [False, True]) +async def test_native_workflow_attaches_tool_sources_only_to_final_answer(stored: bool) -> None: + def create(request: HostedResponseRequest) -> Workflow: + source = _CitationWorkflowExecutor() + return WorkflowBuilder(name="citation-workflow", start_executor=source).add_edge(source, source).build() + + server = _server(create, parser=_parse_count) + response = await _post(server, "1", store=stored) + + assert response["status"] == "completed", response + messages = [item for item in response["output"] if item["type"] == "message"] + assert [item["content"][0]["text"] for item in messages] == ["Checking.", "Final answer"] + assert messages[0]["content"][0]["annotations"] == [] + assert messages[1]["content"][0]["annotations"] == _expected_source_citations("Final answer") + assert "_foundry_pending_url_citations" not in response.get("internal_metadata", {}) + + +async def test_native_workflow_streams_final_sources_consistently() -> None: + def create(request: HostedResponseRequest) -> Workflow: + source = _CitationWorkflowExecutor() + return WorkflowBuilder(name="citation-stream", start_executor=source).add_edge(source, source).build() + + server = _server(create, parser=_parse_count) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=server), base_url="http://test") as client: + response = await client.post("/responses", json={"input": "1", "store": False, "stream": True}) + + assert response.status_code == 200 + events = [ + json.loads(line[5:].strip()) + for line in response.text.splitlines() + if line.startswith("data:") and line[5:].strip() != "[DONE]" + ] + expected = _expected_source_citations("Final answer") + assert [event["annotation"] for event in events if event["type"] == "response.output_text.annotation.added"] == ( + expected + ) + done = [event for event in events if event["type"] == "response.content_part.done"] + assert [event["part"]["annotations"] for event in done] == [[], expected] + completed = next(event["response"] for event in events if event["type"] == "response.completed") + messages = [item for item in completed["output"] if item["type"] == "message"] + assert messages[-1]["content"][0]["annotations"] == expected + + +async def test_native_workflow_discards_sources_without_a_final_answer() -> None: + def create(request: HostedResponseRequest) -> Workflow: + source = _CitationWorkflowExecutor(final_text=None) + return WorkflowBuilder(name="citation-no-answer", start_executor=source).add_edge(source, source).build() + + committed: list[dict[str, Any]] = [] + original = HostedWorkflowRun.commit + + async def commit(run: HostedWorkflowRun, snapshot: Any = None) -> None: + committed.append(snapshot) + await original(run, snapshot) + + with patch.object(HostedWorkflowRun, "commit", commit): + response = await _post(_server(create, parser=_parse_count), "1") + + assert response["status"] == "completed", response + message = next(item for item in response["output"] if item["type"] == "message") + assert message["content"][0]["annotations"] == [] + assert "_foundry_pending_url_citations" not in committed[-1].get("internal_metadata", {}) + + +async def test_native_workflow_does_not_cite_terminal_intermediate_output() -> None: + def create(request: HostedResponseRequest) -> Workflow: + source = _CitationWorkflowExecutor() + return ( + WorkflowBuilder( + name="citation-intermediate", + start_executor=source, + output_from=[], + intermediate_output_from="all", + ) + .add_edge(source, source) + .build() + ) + + response = await _post(_server(create, parser=_parse_count), "1", store=False) + + assert response["status"] == "completed", response + messages = [item for item in response["output"] if item["type"] == "message"] + assert [item["content"][0]["annotations"] for item in messages] == [[], []] + + +async def test_native_workflow_recovers_pending_sources(tmp_path: Any, monkeypatch: pytest.MonkeyPatch) -> None: + from azure.ai.agentserver.core.tasks import resilient_tasks_enabled, set_resilient_tasks_enabled + + enabled = resilient_tasks_enabled() + set_resilient_tasks_enabled(True) + monkeypatch.setenv("AGENTSERVER_SHUTDOWN_GRACE_SECONDS", "0.1") + started, release, stopped, paired = (asyncio.Event() for _ in range(4)) + + def create(request: HostedResponseRequest) -> Workflow: + source = _CitationWorkflowExecutor(started, release, stopped) + return WorkflowBuilder(name="citation-recovery", start_executor=source).add_edge(source, source).build() + + provider = FileResponseStore(storage_dir=tmp_path / "responses") + options = ResponsesServerOptions(resilient_background=True, shutdown_grace_period_seconds=1) + server = _server(create, parser=_parse_count, response_store=provider, options=options) + original = HostedWorkflowRun.stage + + async def stage(run: HostedWorkflowRun, snapshot: Any, **kwargs: Any) -> None: + await original(run, snapshot, **kwargs) + if snapshot.get("internal_metadata", {}).get("_foundry_pending_url_citations"): + paired.set() + + try: + with patch.object(HostedWorkflowRun, "stage", stage): + async with ( + server.router.lifespan_context(server), + httpx.AsyncClient(transport=httpx.ASGITransport(app=server), base_url="http://test") as client, + ): + pending = await client.post( + "/responses", + json={"input": "1", "agent_session_id": "sandbox", "store": True, "background": True}, + ) + assert pending.status_code == 200 + response_id = pending.json()["id"] + await asyncio.wait_for(started.wait(), timeout=3) + await asyncio.wait_for(paired.wait(), timeout=3) + assert stopped.is_set() + release.set() + restarted = _server(create, parser=_parse_count, response_store=provider, options=options) + async with ( + restarted.router.lifespan_context(restarted), + httpx.AsyncClient(transport=httpx.ASGITransport(app=restarted), base_url="http://test") as client, + ): + result = await _poll(client, response_id) + assert result["status"] == "completed", result + messages = [item for item in result["output"] if item["type"] == "message"] + assert [item["content"][0]["text"] for item in messages] == ["Checking.", "Final answer"] + assert messages[0]["content"][0]["annotations"] == [] + assert messages[1]["content"][0]["annotations"] == _expected_source_citations("Final answer") + finally: + release.set() + set_resilient_tasks_enabled(enabled) + + async def test_native_agent_executor_applies_options_without_mutating_defaults_or_double_keywords() -> None: calls: list[dict[str, Any]] = [] lifecycle: list[str] = [] diff --git a/python/packages/foundry_hosting/tests/foundry_hosting/web_search_test_helpers.py b/python/packages/foundry_hosting/tests/foundry_hosting/web_search_test_helpers.py index f97003e1650..ffb17006312 100644 --- a/python/packages/foundry_hosting/tests/foundry_hosting/web_search_test_helpers.py +++ b/python/packages/foundry_hosting/tests/foundry_hosting/web_search_test_helpers.py @@ -4,13 +4,26 @@ from __future__ import annotations +import asyncio import json from typing import Any import httpx2 as httpx -from agent_framework import AgentResponseUpdate, Annotation, Content - -from .toolbox_approval_helpers import ToolboxApprovalServer +from agent_framework import ( + AgentResponseUpdate, + Annotation, + Content, + Executor, + Message, + WorkflowAgent, + WorkflowBuilder, + WorkflowContext, + handler, +) + +from .toolbox_approval_helpers import ( # pyrefly: ignore[missing-import] - pytest's importlib namespace + ToolboxApprovalServer, +) _PYTHON_SOURCE_URL = "https://www.python.org/downloads/release/python-31315/" _DOCS_SOURCE_URL = "https://docs.python.org/3/whatsnew/3.13.html" @@ -32,6 +45,67 @@ async def handle(self, request: httpx.Request) -> httpx.Response: return await super().handle(request) +class _CitationWorkflowExecutor(Executor): + """Emit a cited tool result before the final answer in a later superstep.""" + + def __init__( + self, + started: asyncio.Event | None = None, + release: asyncio.Event | None = None, + stopped: asyncio.Event | None = None, + *, + final_text: str | None = "Final answer", + ) -> None: + super().__init__("citation-workflow") + self.started = started + self.release = release + self.stopped = stopped + self.final_text = final_text + + @handler + async def run(self, step: int, ctx: WorkflowContext[int, Message]) -> None: + if step: + await ctx.yield_output(Message("tool", [_web_search_result("call_1")])) + await ctx.yield_output(Message("assistant", "Checking.")) + await ctx.send_message(0) + return + if self.started is not None: + self.started.set() + try: + if self.release is not None: + await self.release.wait() + finally: + if self.stopped is not None: + self.stopped.set() + if self.final_text is not None: + await ctx.yield_output(Message("assistant", self.final_text)) + + +class _LegacyCitationStart(Executor): + def __init__(self) -> None: + super().__init__("citation-start") + + @handler + async def run(self, messages: list[Message], ctx: WorkflowContext[str, Content]) -> None: + await ctx.yield_output(_web_search_result("call_1")) + await ctx.send_message("answer") + + +class _LegacyCitationFinish(Executor): + def __init__(self) -> None: + super().__init__("citation-finish") + + @handler + async def run(self, message: str, ctx: WorkflowContext[str, Content]) -> None: + await ctx.yield_output(Content.from_text("Final answer")) + + +def _legacy_citation_workflow_agent() -> WorkflowAgent: + start, finish = _LegacyCitationStart(), _LegacyCitationFinish() + workflow = WorkflowBuilder(name="legacy-citation", start_executor=start).add_edge(start, finish).build() + return WorkflowAgent(workflow=workflow, name="Legacy Citation Workflow") + + def _web_search_result(call_id: str) -> Content: """Build a search result with duplicate URLs and non-web citations for response-host tests.""" return Content.from_function_result( diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index 71d52e1e9dc..696ebe568c5 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -124,6 +124,7 @@ _DOCS_SOURCE_URL, _expected_source_citations, _expected_url_citation, + _legacy_citation_workflow_agent, _url_citation, _web_search_citation_updates, _web_search_result, @@ -5489,6 +5490,152 @@ async def test_pending_url_citations_survive_recovery_and_are_discarded_at_compl events.extend(discarded.close(final_answer=True)) assert not any(event["type"] == "response.output_text.annotation.added" for event in events) + @pytest.mark.parametrize("final_already_cites", [False, True]) + async def test_pending_sources_deduplicate_across_final_message_parts_only(self, final_already_cites: bool) -> None: + stream = ResponseEventStream(response_id="resp_multipart_citations") + stream.emit_created() + stream.emit_in_progress() + tracker = _OutputItemTracker(stream) + _ = [event async for event in tracker.handle(Content.from_text("Interim", annotations=[_url_citation()]))] + _ = list(tracker.close()) + _ = [event async for event in tracker.handle(_web_search_result("call_1"))] + update = AgentResponseUpdate( + contents=[ + Content.from_text("First", annotations=[_url_citation()] if final_already_cites else None), + Content.from_text(" final"), + ], + role="assistant", + message_id="final", + ) + events = [event async for event in tracker.handle_update(update, complete_message=True)] + events.extend(tracker.close(final_answer=True)) + + annotations = [ + event["annotation"] for event in events if event["type"] == "response.output_text.annotation.added" + ] + expected = ( + [ + _expected_url_citation(), + _expected_url_citation(url=_DOCS_SOURCE_URL, title="What's New", start_index=6, end_index=6), + ] + if final_already_cites + else _expected_source_citations(" final") + ) + assert annotations == expected + message = next( + event["item"] + for event in events + if event["type"] == "response.output_item.done" and event["item"]["type"] == "message" + ) + parts = [part for part in message["content"] if part["type"] == "output_text"] + assert [part["annotations"] for part in parts] == ( + [[expected[0]], [expected[1]]] if final_already_cites else [[], expected] + ) + + async def test_pending_source_limits_apply_across_results_and_recovery( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ) -> None: + monkeypatch.setattr("agent_framework_foundry_hosting._responses._MAX_PENDING_URL_CITATIONS", 2) + monkeypatch.setattr("agent_framework_foundry_hosting._responses._MAX_PENDING_URL_CITATION_BYTES", 64) + stream = ResponseEventStream(response_id="resp_source_limits") + stream.emit_created() + stream.emit_in_progress() + tracker = _OutputItemTracker(stream) + first = Content.from_function_result( + "call_1", + result=[ + Content.from_text( + "Result", + annotations=[ + Annotation(type="citation", url="https://example.test/1", title="A"), + Annotation(type="citation", url="https://example.test/too-large", title="x" * 64), + ], + ) + ], + ) + _ = [event async for event in tracker.handle(first)] + recovered = _OutputItemTracker(stream) + second = Content.from_function_result( + "call_2", + result=[ + Content.from_text( + "Result", + annotations=[ + Annotation(type="citation", url="https://example.test/2", title="B"), + Annotation(type="citation", url="https://example.test/3", title="C"), + ], + ) + ], + ) + _ = [event async for event in recovered.handle(second)] + events = [event async for event in recovered.handle(Content.from_text("Done."), message_id="final")] + events.extend(recovered.close(final_answer=True)) + + annotations = [ + event["annotation"]["url"] + for event in events + if event["type"] == "response.output_text.annotation.added" + and event["annotation"]["type"] == "url_citation" + ] + assert annotations == [ + "https://example.test/1", + "https://example.test/2", + ] + assert "excess sources were skipped" in caplog.text + + async def test_pending_source_byte_limit_counts_utf8_and_skips_oversized_entries( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr("agent_framework_foundry_hosting._responses._MAX_PENDING_URL_CITATION_BYTES", 33) + stream = ResponseEventStream(response_id="resp_source_bytes") + stream.emit_created() + stream.emit_in_progress() + tracker = _OutputItemTracker(stream) + result = Content.from_function_result( + "call_1", + result=[ + Content.from_text( + "Result", + annotations=[ + Annotation(type="citation", url="https://x", title="\U0001f30e" * 4), + Annotation(type="citation", url="https://y"), + Annotation(type="citation", url="http://z"), + ], + ) + ], + ) + _ = [event async for event in tracker.handle(result)] + recovered = _OutputItemTracker(stream) + events = [event async for event in recovered.handle(Content.from_text("Done."), message_id="final")] + events.extend(recovered.close(final_answer=True)) + + assert [ + event["annotation"]["url"] + for event in events + if event["type"] == "response.output_text.annotation.added" + and event["annotation"]["type"] == "url_citation" + ] == ["https://x", "http://z"] + + async def test_recovery_bounds_previously_persisted_sources(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("agent_framework_foundry_hosting._responses._MAX_PENDING_URL_CITATIONS", 2) + stream = ResponseEventStream(response_id="resp_oversized_source_snapshot") + stream.emit_created() + stream.emit_in_progress() + _hosted_internal_metadata(stream)["_foundry_pending_url_citations"] = [ + {"url": f"https://example.test/{index}", "title": str(index)} for index in range(3) + ] + + recovered = _OutputItemTracker(stream) + events = [event async for event in recovered.handle(Content.from_text("Done."), message_id="final")] + events.extend(recovered.close(final_answer=True)) + + assert [ + event["annotation"]["url"] + for event in events + if event["type"] == "response.output_text.annotation.added" + and event["annotation"]["type"] == "url_citation" + ] == ["https://example.test/0", "https://example.test/1"] + async def test_pending_container_file_citation_survives_tracker_recovery(self) -> None: stream = ResponseEventStream(response_id="resp_container_file") stream.emit_created() @@ -10755,6 +10902,44 @@ async def no_updates() -> AsyncIterator[AgentResponseUpdate]: final_content = cast(dict[str, Any], final_message["content"][0]) assert final_content["annotations"] == _expected_source_citations("Final answer") + async def test_workflow_completion_and_recovery_attach_tool_sources(self, tmp_path: Path) -> None: + request = CreateResponse(model="m", input="hi", background=True, stream=True, store=True) + store = FileResponseStore(storage_dir=tmp_path) + options = ResponsesServerOptions(resilient_background=True) + server = _make_server(_legacy_citation_workflow_agent(), response_store=store, options=options) + context = ResponseContext(response_id="response-citation-workflow", mode_flags=MagicMock()) + + snapshots: list[dict[str, Any]] = [] + events: list[Any] = [] + async for event in server._handle_response(request, context, asyncio.Event()): + if isinstance(event, ResponseCheckpointEvent): + snapshots.append(copy.deepcopy(dict(event.response))) + else: + events.append(event) + + completed = next(event["response"] for event in events if event["type"] == "response.completed") + message = next(item for item in completed["output"] if item["type"] == "message") + assert message["content"][0]["annotations"] == _expected_source_citations("Final answer") + pending = next( + snapshot + for snapshot in snapshots + if snapshot.get("internal_metadata", {}).get("_foundry_pending_url_citations") + ) + + recovered_context = ResponseContext(response_id=context.response_id, mode_flags=MagicMock()) + recovered_context.is_recovery = True + recovered_context.persisted_response = cast(ResponseObject, pending) + restarted = _make_server(_legacy_citation_workflow_agent(), response_store=store, options=options) + recovered_events = [ + event + async for event in restarted._handle_response(request, recovered_context, asyncio.Event()) + if isinstance(event, Mapping) + ] + recovered = next(event["response"] for event in recovered_events if event["type"] == "response.completed") + message = next(item for item in recovered["output"] if item["type"] == "message") + part = next(part for part in message["content"] if part["type"] == "output_text") + assert part["annotations"] == _expected_source_citations("Final answer") + # endregion From 8223cd1d2fc1a817dc0f0793d5f36b80d47de332 Mon Sep 17 00:00:00 2001 From: sophia-ramsey Date: Thu, 12 Nov 2026 21:12:26 -0800 Subject: [PATCH 6/6] test updates --- .../tests/foundry_hosting/test_native_responses.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/packages/foundry_hosting/tests/foundry_hosting/test_native_responses.py b/python/packages/foundry_hosting/tests/foundry_hosting/test_native_responses.py index 92fb84aef8a..c26ef3d5a60 100644 --- a/python/packages/foundry_hosting/tests/foundry_hosting/test_native_responses.py +++ b/python/packages/foundry_hosting/tests/foundry_hosting/test_native_responses.py @@ -1000,9 +1000,13 @@ async def test_real_sdk_background_cancel_drains_suspended_executor_and_blocks_c await asyncio.wait_for(stopped.wait(), timeout=2) result = await _poll(client, response_id) assert result["status"] == "cancelled" - record, _ = await FoundryWorkflowBindingStore(FoundryRequestScope("sandbox", None, None, False)).get_response( - response_id - ) + binding_store = FoundryWorkflowBindingStore(FoundryRequestScope("sandbox", None, None, False)) + # Executor shutdown can precede asynchronous persistence of the blocked binding. + for _ in range(150): + record, _ = await binding_store.get_response(response_id) + if record is not None and record.status == "blocked": + break + await asyncio.sleep(0.01) assert record is not None and record.status == "blocked"