diff --git a/python/packages/core/agent_framework/_mcp.py b/python/packages/core/agent_framework/_mcp.py index 3dbfca07fdc..031b2affeb8 100644 --- a/python/packages/core/agent_framework/_mcp.py +++ b/python/packages/core/agent_framework/_mcp.py @@ -10,6 +10,8 @@ import sys import warnings from abc import abstractmethod +from array import array +from bisect import bisect_left from collections.abc import AsyncGenerator, Awaitable, Callable, Collection, Coroutine, Generator, Mapping, Sequence from contextlib import AsyncExitStack, _AsyncGeneratorContextManager, asynccontextmanager, contextmanager # type: ignore from contextvars import ContextVar @@ -46,6 +48,7 @@ _FunctionResultPayloadBudget, # pyright: ignore[reportPrivateUsage] ) from ._types import ( + Annotation, ChatOptions, Content, Message, @@ -194,7 +197,11 @@ class MCPSpecificApproval(TypedDict, total=False): MCP servers disagree on whether the two fields are duplicates or complementary (#7866). Callers pick an explicit policy: -- ``structured_first`` (default): use ``structuredContent`` when present, else ``content`` +- ``structured_first`` (default): use ``structuredContent`` when present, else ``content``. When the result + has text content and ``url_citation`` entries in ``structuredContent.annotations`` or a content block's + ``_meta.annotations`` (for example, Foundry Toolbox web search), the text content is kept before + ``structuredContent`` and the citations are attached as framework annotations. Without text content, citations + are attached without spans to the selected structured result, when present. - ``content_first``: use ``content`` when non-empty, else ``structuredContent`` - ``content_only``: ignore ``structuredContent`` for the model-visible result - ``structured_only``: ignore ``content`` for the model-visible result @@ -204,6 +211,161 @@ class MCPSpecificApproval(TypedDict, total=False): Custom ``parse_tool_results`` overrides this policy entirely. """ +_MCPCitationKey: TypeAlias = tuple[str, str | None, int | None, int | None] + + +def _utf16_surrogate_offsets(text: str) -> tuple[int, array[int]]: + """Return UTF-16 length and low-surrogate offsets belonging to pairs.""" + offset = 0 + previous_high = False + low_offsets = array("Q") + for character in text: + code = ord(character) + if code > 0xFFFF: + low_offsets.append(offset + 1) + offset += 2 + previous_high = False + else: + if previous_high and 0xDC00 <= code <= 0xDFFF: + low_offsets.append(offset) + previous_high = 0xD800 <= code <= 0xDBFF + offset += 1 + return offset, low_offsets + + +def _parse_mcp_url_citation(value: Any) -> tuple[Annotation, _MCPCitationKey] | None: + """Validate an MCP ``url_citation`` entry.""" + if not isinstance(value, Mapping): + return None + raw = cast(Mapping[str, Any], value) + url = raw.get("url") + if raw.get("type") != "url_citation" or not isinstance(url, str) or not url.strip(): + return None + + raw_title = raw.get("title") + title = raw_title if isinstance(raw_title, str) else None + raw_start, raw_end = raw.get("start_index"), raw.get("end_index") + start = raw_start if type(raw_start) is int else None + end = raw_end if type(raw_end) is int else None + citation = Annotation(type="citation", url=url) + if title is not None: + citation["title"] = title + return citation, (url, title, start, end) + + +def _attach_mcp_citation_span( + citation: Annotation, key: _MCPCitationKey, text_utf16_length: int, low_offsets: Sequence[int] +) -> None: + _, _, start, end = key + if start is not None and end is not None and 0 <= start <= end <= text_utf16_length: + # Citation offsets use UTF-16 code units; framework spans use Python text offsets. + start_pairs = bisect_left(low_offsets, start) + end_pairs = bisect_left(low_offsets, end) + if (start_pairs == len(low_offsets) or low_offsets[start_pairs] != start) and ( + end_pairs == len(low_offsets) or low_offsets[end_pairs] != end + ): + citation["annotated_regions"] = [ + { + "type": "text_span", + "start_index": start - start_pairs, + "end_index": end - end_pairs, + } + ] + + +def _mcp_block_annotations(item: Any) -> list[Any]: + """Collect annotation metadata from an MCP content block and its embedded resource.""" + metadata: list[Any] = [getattr(item, "meta", None)] + resource = getattr(item, "resource", None) + if resource is not None: + metadata.append(getattr(resource, "meta", None)) + annotations: list[Any] = [] + for entry in metadata: + if isinstance(entry, Mapping): + values = cast(Mapping[str, Any], entry).get("annotations") + if isinstance(values, list): + annotations.extend(cast(list[Any], values)) + return annotations + + +def _project_mcp_url_citations( + mcp_type: types.CallToolResult, + content_items: list[Content], + structured_block: Content | None, +) -> list[Content] | None: + """Keep cited text content before structured content and attach its URL citations as annotations. + + When there is no text content, attach recognized citations without spans to the structured block, if present. + Returns ``None`` when the result is an error or has no recognized ``url_citation`` entries. ``content_items`` + must hold exactly one parsed item per MCP content block. + """ + if mcp_type.is_error: + return None + structured: Any = mcp_type.structured_content + structured_values = ( + cast(Mapping[str, Any], structured).get("annotations") if isinstance(structured, Mapping) else None + ) + structured_annotations = cast(list[Any], structured_values) if isinstance(structured_values, list) else [] + block_annotations = [_mcp_block_annotations(item) for item in mcp_type.content] + if not any( + _parse_mcp_url_citation(value) is not None + for value in [*structured_annotations, *(value for values in block_annotations for value in values)] + ): + return None + has_text_content = any(item.type == "text" and item.text for item in content_items) + if not has_text_content and structured_block is not None: + citations: list[Annotation] = [] + seen: set[_MCPCitationKey] = set() + all_annotations = [ + *structured_annotations, + *(value for values in block_annotations for value in values), + ] + for value in all_annotations: + parsed = _parse_mcp_url_citation(value) + if parsed is not None and parsed[1] not in seen: + citations.append(parsed[0]) + seen.add(parsed[1]) + if citations: + structured_block.annotations = [*(structured_block.annotations or []), *citations] + return [structured_block] + + # Match aggregate and block-owned occurrences without collapsing repeated citations. + block_citations: dict[_MCPCitationKey, int] = {} + for item, values in zip(content_items, block_annotations, strict=True): + text_utf16_length, low_offsets = ( + _utf16_surrogate_offsets(item.text) + if values and item.type == "text" and item.text is not None + else (None, array("Q")) + ) + item_citations = list(item.annotations or []) + item_seen: set[_MCPCitationKey] = set() + for value in values: + parsed = _parse_mcp_url_citation(value) + if parsed is not None and parsed[1] not in item_seen: + if text_utf16_length is not None: + _attach_mcp_citation_span(parsed[0], parsed[1], text_utf16_length, low_offsets) + item_citations.append(parsed[0]) + item_seen.add(parsed[1]) + block_citations[parsed[1]] = block_citations.get(parsed[1], 0) + 1 + if item_citations: + item.annotations = item_citations + + unbound: list[Annotation] = [] + for value in structured_annotations: + parsed = _parse_mcp_url_citation(value) + if parsed is None: + continue + if block_citations.get(parsed[1]): + block_citations[parsed[1]] -= 1 + else: + unbound.append(parsed[0]) + if structured_block is None: + return content_items + if unbound: + structured_block.annotations = [*(structured_block.annotations or []), *unbound] + return [*content_items, structured_block] + + _MCP_REMOTE_NAME_KEY = "_mcp_remote_name" _MCP_IS_TOOL_KEY = "_mcp_is_tool" _MCP_NORMALIZED_NAME_KEY = "_mcp_normalized_name" @@ -1521,7 +1683,10 @@ def _parse_tool_result_from_mcp( if not result and structured_block is not None: result.append(structured_block) elif mode == "structured_first": - if structured_block is not None: + cited = _project_mcp_url_citations(mcp_type, result, structured_block) + if cited is not None: + result = cited + elif structured_block is not None: result = [structured_block] else: # pragma: no cover - validated in __init__ raise ValueError(f"Unknown tool_result_content mode: {mode!r}") diff --git a/python/packages/core/tests/core/test_mcp.py b/python/packages/core/tests/core/test_mcp.py index 9362851adb1..485548b327f 100644 --- a/python/packages/core/tests/core/test_mcp.py +++ b/python/packages/core/tests/core/test_mcp.py @@ -1412,6 +1412,98 @@ def test_parse_tool_result_content_modes_for_complementary_and_duplicate_payload assert content_only._parse_tool_result_from_mcp(empty)[0].text == "null" +def test_structured_first_keeps_cited_content_and_attaches_url_citations(): + """Default MCP parsing keeps search text when url_citation metadata is present (for example, Foundry Toolbox).""" + block_citation = { + "type": "url_citation", + "url": "https://example.test/block", + "title": "Block", + "start_index": 0, + "end_index": 4, + } + aggregate_citation = {"type": "url_citation", "url": "https://example.test/aggregate"} + structured = {"annotations": [block_citation, aggregate_citation, {"type": "other"}]} + mcp_result = types.CallToolResult( + content=[types.TextContent(type="text", text="Mars is red.", _meta={"annotations": [block_citation]})], + structured_content=structured, + _meta={"source": "server"}, + ) + + result = _HELPER_MCP_TOOL._parse_tool_result_from_mcp(mcp_result) + + assert [item.text for item in result] == ["Mars is red.", json.dumps(structured)] + assert result[0].annotations == [ + { + "type": "citation", + "url": "https://example.test/block", + "title": "Block", + "annotated_regions": [{"type": "text_span", "start_index": 0, "end_index": 4}], + } + ] + assert result[1].annotations == [{"type": "citation", "url": "https://example.test/aggregate"}] + assert all(item.additional_properties["_meta"] == {"source": "server"} for item in result) + + +def test_cited_text_converts_multiple_utf16_spans_and_deduplicates(): + text = "A\U0001f30eB\U0001f680C" + citations = [ + {"type": "url_citation", "url": "https://example.test/earth", "start_index": 1, "end_index": 3}, + {"type": "url_citation", "url": "https://example.test/rocket", "start_index": 4, "end_index": 6}, + {"type": "url_citation", "url": "https://example.test/invalid", "start_index": 2, "end_index": 6}, + ] + mcp_result = types.CallToolResult( + content=[types.TextContent(type="text", text=text, _meta={"annotations": [*citations, *citations]})] + ) + + result = _HELPER_MCP_TOOL._parse_tool_result_from_mcp(mcp_result) + + assert result[0].annotations == [ + { + "type": "citation", + "url": "https://example.test/earth", + "annotated_regions": [{"type": "text_span", "start_index": 1, "end_index": 2}], + }, + { + "type": "citation", + "url": "https://example.test/rocket", + "annotated_regions": [{"type": "text_span", "start_index": 3, "end_index": 4}], + }, + {"type": "citation", "url": "https://example.test/invalid"}, + ] + + +def test_structured_first_does_not_attach_url_citations_to_error_results(): + mcp_result = types.CallToolResult( + content=[types.TextContent(type="text", text="Search failed")], + structured_content={"annotations": [{"type": "url_citation", "url": "https://example.test"}]}, + is_error=True, + ) + + result = _HELPER_MCP_TOOL._parse_tool_result_from_mcp(mcp_result) + + assert len(result) == 1 + assert result[0].annotations is None + + +@pytest.mark.parametrize( + "content", + [ + [], + [types.TextContent(type="text", text="")], + ], +) +def test_structured_first_attaches_unspanned_citations_without_search_text(content): + citation = {"type": "url_citation", "url": "https://example.test/source", "title": "Source"} + structured = {"annotations": [citation]} + mcp_result = types.CallToolResult(content=content, structured_content=structured) + + result = _HELPER_MCP_TOOL._parse_tool_result_from_mcp(mcp_result) + + assert len(result) == 1 + assert result[0].text == json.dumps(structured) + assert result[0].annotations == [{"type": "citation", "url": "https://example.test/source", "title": "Source"}] + + async def test_generated_mcp_tool_preserves_complete_host_payload_once() -> None: """The generated FunctionTool path retains one complete, persistent Host payload.""" file_meta = { diff --git a/python/packages/foundry_hosting/README.md b/python/packages/foundry_hosting/README.md index f5fe6c49b17..0d02404e200 100644 --- a/python/packages/foundry_hosting/README.md +++ b/python/packages/foundry_hosting/README.md @@ -142,6 +142,14 @@ supplied through their configured Toolbox. The private metadata parser is intend Toolbox; reusing it elsewhere provides no security or compatibility guarantees. It is not a general-purpose MCP approval policy. +## Toolbox search citations + +`FoundryToolbox` and generic MCP tools use the default `structured_first` result handling for Toolbox search results. +When a result contains text content and `url_citation` annotations, the text stays visible to the model before the +structured JSON, and the citations are attached to the tool result. If the result has citations but no text content, +they are attached without spans to the selected structured result. Another `tool_result_content` mode or a custom +`parse_tool_results` replaces this handling, and hosted answers then receive no Toolbox citations. + ## Agent instances and factories `ResponsesHostServer` and `InvocationsHostServer` accept an agent instance or a callable through the @@ -446,6 +454,13 @@ Completed provider responses retain separate text content parts, keeping citatio their original part. Ordinary streaming deltas still merge within their existing text occurrence, without shifting citation offsets. +Web URL citations on function results, such as `FoundryToolbox` search results, are listed as sources on the +final answer text when the response completes. These citations do not identify which part of the answer each +source supports, so the host emits each one as a zero-length `url_citation` at the end of that text. Sources are +deduplicated by URL, limited to `http` and `https`, skipped when the answer already cites the same URL, and not +attached to interim text. A response retains at most 100 sources and 64 KiB of UTF-8 URL and title data; +additional or oversized sources are skipped with a warning. Plain `agent.run()` responses do not include these sources. + ## State store ### Local persistence diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses_agent.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses_agent.py index df0f9b314f9..9ad174718f2 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses_agent.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses_agent.py @@ -688,7 +688,7 @@ async def _handle_prepared_response( # below so its partial output is not misreported as a failure. return - for event in tracker.close(): + for event in tracker.close(final_answer=True): yield event if cancellation_signal.is_set() and context.client_cancelled: @@ -699,7 +699,7 @@ async def _handle_prepared_response( # selecting the terminal event. Same ``client_cancelled`` gate as above. return - tracker.discard_pending_container_file_citations() + tracker.discard_pending_citations() incomplete_reason = tracker.incomplete_reason if tracker.oauth_consent_requested or incomplete_reason is not None: yield response_event_stream.emit_incomplete(reason=incomplete_reason, usage=tracker.usage) @@ -1124,7 +1124,9 @@ async def publish_update( yield event if context.shutdown.is_set() or cancellation_signal.is_set(): continue - for event in tracker.close(): + # Unfinished provider outputs end at a function result, so text is still + # active here only when it ends the finished answer. + for event in tracker.close(final_answer=True): yield event private_metadata = _hosted_internal_metadata(response_event_stream) private_metadata[_HOSTED_PROVIDER_OUTPUT_COUNT_KEY] = output_count @@ -1517,7 +1519,7 @@ def _emit_failure( yield from tracker.close() except Exception as close_error: _log_response_failure("failed tracker cleanup", response_id, close_error) - tracker.discard_pending_container_file_citations() + tracker.discard_pending_citations() if public_message is not None: message = public_message elif isinstance(ex, _PublicResponseFailure): diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses_shared.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses_shared.py index 1639c3d575a..bdf46f63700 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses_shared.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses_shared.py @@ -15,13 +15,16 @@ Awaitable, Callable, Generator, + Iterator, Mapping, Sequence, ) from contextlib import suppress from dataclasses import asdict, dataclass, is_dataclass from decimal import Decimal +from itertools import chain from typing import Any, Generic, Literal, TypeVar, cast +from urllib.parse import urlparse from agent_framework import ( AgentResponse, @@ -111,6 +114,9 @@ _HOSTED_PROVIDER_OUTPUT_COUNT_KEY = "_foundry_provider_output_count" _HOSTED_PROVIDER_USAGE_KEY = "_foundry_provider_usage" _PENDING_CONTAINER_FILE_CITATIONS_KEY = "_foundry_pending_container_file_citations" +_PENDING_URL_CITATIONS_KEY = "_foundry_pending_url_citations" +_MAX_PENDING_URL_CITATIONS = 100 +_MAX_PENDING_URL_CITATION_BYTES = 64 * 1024 _CONTAINER_FILE_CITATIONS_KEY = "container_file_citations" _PUBLIC_STREAM_FAILURE_MESSAGE = "An internal server error occurred." @@ -242,6 +248,30 @@ def _container_file_citations_from_function_result(content: Content) -> list[_Co return citations +def _url_citations_from_function_result(content: Content) -> Iterator[tuple[str, str]]: + """Yield web sources cited by a function result in their original order.""" + candidates = chain( + content.annotations or [], chain.from_iterable(item.annotations or [] for item in content.items or []) + ) + for annotation in candidates: + if not isinstance(annotation, Mapping): + continue + annotation_mapping = cast(Mapping[str, Any], annotation) + if annotation_mapping.get("type") != "citation" or annotation_mapping.get("file_id"): + continue + url = annotation_mapping.get("url") + if not isinstance(url, str): + continue + try: + scheme = urlparse(url).scheme.lower() + except ValueError: + continue + if scheme not in {"http", "https"}: + continue + title = annotation_mapping.get("title") + yield url, title if isinstance(title, str) else "" + + def _annotations_to_output_text( annotations: Sequence[FrameworkAnnotation] | None, ) -> list[ResponsesAnnotation]: @@ -686,6 +716,8 @@ def __init__( self._outstanding_computer_calls: set[str] = set() self._oauth_consent_requests: set[tuple[str, str]] = set() self._pending_container_file_citations: dict[str, _ContainerFileCitation] = {} + self._pending_url_citations: dict[str, str] = {} + self._pending_url_citation_bytes = 0 self._text_annotations: list[ResponsesAnnotation] = [] self._message_text_annotations: dict[int, list[ResponsesAnnotation]] = {} # Set when an agent update reports the model stopped early (content filter, token @@ -725,6 +757,25 @@ def __init__( file_id=file_id, filename=filename, ) + persisted_url_citations = _hosted_internal_metadata(stream).get(_PENDING_URL_CITATIONS_KEY) + if persisted_url_citations is not None: + if not isinstance(persisted_url_citations, Sequence) or isinstance( + persisted_url_citations, (str, bytes, bytearray) + ): + raise RuntimeError("The persisted URL citations are invalid.") + skipped_url_citation = False + for raw_url_citation in cast(Sequence[Any], persisted_url_citations): + if not isinstance(raw_url_citation, Mapping): + raise RuntimeError("The persisted URL citations are invalid.") + url_citation = cast(Mapping[str, Any], raw_url_citation) + url = url_citation.get("url") + title = url_citation.get("title") + if not isinstance(url, str) or not url or not isinstance(title, str): + raise RuntimeError("The persisted URL citations are invalid.") + skipped_url_citation |= not self._retain_pending_url_citation(url, title) + if skipped_url_citation: + logger.warning("Pending URL citations exceeded the hosted source limits; excess sources were skipped.") + self._persist_pending_url_citations() for item in stream.response.get("output", []): if not isinstance(item, Mapping): continue @@ -918,6 +969,15 @@ async def handle( for citation in citations: self._pending_container_file_citations[citation.filename] = citation self._persist_pending_container_file_citations() + skipped_url_citation = False + had_url_citations = False + for url, title in _url_citations_from_function_result(content): + had_url_citations = True + skipped_url_citation |= not self._retain_pending_url_citation(url, title) + if skipped_url_citation: + logger.warning("Pending URL citations exceeded the hosted source limits; excess sources were skipped.") + if had_url_citations: + self._persist_pending_url_citations() async for event in self._stream.output_item_function_call_output( content.call_id, # type: ignore[arg-type] _json_safe_to_str(content.result), @@ -1164,14 +1224,21 @@ async def handle( # response stream isn't broken by one unsupported content item. logger.warning(f"Content type '{content.type}' is not supported yet. This is usually safe to ignore.") - def close(self) -> Generator[ResponseStreamEvent]: - """Flush any remaining active builder without discarding recoverable state.""" - yield from self._close() + def close(self, *, final_answer: bool = False) -> Generator[ResponseStreamEvent]: + """Flush any remaining active builder without discarding recoverable state. + + Set ``final_answer`` only when the active text ends the agent's answer, so pending tool-result + sources attach there instead of to interim text. + """ + yield from self._close(attach_sources=final_answer) - def discard_pending_container_file_citations(self) -> None: + def discard_pending_citations(self) -> None: """Discard unmatched citations immediately before a terminal response event.""" self._pending_container_file_citations.clear() self._persist_pending_container_file_citations() + self._pending_url_citations.clear() + self._pending_url_citation_bytes = 0 + self._persist_pending_url_citations() # -- Private open/close helpers -- @@ -1260,9 +1327,9 @@ def _open_mcp_call(self, content: Content) -> Generator[ResponseStreamEvent]: self._active_id = content.call_id or f"{content.server_name or 'default'}::{content.tool_name}" yield self._mcp_builder.emit_added() - def _close(self) -> Generator[ResponseStreamEvent]: + def _close(self, *, attach_sources: bool = False) -> Generator[ResponseStreamEvent]: if self._active_type in {"text", "refusal"}: - yield from self._close_message_content() + yield from self._close_message_content(attach_sources=attach_sources) if self._message_item is not None: message_done = self._message_item.emit_done() if self._message_text_annotations: @@ -1320,13 +1387,18 @@ def _close(self) -> Generator[ResponseStreamEvent]: self._active_message_id = None self._accumulated.clear() - def _close_message_content(self) -> Generator[ResponseStreamEvent]: + def _close_message_content(self, *, attach_sources: bool = False) -> Generator[ResponseStreamEvent]: accumulated = "".join(self._accumulated) if self._active_type == "text" and self._text_content is not None: annotations = list(self._text_annotations) for annotation in self._container_file_annotations_for_text(accumulated): if annotation not in annotations: annotations.append(annotation) + if attach_sources: + earlier_annotations = chain.from_iterable(self._message_text_annotations.values()) + annotations.extend( + self._source_url_annotations_for_text(accumulated, [*earlier_annotations, *annotations]) + ) yield self._text_content.emit_text_done(accumulated) for annotation in annotations: yield self._text_content.emit_annotation_added(annotation) @@ -1392,6 +1464,53 @@ def _container_file_annotations_for_text(self, text: str) -> list[ContainerFileC self._persist_pending_container_file_citations() return annotations + def _source_url_annotations_for_text( + self, + text: str, + existing: Sequence[ResponsesAnnotation], + ) -> list[UrlCitationBody]: + """Cite pending tool-result sources at the end of the final answer text, without claiming a supported span.""" + if not text or not self._pending_url_citations: + return [] + cited_urls = { + cast(Mapping[str, Any], annotation).get("url") + for annotation in existing + if cast(Mapping[str, Any], annotation).get("type") == "url_citation" + } + end_index = len(text) + annotations = [ + UrlCitationBody(type="url_citation", url=url, title=title, start_index=end_index, end_index=end_index) + for url, title in self._pending_url_citations.items() + if url not in cited_urls + ] + self._pending_url_citations.clear() + self._pending_url_citation_bytes = 0 + self._persist_pending_url_citations() + return annotations + + def _retain_pending_url_citation(self, url: str, title: str) -> bool: + if url in self._pending_url_citations: + return True + if ( + len(self._pending_url_citations) >= _MAX_PENDING_URL_CITATIONS + or len(url) + len(title) > _MAX_PENDING_URL_CITATION_BYTES + ): + return False + size = len(url.encode("utf-8", errors="surrogatepass")) + len(title.encode("utf-8", errors="surrogatepass")) + if self._pending_url_citation_bytes + size > _MAX_PENDING_URL_CITATION_BYTES: + return False + self._pending_url_citations[url] = title + self._pending_url_citation_bytes += size + return True + + def _persist_pending_url_citations(self) -> None: + if not self._pending_url_citations: + _hosted_internal_metadata(self._stream).pop(_PENDING_URL_CITATIONS_KEY, None) + return + _hosted_internal_metadata(self._stream)[_PENDING_URL_CITATIONS_KEY] = [ + {"url": url, "title": title} for url, title in self._pending_url_citations.items() + ] + def _persist_pending_container_file_citations(self) -> None: if not self._pending_container_file_citations: _hosted_internal_metadata(self._stream).pop(_PENDING_CONTAINER_FILE_CITATIONS_KEY, None) diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses_workflow.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses_workflow.py index f63307c281b..dd13d1cf459 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses_workflow.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses_workflow.py @@ -583,7 +583,7 @@ async def abort() -> bool: buffered: list[WorkflowEvent[Any]] = [] tracker = _OutputItemTracker(stream, self._allowed_origins) - async def flush(checkpoint_id: str | None) -> list[ResponseStreamEvent]: + async def flush(checkpoint_id: str | None, *, final_answer: bool = False) -> list[ResponseStreamEvent]: nonlocal stream, tracker, aliases checkpoint = await run.get_checkpoint(checkpoint_id) if run.stored else None pending = checkpoint.pending_request_info_events if checkpoint is not None else {} @@ -597,7 +597,10 @@ async def flush(checkpoint_id: str | None) -> list[ResponseStreamEvent]: events: list[ResponseStreamEvent] = [] for event in buffered: events.extend(await _project(event, draft, next_tracker, checkpoint, next_aliases)) - events.extend(next_tracker.close()) + is_final_output = final_answer and bool(buffered) and buffered[-1].type == "output" + events.extend(next_tracker.close(final_answer=is_final_output)) + if final_answer: + next_tracker.discard_pending_citations() _hosted_internal_metadata(draft)[_HOSTED_PROVIDER_USAGE_KEY] = next_tracker.usage_details await run.stage(_snapshot(draft), approvals=next_aliases, checkpoint_id=checkpoint_id) stream, tracker, aliases = draft, next_tracker, next_aliases @@ -609,6 +612,8 @@ async def checkpoint_stamp() -> str | None: # ruff: ignore[unused-async] phase = "execution" if not run.has_completed_output: + pending_flush = False + pending_checkpoint: str | None = None iterator = _SignalledIterator( run.events(turn, client_kwargs=client_kwargs, function_invocation_kwargs=function_kwargs), context.shutdown, @@ -617,6 +622,18 @@ async def checkpoint_stamp() -> str | None: # ruff: ignore[unused-async] ) async with aclosing(iterator): async for event in iterator: + if pending_flush and event.type in ( + "superstep_started", + "request_info", + "output", + "intermediate", + "data", + ): + for output_event in await flush(pending_checkpoint): + yield wire(output_event) + if resilient: + yield stream.checkpoint() + pending_flush = False if ( event.type == "request_info" and event.request_id not in represented_pending ) or event.type in ("output", "intermediate", "data"): @@ -626,10 +643,8 @@ async def checkpoint_stamp() -> str | None: # ruff: ignore[unused-async] if resilient: yield stream.checkpoint() elif event.type == "superstep_completed": - for output_event in await flush(iterator.stamp): - yield wire(output_event) - if resilient: - yield stream.checkpoint() + pending_checkpoint = iterator.stamp + pending_flush = True if iterator.signalled: if context.shutdown.is_set() and resilient and run.snapshot is not None: preserve_for_recovery = True @@ -638,13 +653,19 @@ async def checkpoint_stamp() -> str | None: # ruff: ignore[unused-async] await abort() return raise _WorkflowRequestError("The workflow stopped without a recoverable background pair.") + if pending_flush: + for output_event in await flush(pending_checkpoint, final_answer=True): + yield wire(output_event) + if resilient: + yield stream.checkpoint() if cancellation_signal.is_set(): await abort() return phase = "final output persistence" if buffered: - for event in await flush(run.checkpoint_id): + for event in await flush(run.checkpoint_id, final_answer=True): yield wire(event) + tracker.discard_pending_citations() terminal = ( stream.emit_incomplete(reason=tracker.incomplete_reason, usage=tracker.usage) if tracker.oauth_consent_requested or tracker.incomplete_reason is not None diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py index d82a69c85e1..c578c7420a7 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py @@ -367,6 +367,13 @@ class FoundryToolbox(MCPStreamableHTTPTool): - forwards the platform per-request call-id (``x-agent-foundry-call-id``) so the Foundry MCP proxy can resolve the caller context server-side. + Web-search results with URL citation metadata use the core MCP ``structured_first`` + handling: content blocks stay model-visible before the structured content, and the + citations become framework annotations. Aggregate citations without an identified + content block remain attached to the structured content without spans. Citation spans + refer to the search result, not a later assistant answer. Another ``tool_result_content`` + mode or a custom ``parse_tool_results`` replaces this handling. + Platform headers are resolved at each operation boundary before work crosses into the MCP connection's writer task. The complete effective header set is bound to the MCP session; when the caller context changes, the Toolbox diff --git a/python/packages/foundry_hosting/tests/foundry_hosting/test_native_responses.py b/python/packages/foundry_hosting/tests/foundry_hosting/test_native_responses.py index a00942579a5..3480c23015e 100644 --- a/python/packages/foundry_hosting/tests/foundry_hosting/test_native_responses.py +++ b/python/packages/foundry_hosting/tests/foundry_hosting/test_native_responses.py @@ -49,6 +49,11 @@ ) from agent_framework_foundry_hosting._shared import FoundryWorkflowBindingStore, HostedWorkflowRun +from .web_search_test_helpers import ( # pyrefly: ignore[missing-import] - pytest's importlib namespace + _CitationWorkflowExecutor, + _expected_source_citations, +) + class _Ticket(BaseModel): text: str @@ -645,6 +650,145 @@ async def start(self, message: str, ctx: WorkflowContext[str, Any]) -> None: assert message["content"][0]["annotations"] == [_expected_url_citation(start_index=4, end_index=10)] +@pytest.mark.parametrize("stored", [False, True]) +async def test_native_workflow_attaches_tool_sources_only_to_final_answer(stored: bool) -> None: + def create(request: ResponsesRequest) -> Workflow: + source = _CitationWorkflowExecutor() + return WorkflowBuilder(name="citation-workflow", start_executor=source).add_edge(source, source).build() + + server = _server(create, parser=_parse_count) + response = await _post(server, "1", store=stored) + + assert response["status"] == "completed", response + messages = [item for item in response["output"] if item["type"] == "message"] + assert [item["content"][0]["text"] for item in messages] == ["Checking.", "Final answer"] + assert messages[0]["content"][0]["annotations"] == [] + assert messages[1]["content"][0]["annotations"] == _expected_source_citations("Final answer") + assert "_foundry_pending_url_citations" not in response.get("internal_metadata", {}) + + +async def test_native_workflow_streams_final_sources_consistently() -> None: + def create(request: ResponsesRequest) -> Workflow: + source = _CitationWorkflowExecutor() + return WorkflowBuilder(name="citation-stream", start_executor=source).add_edge(source, source).build() + + server = _server(create, parser=_parse_count) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=server), base_url="http://test") as client: + response = await client.post("/responses", json={"input": "1", "store": False, "stream": True}) + + assert response.status_code == 200 + events = [ + json.loads(line[5:].strip()) + for line in response.text.splitlines() + if line.startswith("data:") and line[5:].strip() != "[DONE]" + ] + expected = _expected_source_citations("Final answer") + assert [event["annotation"] for event in events if event["type"] == "response.output_text.annotation.added"] == ( + expected + ) + done = [event for event in events if event["type"] == "response.content_part.done"] + assert [event["part"]["annotations"] for event in done] == [[], expected] + completed = next(event["response"] for event in events if event["type"] == "response.completed") + messages = [item for item in completed["output"] if item["type"] == "message"] + assert messages[-1]["content"][0]["annotations"] == expected + + +async def test_native_workflow_discards_sources_without_a_final_answer() -> None: + def create(request: ResponsesRequest) -> Workflow: + source = _CitationWorkflowExecutor(final_text=None) + return WorkflowBuilder(name="citation-no-answer", start_executor=source).add_edge(source, source).build() + + committed: list[dict[str, Any]] = [] + original = HostedWorkflowRun.commit + + async def commit(run: HostedWorkflowRun, snapshot: Any = None) -> None: + committed.append(snapshot) + await original(run, snapshot) + + with patch.object(HostedWorkflowRun, "commit", commit): + response = await _post(_server(create, parser=_parse_count), "1") + + assert response["status"] == "completed", response + message = next(item for item in response["output"] if item["type"] == "message") + assert message["content"][0]["annotations"] == [] + assert "_foundry_pending_url_citations" not in committed[-1].get("internal_metadata", {}) + + +async def test_native_workflow_does_not_cite_terminal_intermediate_output() -> None: + def create(request: ResponsesRequest) -> Workflow: + source = _CitationWorkflowExecutor() + return ( + WorkflowBuilder( + name="citation-intermediate", + start_executor=source, + output_from=[], + intermediate_output_from="all", + ) + .add_edge(source, source) + .build() + ) + + response = await _post(_server(create, parser=_parse_count), "1", store=False) + + assert response["status"] == "completed", response + messages = [item for item in response["output"] if item["type"] == "message"] + assert [item["content"][0]["annotations"] for item in messages] == [[], []] + + +async def test_native_workflow_recovers_pending_sources(tmp_path: Any, monkeypatch: pytest.MonkeyPatch) -> None: + from azure.ai.agentserver.core.tasks import resilient_tasks_enabled, set_resilient_tasks_enabled + + enabled = resilient_tasks_enabled() + set_resilient_tasks_enabled(True) + monkeypatch.setenv("AGENTSERVER_SHUTDOWN_GRACE_SECONDS", "0.1") + started, release, stopped, paired = (asyncio.Event() for _ in range(4)) + + def create(request: ResponsesRequest) -> Workflow: + source = _CitationWorkflowExecutor(started, release, stopped) + return WorkflowBuilder(name="citation-recovery", start_executor=source).add_edge(source, source).build() + + provider = FileResponseStore(storage_dir=tmp_path / "responses") + options: dict[str, Any] = {"resilient_background": True, "shutdown_grace_period_seconds": 1} + server = _server(create, parser=_parse_count, response_store=provider, **options) + original = HostedWorkflowRun.stage + + async def stage(run: HostedWorkflowRun, snapshot: Any, **kwargs: Any) -> None: + await original(run, snapshot, **kwargs) + if snapshot.get("internal_metadata", {}).get("_foundry_pending_url_citations"): + paired.set() + + try: + with patch.object(HostedWorkflowRun, "stage", stage): + async with ( + server.router.lifespan_context(server), + httpx.AsyncClient(transport=httpx.ASGITransport(app=server), base_url="http://test") as client, + ): + pending = await client.post( + "/responses", + json={"input": "1", "agent_session_id": "sandbox", "store": True, "background": True}, + ) + assert pending.status_code == 200 + response_id = pending.json()["id"] + await asyncio.wait_for(started.wait(), timeout=3) + await asyncio.wait_for(paired.wait(), timeout=3) + assert stopped.is_set() + release.set() + restarted = _server(create, parser=_parse_count, response_store=provider, **options) + async with ( + restarted.router.lifespan_context(restarted), + httpx.AsyncClient(transport=httpx.ASGITransport(app=restarted), base_url="http://test") as client, + ): + result = await _poll(client, response_id) + assert result["status"] == "completed", result + messages = [item for item in result["output"] if item["type"] == "message"] + assert [item["content"][0]["text"] for item in messages] == ["Checking.", "Final answer"] + assert messages[0]["content"][0]["annotations"] == [] + assert messages[1]["content"][0]["annotations"] == _expected_source_citations("Final answer") + finally: + release.set() + set_resilient_tasks_enabled(enabled) + + async def test_native_agent_executor_applies_options_without_mutating_defaults_or_double_keywords() -> None: calls: list[dict[str, Any]] = [] lifecycle: list[str] = [] @@ -897,9 +1041,13 @@ async def test_real_sdk_background_cancel_drains_suspended_executor_and_blocks_c await asyncio.wait_for(stopped.wait(), timeout=2) result = await _poll(client, response_id) assert result["status"] == "cancelled" - record, _ = await FoundryWorkflowBindingStore(FoundryRequestScope("sandbox", None, None, False)).get_response( - response_id - ) + binding_store = FoundryWorkflowBindingStore(FoundryRequestScope("sandbox", None, None, False)) + # Executor shutdown can precede asynchronous persistence of the blocked binding. + for _ in range(150): + record, _ = await binding_store.get_response(response_id) + if record is not None and record.status == "blocked": + break + await asyncio.sleep(0.01) assert record is not None and record.status == "blocked" diff --git a/python/packages/foundry_hosting/tests/foundry_hosting/web_search_test_helpers.py b/python/packages/foundry_hosting/tests/foundry_hosting/web_search_test_helpers.py new file mode 100644 index 00000000000..a81e9a1a849 --- /dev/null +++ b/python/packages/foundry_hosting/tests/foundry_hosting/web_search_test_helpers.py @@ -0,0 +1,154 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Reusable offline test helpers for web-search citations.""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import httpx2 as httpx +from agent_framework import ( + AgentResponseUpdate, + Annotation, + Content, + Executor, + Message, + WorkflowContext, + handler, +) + +from .toolbox_approval_helpers import ( # pyrefly: ignore[missing-import] - pytest's importlib namespace + ToolboxApprovalServer, +) + +_PYTHON_SOURCE_URL = "https://www.python.org/downloads/release/python-31315/" +_DOCS_SOURCE_URL = "https://docs.python.org/3/whatsnew/3.13.html" + + +class _ToolResultServer(ToolboxApprovalServer): + """Offline Toolbox server whose ``tools/call`` response is configurable per test.""" + + def __init__(self) -> None: + super().__init__() + self.call_result: dict[str, Any] = {} + + async def handle(self, request: httpx.Request) -> httpx.Response: + if request.method == "POST": + payload = json.loads(request.content) + if payload.get("method") == "tools/call" and "id" in payload: + self.calls.append(payload["params"]) + return httpx.Response(200, json={"jsonrpc": "2.0", "id": payload["id"], "result": self.call_result}) + return await super().handle(request) + + +class _CitationWorkflowExecutor(Executor): + """Emit a cited tool result before the final answer in a later superstep.""" + + def __init__( + self, + started: asyncio.Event | None = None, + release: asyncio.Event | None = None, + stopped: asyncio.Event | None = None, + *, + final_text: str | None = "Final answer", + ) -> None: + super().__init__("citation-workflow") + self.started = started + self.release = release + self.stopped = stopped + self.final_text = final_text + + @handler + async def run(self, step: int, ctx: WorkflowContext[int, Message]) -> None: + if step: + await ctx.yield_output(Message("tool", [_web_search_result("call_1")])) + await ctx.yield_output(Message("assistant", "Checking.")) + await ctx.send_message(0) + return + if self.started is not None: + self.started.set() + try: + if self.release is not None: + await self.release.wait() + finally: + if self.stopped is not None: + self.stopped.set() + if self.final_text is not None: + await ctx.yield_output(Message("assistant", self.final_text)) + + +def _web_search_result(call_id: str) -> Content: + """Build a search result with duplicate URLs and non-web citations for response-host tests.""" + return Content.from_function_result( + call_id, + result=[ + Content.from_text( + "Python 3.13.15 is the latest release.", + annotations=[ + _url_citation(), + Annotation(type="citation", url=_DOCS_SOURCE_URL, title="What's New"), + ], + ), + Content.from_text( + '{"annotations": []}', + annotations=[ + Annotation(type="citation", url=_PYTHON_SOURCE_URL, title="Duplicate title"), + Annotation(type="citation", url="javascript:alert(1)", title="Unsafe"), + Annotation(type="citation", url="notes.txt", file_id="file_123", title="File"), + ], + ), + ], + ) + + +def _expected_source_citations(text: str) -> list[dict[str, Any]]: + """Return the zero-length end-of-text citations expected for ``_web_search_result``.""" + return [ + _expected_url_citation(start_index=len(text), end_index=len(text)), + _expected_url_citation(url=_DOCS_SOURCE_URL, title="What's New", start_index=len(text), end_index=len(text)), + ] + + +def _web_search_citation_updates(answer: str) -> list[AgentResponseUpdate]: + return [ + AgentResponseUpdate( + contents=[Content.from_function_call("call_1", "web_search", arguments={})], + role="assistant", + message_id="msg_call", + ), + AgentResponseUpdate(contents=[_web_search_result("call_1")], role="tool", message_id="msg_result"), + AgentResponseUpdate(contents=[Content.from_text(answer)], role="assistant", message_id="msg_answer"), + ] + + +def _url_citation( + *, + url: str = _PYTHON_SOURCE_URL, + title: str = "Python 3.13.15", + start_index: int = 0, + end_index: int = 6, +) -> Annotation: + return Annotation( + type="citation", + url=url, + title=title, + annotated_regions=[{"type": "text_span", "start_index": start_index, "end_index": end_index}], + ) + + +def _expected_url_citation( + *, + url: str = _PYTHON_SOURCE_URL, + title: str = "Python 3.13.15", + start_index: int = 0, + end_index: int = 6, +) -> dict[str, Any]: + return { + "type": "url_citation", + "url": url, + "title": title, + "start_index": start_index, + "end_index": end_index, + } diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index 72cd8b3fd28..cd579bfcfea 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -114,6 +114,15 @@ consent_url_from_error, ) +from .foundry_hosting.web_search_test_helpers import ( # pyrefly: ignore[missing-import] - pytest's importlib namespace + _DOCS_SOURCE_URL, + _expected_source_citations, + _expected_url_citation, + _url_citation, + _web_search_citation_updates, + _web_search_result, +) + _OPENAI_HTTPX = cast(Any, import_module(DefaultAsyncHttpxClient.__mro__[1].__module__.partition(".")[0])) _PRIVATE_ERROR_DETAIL = "test-token-value at /srv/private/tool.py" _SESSION_STORAGE_ERROR_DETAIL = "session-storage-secret at C:\\sensitive\\session_store.py" @@ -235,37 +244,6 @@ def _expected_container_file_citation( } -def _url_citation( - *, - url: str = "https://www.python.org/downloads/release/python-31315/", - title: str = "Python 3.13.15", - start_index: int = 0, - end_index: int = 6, -) -> Annotation: - return Annotation( - type="citation", - url=url, - title=title, - annotated_regions=[{"type": "text_span", "start_index": start_index, "end_index": end_index}], - ) - - -def _expected_url_citation( - *, - url: str = "https://www.python.org/downloads/release/python-31315/", - title: str = "Python 3.13.15", - start_index: int = 0, - end_index: int = 6, -) -> dict[str, Any]: - return { - "type": "url_citation", - "url": url, - "title": title, - "start_index": start_index, - "end_index": end_index, - } - - def _file_citation( *, file_id: str = "file_123", @@ -2528,6 +2506,58 @@ async def run(messages: Any = None, *, session: AgentSession, **kwargs: Any) -> assert output_done["item"] == message assert response.to_dict() == original_response + async def test_provider_background_attaches_tool_result_sources_to_final_answer( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + answer = "Python 3.13.15 is out." + response = AgentResponse( + messages=[ + Message( + role="assistant", + contents=[ + Content.from_text("Let me search."), + Content.from_function_call("call_1", "web_search", arguments={}), + ], + ), + Message(role="tool", contents=[_web_search_result("call_1")]), + Message(role="assistant", contents=[Content.from_text(answer)]), + ], + finish_reason="stop", + ) + agent = Agent(client=_ServiceStorageRecordingClient()) + + async def run(messages: Any = None, *, session: AgentSession, **kwargs: Any) -> AgentResponse: + del messages, kwargs + session.service_session_id = "private-service-conversation" + return response + + monkeypatch.setattr(agent, "run", MagicMock(side_effect=run)) + server = _make_server( + agent, + session_store=SessionStore(), + history_source="inference_service", + background_source="inference_service", + resilient_background=True, + response_store=FileResponseStore(storage_dir=tmp_path), + ) + request = CreateResponse(input="hello", store=True, background=True) + context = ResponseContext(response_id="outer-response", mode_flags=MagicMock()) + + with patch.object(ResponseContext, "get_input_items", new=AsyncMock(return_value=[])): + events = [ + event + async for event in server._handle_response(request, context, asyncio.Event()) + if isinstance(event, Mapping) + ] + + completed = next(event["response"] for event in events if event["type"] == "response.completed") + messages = [item for item in completed["output"] if item["type"] == "message"] + parts = [part for message in messages for part in message["content"] if part["type"] == "output_text"] + assert len(parts) == len(messages) + assert [part["text"] for part in parts] == ["Let me search.", answer] + assert parts[0]["annotations"] == [] + assert parts[1]["annotations"] == _expected_source_citations(answer) + async def test_provider_background_keeps_private_token_under_outer_response( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -4498,6 +4528,56 @@ async def test_framework_url_and_file_citations_are_added_to_output_text(self) - _expected_file_citation(), ] + async def test_function_result_url_citations_wait_for_final_output_text(self) -> None: + answer = "Python 3.13.15 is out." + releases_url = "https://www.python.org/downloads/" + second_search = Content.from_function_result( + "call_2", + result=[ + Content.from_text( + "Release list.", annotations=[Annotation(type="citation", url=releases_url, title="Releases")] + ) + ], + ) + updates = [ + *_web_search_citation_updates("Let me search more.")[:-1], + AgentResponseUpdate( + contents=[Content.from_text("Let me search more.")], role="assistant", message_id="msg_interim" + ), + AgentResponseUpdate( + contents=[Content.from_function_call("call_2", "web_search", arguments={})], + role="assistant", + message_id="msg_call_2", + ), + AgentResponseUpdate(contents=[second_search], role="tool", message_id="msg_result_2"), + AgentResponseUpdate( + contents=[Content.from_text(answer, annotations=[_url_citation()])], + role="assistant", + message_id="msg_answer", + ), + ] + + resp = await _post(_make_server(_make_agent(stream_updates=updates)), stream=False) + + assert resp.status_code == 200 + output = resp.json()["output"] + assert [item["type"] for item in output] == [ + "function_call", + "function_call_output", + "message", + "function_call", + "function_call_output", + "message", + ] + assert output[2]["content"][0]["annotations"] == [] + assert output[5]["content"][0]["annotations"] == [ + _expected_url_citation(), + _expected_url_citation( + url=_DOCS_SOURCE_URL, title="What's New", start_index=len(answer), end_index=len(answer) + ), + _expected_url_citation(url=releases_url, title="Releases", start_index=len(answer), end_index=len(answer)), + ] + async def test_latest_pending_container_file_citation_replaces_same_filename(self) -> None: agent = _make_agent( stream_updates=[ @@ -5375,6 +5455,199 @@ async def test_framework_url_citation_is_consistent_across_stream_events(self) - final_message = next(item for item in completed["output"] if item["type"] == "message") assert final_message["content"][0]["annotations"] == [expected_annotation] + async def test_function_result_url_citations_are_consistent_across_stream_events(self) -> None: + answer = "Python 3.13.15 is out." + resp = await _post( + _make_server(_make_agent(stream_updates=_web_search_citation_updates(answer))), + stream=True, + ) + + assert resp.status_code == 200 + events = _parse_sse_events(resp.text) + expected = _expected_source_citations(answer) + annotation_events = [event for event in events if event["event"] == "response.output_text.annotation.added"] + assert [event["data"]["annotation"] for event in annotation_events] == expected + content_done = next(event for event in events if event["event"] == "response.content_part.done") + assert content_done["data"]["part"]["annotations"] == expected + completed = next(event["data"]["response"] for event in events if event["event"] == "response.completed") + final_message = next(item for item in completed["output"] if item["type"] == "message") + assert final_message["content"][0]["annotations"] == expected + + async def test_pending_url_citations_survive_recovery_and_are_discarded_at_completion(self) -> None: + stream = ResponseEventStream(response_id="resp_url_citations") + stream.emit_created() + stream.emit_in_progress() + tracker = _OutputItemTracker(stream) + _ = [event async for event in tracker.handle(Content.from_function_call("call_1", "web_search", arguments={}))] + _ = [event async for event in tracker.handle(_web_search_result("call_1"))] + _ = list(tracker.close()) + _ = stream.checkpoint() + + recovered = _OutputItemTracker(stream) + interim = [event async for event in recovered.handle(Content.from_text("Checking."), message_id="msg_interim")] + interim.extend(recovered.close()) + assert not any(event["type"] == "response.output_text.annotation.added" for event in interim) + + events = [event async for event in recovered.handle(Content.from_text("Done."), message_id="msg_final")] + events.extend(recovered.close(final_answer=True)) + annotations = [ + event["annotation"] for event in events if event["type"] == "response.output_text.annotation.added" + ] + assert annotations == _expected_source_citations("Done.") + + _ = [event async for event in recovered.handle(_web_search_result("call_2"))] + recovered.discard_pending_citations() + discarded = _OutputItemTracker(stream) + events = [event async for event in discarded.handle(Content.from_text("Later."), message_id="msg_later")] + events.extend(discarded.close(final_answer=True)) + assert not any(event["type"] == "response.output_text.annotation.added" for event in events) + + @pytest.mark.parametrize("final_already_cites", [False, True]) + async def test_pending_sources_deduplicate_across_final_message_parts_only(self, final_already_cites: bool) -> None: + stream = ResponseEventStream(response_id="resp_multipart_citations") + stream.emit_created() + stream.emit_in_progress() + tracker = _OutputItemTracker(stream) + _ = [event async for event in tracker.handle(Content.from_text("Interim", annotations=[_url_citation()]))] + _ = list(tracker.close()) + _ = [event async for event in tracker.handle(_web_search_result("call_1"))] + update = AgentResponseUpdate( + contents=[ + Content.from_text("First", annotations=[_url_citation()] if final_already_cites else None), + Content.from_text(" final"), + ], + role="assistant", + message_id="final", + ) + events = [event async for event in tracker.handle_update(update, complete_message=True)] + events.extend(tracker.close(final_answer=True)) + + annotations = [ + event["annotation"] for event in events if event["type"] == "response.output_text.annotation.added" + ] + expected = ( + [ + _expected_url_citation(), + _expected_url_citation(url=_DOCS_SOURCE_URL, title="What's New", start_index=6, end_index=6), + ] + if final_already_cites + else _expected_source_citations(" final") + ) + assert annotations == expected + message = next( + event["item"] + for event in events + if event["type"] == "response.output_item.done" and event["item"]["type"] == "message" + ) + parts = [part for part in message["content"] if part["type"] == "output_text"] + assert [part["annotations"] for part in parts] == ( + [[expected[0]], [expected[1]]] if final_already_cites else [[], expected] + ) + + async def test_pending_source_limits_apply_across_results_and_recovery( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ) -> None: + monkeypatch.setattr("agent_framework_foundry_hosting._responses_shared._MAX_PENDING_URL_CITATIONS", 2) + monkeypatch.setattr("agent_framework_foundry_hosting._responses_shared._MAX_PENDING_URL_CITATION_BYTES", 64) + stream = ResponseEventStream(response_id="resp_source_limits") + stream.emit_created() + stream.emit_in_progress() + tracker = _OutputItemTracker(stream) + first = Content.from_function_result( + "call_1", + result=[ + Content.from_text( + "Result", + annotations=[ + Annotation(type="citation", url="https://example.test/1", title="A"), + Annotation(type="citation", url="https://example.test/too-large", title="x" * 64), + ], + ) + ], + ) + _ = [event async for event in tracker.handle(first)] + recovered = _OutputItemTracker(stream) + second = Content.from_function_result( + "call_2", + result=[ + Content.from_text( + "Result", + annotations=[ + Annotation(type="citation", url="https://example.test/2", title="B"), + Annotation(type="citation", url="https://example.test/3", title="C"), + ], + ) + ], + ) + _ = [event async for event in recovered.handle(second)] + events = [event async for event in recovered.handle(Content.from_text("Done."), message_id="final")] + events.extend(recovered.close(final_answer=True)) + + annotations = [ + event["annotation"]["url"] + for event in events + if event["type"] == "response.output_text.annotation.added" + and event["annotation"]["type"] == "url_citation" + ] + assert annotations == [ + "https://example.test/1", + "https://example.test/2", + ] + assert "excess sources were skipped" in caplog.text + + async def test_pending_source_byte_limit_counts_utf8_and_skips_oversized_entries( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr("agent_framework_foundry_hosting._responses_shared._MAX_PENDING_URL_CITATION_BYTES", 33) + stream = ResponseEventStream(response_id="resp_source_bytes") + stream.emit_created() + stream.emit_in_progress() + tracker = _OutputItemTracker(stream) + result = Content.from_function_result( + "call_1", + result=[ + Content.from_text( + "Result", + annotations=[ + Annotation(type="citation", url="https://x", title="\U0001f30e" * 4), + Annotation(type="citation", url="https://y"), + Annotation(type="citation", url="http://z"), + ], + ) + ], + ) + _ = [event async for event in tracker.handle(result)] + recovered = _OutputItemTracker(stream) + events = [event async for event in recovered.handle(Content.from_text("Done."), message_id="final")] + events.extend(recovered.close(final_answer=True)) + + assert [ + event["annotation"]["url"] + for event in events + if event["type"] == "response.output_text.annotation.added" + and event["annotation"]["type"] == "url_citation" + ] == ["https://x", "http://z"] + + async def test_recovery_bounds_previously_persisted_sources(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("agent_framework_foundry_hosting._responses_shared._MAX_PENDING_URL_CITATIONS", 2) + stream = ResponseEventStream(response_id="resp_oversized_source_snapshot") + stream.emit_created() + stream.emit_in_progress() + _hosted_internal_metadata(stream)["_foundry_pending_url_citations"] = [ + {"url": f"https://example.test/{index}", "title": str(index)} for index in range(3) + ] + + recovered = _OutputItemTracker(stream) + events = [event async for event in recovered.handle(Content.from_text("Done."), message_id="final")] + events.extend(recovered.close(final_answer=True)) + + assert [ + event["annotation"]["url"] + for event in events + if event["type"] == "response.output_text.annotation.added" + and event["annotation"]["type"] == "url_citation" + ] == ["https://example.test/0", "https://example.test/1"] + async def test_pending_container_file_citation_survives_tracker_recovery(self) -> None: stream = ResponseEventStream(response_id="resp_container_file") stream.emit_created() diff --git a/python/packages/foundry_hosting/tests/test_toolbox.py b/python/packages/foundry_hosting/tests/test_toolbox.py index 6dc701fc5bd..7ca0db80527 100644 --- a/python/packages/foundry_hosting/tests/test_toolbox.py +++ b/python/packages/foundry_hosting/tests/test_toolbox.py @@ -19,6 +19,7 @@ from agent_framework import ( Agent, AgentSession, + Content, FunctionInvocationContext, MCPStreamableHTTPTool, Message, @@ -27,6 +28,7 @@ SupportsAgentRun, ) from agent_framework.exceptions import ToolException, ToolExecutionException +from agent_framework.openai import OpenAIChatClient from azure.ai.agentserver.core import ( FoundryAgentRequestContext, reset_request_context, @@ -60,6 +62,9 @@ toolbox_connection, toolbox_tool, ) +from .foundry_hosting.web_search_test_helpers import ( # pyrefly: ignore[missing-import] - pytest's importlib namespace + _ToolResultServer, +) class _StubAgent: @@ -396,6 +401,368 @@ def test_init_forwards_additional_tool_arguments_and_parent_kwargs() -> None: assert toolbox._tool_extra_arg_names == {"search": {"thread"}} +@pytest.mark.parametrize( + "metadata_location", ["structured", "block", "resource", "block_and_structured", "resource_and_structured"] +) +async def test_web_search_preserves_prose_and_citation_metadata(metadata_location: str) -> None: + citation = { + "type": "url_citation", + "url": "https://example.test/source", + "title": "Source", + "start_index": 0, + "end_index": 4, + } + resource: dict[str, Any] = {"uri": "about:web-search", "text": "Mars is red."} + block: dict[str, Any] = {"type": "resource", "resource": resource} + if metadata_location.startswith("block"): + block["_meta"] = {"annotations": [citation]} + elif metadata_location.startswith("resource"): + resource["_meta"] = {"annotations": [citation]} + structured: dict[str, Any] = {"query": "Mars"} + if "structured" in metadata_location: + structured["annotations"] = [citation] + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = {"content": [block], "structuredContent": structured} + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert [item.text for item in result] == ["Mars is red.", json.dumps(structured)] + expected: dict[str, Any] = { + "type": "citation", + "url": "https://example.test/source", + "title": "Source", + } + if metadata_location != "structured": + expected["annotated_regions"] = [{"type": "text_span", "start_index": 0, "end_index": 4}] + citation_item = result[1] if metadata_location == "structured" else result[0] + assert citation_item.annotations == [expected] + + +async def test_web_search_keeps_aggregate_citations_unbound_when_content_has_multiple_blocks() -> None: + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [{"type": "text", "text": "First"}, {"type": "text", "text": "Second"}], + "structuredContent": { + "annotations": [ + { + "type": "url_citation", + "url": "https://example.test/source", + "start_index": 0, + "end_index": 5, + } + ] + }, + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert [item.text for item in result] == ["First", "Second", json.dumps(server.call_result["structuredContent"])] + assert result[0].annotations is None + assert result[1].annotations is None + assert result[2].annotations == [{"type": "citation", "url": "https://example.test/source"}] + + +async def test_web_search_keeps_block_citations_with_their_owning_text() -> None: + first = {"type": "url_citation", "url": "https://example.test/first", "start_index": 0, "end_index": 5} + second = {"type": "url_citation", "url": "https://example.test/second", "start_index": 0, "end_index": 6} + unbound = {"type": "url_citation", "url": "https://example.test/other", "start_index": 0, "end_index": 5} + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [ + {"type": "text", "text": "First", "_meta": {"annotations": [first]}}, + {"type": "text", "text": "Second", "_meta": {"annotations": [second]}}, + ], + "structuredContent": {"annotations": [first, second, unbound]}, + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert [item.text for item in result] == ["First", "Second", json.dumps(server.call_result["structuredContent"])] + assert result[0].annotations == [ + { + "type": "citation", + "url": "https://example.test/first", + "annotated_regions": [{"type": "text_span", "start_index": 0, "end_index": 5}], + } + ] + assert result[1].annotations == [ + { + "type": "citation", + "url": "https://example.test/second", + "annotated_regions": [{"type": "text_span", "start_index": 0, "end_index": 6}], + } + ] + assert result[2].annotations == [{"type": "citation", "url": "https://example.test/other"}] + + +async def test_web_search_keeps_repeated_aggregate_citations_not_matched_to_a_block() -> None: + citation = {"type": "url_citation", "url": "https://example.test/source", "start_index": 0, "end_index": 5} + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [ + {"type": "text", "text": "First", "_meta": {"annotations": [citation]}}, + {"type": "text", "text": "Second"}, + ], + "structuredContent": {"annotations": [citation, citation]}, + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert result[0].annotations == [ + { + "type": "citation", + "url": "https://example.test/source", + "annotated_regions": [{"type": "text_span", "start_index": 0, "end_index": 5}], + } + ] + assert result[1].annotations is None + assert result[2].annotations == [{"type": "citation", "url": "https://example.test/source"}] + + +async def test_web_search_keeps_citations_on_non_text_blocks_without_a_span() -> None: + citation = {"type": "url_citation", "url": "https://example.test/image", "start_index": 0, "end_index": 5} + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [ + {"type": "image", "data": "eA==", "mimeType": "image/png", "_meta": {"annotations": [citation]}}, + {"type": "text", "text": "Search prose"}, + ], + "structuredContent": {"annotations": [citation]}, + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert result[0].type == "data" + assert result[0].annotations == [{"type": "citation", "url": "https://example.test/image"}] + assert result[1].text == "Search prose" + assert result[2].text == json.dumps(server.call_result["structuredContent"]) + + +@pytest.mark.parametrize( + ("start", "end", "expected_region"), + [ + (1, 3, {"type": "text_span", "start_index": 1, "end_index": 2}), + (2, 3, None), + (True, 3, None), + (-1, 3, None), + (1, 6, None), + ], +) +async def test_web_search_ignores_invalid_utf16_citation_spans( + start: Any, end: Any, expected_region: dict[str, Any] | None +) -> None: + citation = {"type": "url_citation", "url": "https://example.test/source", "start_index": start, "end_index": end} + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [{"type": "text", "text": "A\U0001f30eBC", "_meta": {"annotations": [citation]}}], + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert result[0].annotations is not None + assert result[0].annotations[0].get("annotated_regions") == ( + [expected_region] if expected_region is not None else None + ) + if expected_region is not None: + assert result[0].text is not None + assert result[0].text[expected_region["start_index"] : expected_region["end_index"]] == "\U0001f30e" + + +async def test_web_search_preserves_unrecognized_structured_entries() -> None: + unrecognized = [ + {"type": "url_citation", "title": "missing URL"}, + {"type": "url_citation", "url": " "}, + {"type": "other", "value": 7}, + "invalid", + ] + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [{"type": "text", "text": "Search prose"}], + "structuredContent": { + "annotations": [{"type": "url_citation", "url": "https://example.test"}, *unrecognized], + "query": "Mars", + }, + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert result[0].text == "Search prose" + assert result[1].text is not None + assert result[1].annotations == [{"type": "citation", "url": "https://example.test"}] + assert json.loads(result[1].text) == server.call_result["structuredContent"] + + +@pytest.mark.parametrize("content", [[], [{"type": "text", "text": ""}]]) +async def test_web_search_without_prose_keeps_structured_result(content: list[dict[str, Any]]) -> None: + structured = {"annotations": [{"type": "url_citation", "url": "https://example.test"}]} + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = {"content": content, "structuredContent": structured} + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert result[0].text is not None + assert json.loads(result[0].text) == structured + assert result[0].annotations == [{"type": "citation", "url": "https://example.test"}] + + +@pytest.mark.parametrize( + ("mode", "expected"), + [ + ("content_first", ["Search prose"]), + ("content_only", ["Search prose"]), + ("structured_only", ['{"annotations": [{"type": "url_citation", "url": "https://example.test"}]}']), + ("both", ["Search prose", '{"annotations": [{"type": "url_citation", "url": "https://example.test"}]}']), + ], +) +async def test_other_mcp_content_modes_replace_citation_handling(mode: str, expected: list[str]) -> None: + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [{"type": "text", "text": "Search prose"}], + "structuredContent": {"annotations": [{"type": "url_citation", "url": "https://example.test"}]}, + } + async with toolbox_connection(server, tool_result_content=mode) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert [item.text for item in result] == expected + assert all(item.annotations is None for item in result) + + +async def test_custom_mcp_result_parser_overrides_toolbox_projection() -> None: + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [{"type": "text", "text": "Search prose"}], + "structuredContent": {"annotations": [{"type": "url_citation", "url": "https://example.test"}]}, + } + async with toolbox_connection(server, parse_tool_results=lambda _: "Custom result") as toolbox: + assert await toolbox.call_tool("learn___microsoft_docs_search") == "Custom result" + + +async def test_non_web_search_result_keeps_default_mcp_selection() -> None: + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [{"type": "text", "text": "Summary"}], + "structuredContent": {"value": 42}, + } + async with toolbox_connection(server) as toolbox: + result = await toolbox.call_tool("learn___microsoft_docs_search") + + assert isinstance(result, list) + assert [item.text for item in result] == ['{"value": 42}'] + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_web_search_provenance_survives_function_invocation(stream: bool) -> None: + citation = { + "type": "url_citation", + "url": "https://example.test/source", + "start_index": 0, + "end_index": 4, + } + server = _ToolResultServer() + server.tools = [toolbox_tool("never")] + server.call_result = { + "content": [ + { + "type": "resource", + "resource": { + "uri": "about:web-search", + "text": "Mars is red.", + "_meta": {"annotations": [citation]}, + }, + } + ], + "structuredContent": {"annotations": [citation]}, + "_meta": {"source": "server"}, + } + client = ToolboxApprovalClient("learn___microsoft_docs_search") + async with toolbox_connection(server) as toolbox: + agent = Agent(client=client, tools=toolbox) + session = agent.create_session() + if stream: + response_stream = agent.run("Search", session=session, stream=True) + updates = [update async for update in response_stream] + response = await response_stream.get_final_response() + assert sum(content.type == "function_result" for update in updates for content in update.contents) == 1 + else: + response = await agent.run("Search", session=session) + + restored_session = AgentSession.from_dict(json.loads(json.dumps(session.to_dict()))) + if stream: + continued_stream = agent.run("Continue", session=restored_session, stream=True) + _ = [update async for update in continued_stream] + await continued_stream.get_final_response() + else: + await agent.run("Continue", session=restored_session) + + results = [ + content for message in response.messages for content in message.contents if content.type == "function_result" + ] + assert len(results) == 1 + assert results[0].call_id == "call-search" + assert results[0].items is not None + assert results[0].items[0].text == "Mars is red." + assert results[0].items[0].annotations == [ + { + "type": "citation", + "url": "https://example.test/source", + "annotated_regions": [{"type": "text_span", "start_index": 0, "end_index": 4}], + } + ] + assert results[0].result == "Mars is red.\n" + json.dumps(server.call_result["structuredContent"]) + model_results = [ + content for message in client.calls[1] for content in message.contents if content.type == "function_result" + ] + assert len(model_results) == 1 + assert model_results[0].call_id == "call-search" + assert model_results[0].items is not None + assert model_results[0].items[0].annotations == results[0].items[0].annotations + replayed = [content for message in client.calls[2] for content in message.contents] + assert sum(content.type == "function_call" and content.call_id == "call-search" for content in replayed) == 1 + replayed_results = [content for content in replayed if content.type == "function_result"] + assert len(replayed_results) == 1 + assert replayed_results[0].call_id == "call-search" + assert replayed_results[0].items is not None + assert replayed_results[0].items[0].annotations == results[0].items[0].annotations + provider = OpenAIChatClient(model="test-model", api_key="test-key") + try: + assert provider._prepare_content_for_openai("tool", model_results[0]) == { + "type": "function_call_output", + "call_id": "call-search", + "output": results[0].result, + } + finally: + await provider.client.close() + assert results[0].items[0].additional_properties["_meta"] == {"source": "server"} + restored = Content.from_dict(results[0].to_dict()) + assert restored.items is not None + assert restored.items[0].annotations == results[0].items[0].annotations + host_payload = restored.additional_properties["_mcp_tool_result_host_payload"] + assert host_payload["content"] == server.call_result["content"] + assert host_payload["structuredContent"] == server.call_result["structuredContent"] + assert host_payload["_meta"] == {"source": "server"} + assert len(server.calls) == 1 + + def test_toolbox_owns_feature_index_53() -> None: assert FeatureIndex.FOUNDRY_TOOLBOX == 53