From 254b89ba33dc914ee333283a5de59b711d5a7ba6 Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Fri, 23 Oct 2026 01:42:56 +0800 Subject: [PATCH 1/9] Python: fix(core): do not replay MCP tools/call without an idempotency hint Fixes #9204. When the MCP connection drops during an ordinary tools/call, the request may already have executed server-side before the response was lost. The reconnect-and-retry added for #2884 re-issues the same call unconditionally on ClosedResourceError / session-terminated McpError, so a non-idempotent operation (a charge, a write, a ticket creation) can run twice. The long-running task path already refuses to re-issue an augmented call before a task_id is known for exactly this reason; the plain call path now follows the same policy. _call_tool_with_retries gates the replay on the tool's advertised ToolAnnotations, captured from tools/list into a _tool_annotations_by_name snapshot alongside the existing meta/task-support maps. Replay requires both a server idempotentHint or readOnlyHint and the client-owned trust_server_idempotency_hints opt-in: annotations are server-supplied and untrusted by default, so a misannotated server must not get a non-idempotent call executed twice on its own say-so. Any other tool raises ToolExecutionException stating the connection was lost and the remote execution outcome is unknown, with the connection error as the inner exception. The gate runs after the reconnect attempt, so a caller-owned connection still fails with its existing 'cannot be recreated' error and never reaches the ambiguous-outcome path. tools/list, get_prompt, and the long-running task lifecycle keep their current retry behavior. Tests: test_mcp.py covers a hinted tool replaying once after reconnect, an unhinted tool raising without a replay, a hinted tool without the client opt-in raising without a replay, a lost-response call executing at most once, and the annotations snapshot rolling back when discovery fails after load_tools. The three no-replay cases go red when the gate is disabled. Full test_mcp.py: 467 passed, 2 skipped. poe syntax -P core and poe pyright -P core are clean. --- python/packages/core/AGENTS.md | 1 + python/packages/core/agent_framework/_mcp.py | 59 +++++++ python/packages/core/tests/core/test_mcp.py | 174 ++++++++++++++++++- 3 files changed, 230 insertions(+), 4 deletions(-) diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index 895764eb93d..271639d32e6 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -242,6 +242,7 @@ The vector store API is experimental under the shared `VECTOR_STORES` feature ID and reconnect or effective header-identity changes replace a framework-owned Client's per-Client cache. Agent Framework does not expose shared-store or cache-mode configuration. - **MCP Tasks extension** - MCP Python SDK 2.2 does not implement the 2026 `io.modelcontextprotocol/tasks` runtime. Tools that advertise `execution.taskSupport == "required"` are skipped with a warning; optional and task-free tools continue through ordinary `tools/call`. Restoration of the experimental Tasks API and removed sample is tracked in microsoft/agent-framework#8245. +- **Plain `tools/call` replay gating** - On `ClosedResourceError` / session-terminated `MCPError` the request may already have executed server-side, so `_call_tool_with_retries` only replays after reconnect when the server advertised the tool with `idempotentHint` or `readOnlyHint` and the client opted in with `trust_server_idempotency_hints` (`_is_call_replayable`, backed by the `_tool_annotations_by_name` snapshot from `tools/list`); any other tool raises `ToolExecutionException` stating the outcome is unknown once the connection is back up, with the connection error as the inner exception, and leaves the retry decision to the caller. Caller-owned connections keep failing with their existing 'cannot be recreated' error before any of this. Read-only discovery (`tools/list`) and `get_prompt` keep their reconnect-and-retry behavior. ### File Access Harness (`_harness/_file_access.py`) diff --git a/python/packages/core/agent_framework/_mcp.py b/python/packages/core/agent_framework/_mcp.py index c4f93237f03..e2276250294 100644 --- a/python/packages/core/agent_framework/_mcp.py +++ b/python/packages/core/agent_framework/_mcp.py @@ -1114,6 +1114,7 @@ def __init__( *, max_host_payload_size_bytes: int | None = _DEFAULT_MCP_HOST_PAYLOAD_SIZE_BYTES, tool_result_content: MCPToolResultContentMode = "structured_first", + trust_server_idempotency_hints: bool = False, ) -> None: """Initialize the MCP Tool base. @@ -1161,6 +1162,7 @@ def __init__( self.load_prompts_flag = load_prompts self.parse_prompt_results = parse_prompt_results self.max_host_payload_size_bytes = max_host_payload_size_bytes + self.trust_server_idempotency_hints = trust_server_idempotency_hints self._exit_stack = AsyncExitStack() self._lifecycle_lock = asyncio.Lock() self._lifecycle_request_lock = asyncio.Lock() @@ -1207,6 +1209,7 @@ def __init__( self._progressive_loader_functions: list[FunctionTool] | None = None self._progressive_loaded_tool_names: set[str] = set() self._tool_call_meta_by_name: dict[str, dict[str, Any]] = {} + self._tool_annotations_by_name: dict[str, types.ToolAnnotations | None] = {} self._tool_param_names_by_name: dict[str, set[str]] = {} self._global_extra_arg_names, self._tool_extra_arg_names = _normalize_additional_tool_argument_names( additional_tool_argument_names @@ -2254,6 +2257,7 @@ def _reset_session_discovery_state(self) -> None: if not isinstance((function.additional_properties or {}).get(_MCP_REMOTE_NAME_KEY), str) ] self._tool_call_meta_by_name.clear() + self._tool_annotations_by_name.clear() self._tool_param_names_by_name.clear() self._progressive_loaded_tool_names.clear() @@ -2485,6 +2489,7 @@ async def _connect_on_owner( functions_before_discovery = self._functions.copy() call_meta_before_discovery = self._tool_call_meta_by_name param_names_before_discovery = self._tool_param_names_by_name + annotations_before_discovery = self._tool_annotations_by_name try: connection = self._require_connection() logger.debug("Connected to MCP server: %s", connection.client or connection.session) @@ -2520,6 +2525,7 @@ async def _connect_on_owner( self._functions[:] = functions_before_discovery self._tool_call_meta_by_name = call_meta_before_discovery self._tool_param_names_by_name = param_names_before_discovery + self._tool_annotations_by_name = annotations_before_discovery raise async def _prepare_for_run(self, kwargs: Mapping[str, Any]) -> None: @@ -3207,6 +3213,7 @@ async def _load_tools_locked(self) -> None: self._function_load_callback(function, tool_annotations_by_name[remote_name]) self._functions[:] = current_functions self._tool_call_meta_by_name = tool_call_meta_by_name + self._tool_annotations_by_name = tool_annotations_by_name self._tool_param_names_by_name = tool_param_names_by_name self._progressive_loaded_tool_names.difference_update(existing_tools.keys() - reused_tool_names) @@ -3428,6 +3435,21 @@ async def _call_tool_with_retries( if span.is_recording(): set_mcp_span_error(span, type(recovery_error).__name__, str(recovery_error)) raise + if not self._is_call_replayable(tool_name): + # The request may have reached the server before the connection + # dropped, so replaying it could execute a non-idempotent operation + # twice. Surface the unknown outcome instead of a replay; the + # connection is already back up for whoever decides to retry. + if span.is_recording(): + set_mcp_span_error(span, type(call_ex).__name__, str(call_ex)) + raise _mcp_tool_execution_exception( + f"Failed to call tool '{tool_name}' - connection lost and the remote " + "execution outcome is unknown. The connection was re-established but " + "the call was not replayed, because replay requires both a server " + "idempotent/read-only hint and the client-owned " + "trust_server_idempotency_hints opt-in; retry explicitly if safe.", + inner_exception=call_ex, + ) from call_ex continue # Second attempt also failed, give up. @@ -3453,6 +3475,22 @@ async def _call_tool_with_retries( set_mcp_span_error(span, type(retries_exhausted).__name__, str(retries_exhausted)) raise retries_exhausted + def _is_call_replayable(self, tool_name: str) -> bool: + """Return True if *tool_name* may be replayed after connection loss. + + Replay requires both a client-owned trust decision + (``trust_server_idempotency_hints=True``) and a server annotation carrying + ``idempotentHint`` or ``readOnlyHint``. Annotations are supplied by the + server and are untrusted by default, consistent with how the rest of the + package treats ``ToolAnnotations`` as restriction-only hints: without the + client's explicit opt-in, a misannotated or untrusted server could get a + non-idempotent ``tools/call`` executed twice on a lost response. + """ + if not self.trust_server_idempotency_hints: + return False + annotations = self._tool_annotations_by_name.get(tool_name) + return bool(annotations and (annotations.idempotent_hint or annotations.read_only_hint)) + def _resolved_extra_args(self, tool_name: str) -> set[str]: """Return the user-configured extra argument names allowed for a tool.""" return self._global_extra_arg_names | self._tool_extra_arg_names.get(tool_name, set()) @@ -3682,6 +3720,7 @@ def __init__( additional_tool_argument_names: Sequence[str] | Mapping[str, Sequence[str]] | None = None, max_host_payload_size_bytes: int | None = _DEFAULT_MCP_HOST_PAYLOAD_SIZE_BYTES, tool_result_content: MCPToolResultContentMode = "structured_first", + trust_server_idempotency_hints: bool = False, **kwargs: Any, ) -> None: """Initialize the MCP stdio tool. @@ -3789,6 +3828,11 @@ def __init__( transports. ``None`` disables the limit. tool_result_content: How to choose model-visible text when both ``content`` and ``structuredContent`` are present. See :data:`MCPToolResultContentMode`. + trust_server_idempotency_hints: When ``True``, treat server-declared + ``idempotentHint``/``readOnlyHint`` tool annotations as trustworthy, allowing + a ``tools/call`` whose response was lost to a dropped connection to be retried + once after reconnect. Annotations are server-supplied and untrusted by default, + so the default ``False`` never replays an ambiguous call. kwargs: Additional arguments for framework-owned ``StdioServerParameters`` construction. """ if mcp_client is not None and session is not None: @@ -3842,6 +3886,7 @@ def __init__( task_options=task_options, max_host_payload_size_bytes=max_host_payload_size_bytes, tool_result_content=tool_result_content, + trust_server_idempotency_hints=trust_server_idempotency_hints, ) self.command = command self.args = args or [] @@ -3937,6 +3982,7 @@ def __init__( task_options: Any = None, max_host_payload_size_bytes: int | None = _DEFAULT_MCP_HOST_PAYLOAD_SIZE_BYTES, tool_result_content: MCPToolResultContentMode = "structured_first", + trust_server_idempotency_hints: bool = False, **kwargs: Any, ) -> None: """Initialize the MCP streamable HTTP tool. @@ -4115,6 +4161,11 @@ def __init__( transports. ``None`` disables the limit. tool_result_content: How to choose model-visible text when both ``content`` and ``structuredContent`` are present. See :data:`MCPToolResultContentMode`. + trust_server_idempotency_hints: When ``True``, treat server-declared + ``idempotentHint``/``readOnlyHint`` tool annotations as trustworthy, allowing + a ``tools/call`` whose response was lost to a dropped connection to be retried + once after reconnect. Annotations are server-supplied and untrusted by default, + so the default ``False`` never replays an ambiguous call. kwargs: Additional keyword arguments (accepted for backward compatibility but not used). """ if mcp_client is not None and session is not None: @@ -4168,6 +4219,7 @@ def __init__( task_options=task_options, max_host_payload_size_bytes=max_host_payload_size_bytes, tool_result_content=tool_result_content, + trust_server_idempotency_hints=trust_server_idempotency_hints, ) self.url = url self.terminate_on_close = terminate_on_close @@ -4599,6 +4651,7 @@ def __init__( task_options: Any = None, max_host_payload_size_bytes: int | None = _DEFAULT_MCP_HOST_PAYLOAD_SIZE_BYTES, tool_result_content: MCPToolResultContentMode = "structured_first", + trust_server_idempotency_hints: bool = False, **kwargs: Any, ) -> None: """Initialize the deprecated MCP WebSocket compatibility wrapper. @@ -4691,6 +4744,11 @@ def __init__( transports. ``None`` disables the limit. tool_result_content: How to choose model-visible text when both ``content`` and ``structuredContent`` are present. See :data:`MCPToolResultContentMode`. + trust_server_idempotency_hints: When ``True``, treat server-declared + ``idempotentHint``/``readOnlyHint`` tool annotations as trustworthy, allowing + a ``tools/call`` whose response was lost to a dropped connection to be retried + once after reconnect. Annotations are server-supplied and untrusted by default, + so the default ``False`` never replays an ambiguous call. kwargs: Deprecated compatibility arguments. They are not used. """ super().__init__( @@ -4716,6 +4774,7 @@ def __init__( task_options=task_options, max_host_payload_size_bytes=max_host_payload_size_bytes, tool_result_content=tool_result_content, + trust_server_idempotency_hints=trust_server_idempotency_hints, ) self.url = url self._client_kwargs = kwargs diff --git a/python/packages/core/tests/core/test_mcp.py b/python/packages/core/tests/core/test_mcp.py index 757e2a15f88..6d8f41a13ec 100644 --- a/python/packages/core/tests/core/test_mcp.py +++ b/python/packages/core/tests/core/test_mcp.py @@ -14,7 +14,7 @@ from textwrap import dedent from types import TracebackType from typing import Any, cast -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest from httpx2 import AsyncClient, MockTransport, Request, Response @@ -2880,7 +2880,7 @@ def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: async def test_mcp_tool_reconnects_after_session_terminated_error(): - """Session termination errors should reconnect once and retry the tool call.""" + """Trusted idempotent tools reconnect once and retry after session termination.""" class TestServer(MCPTool): def __init__(self, **kwargs: Any) -> None: @@ -2904,8 +2904,9 @@ async def connect(self, *, reset: bool = False) -> None: def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: return None # type: ignore[return-value] # pyrefly: ignore[bad-return] # ty: ignore[invalid-return-type] - server = TestServer(name="test_server") + server = TestServer(name="test_server", trust_server_idempotency_hints=True) await server.connect() + server._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotentHint=True) result = await server.call_tool("test_tool", param="test_value") @@ -2915,6 +2916,159 @@ def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: assert server.sessions[1].call_tool.await_count == 1 +async def test_mcp_tool_call_not_replayed_without_idempotency_hint(): + """Tools without an idempotency hint raise an unknown-outcome error instead of replaying. + + The connection can drop after the server executed the request, so re-issuing the same + tools/call could duplicate a non-idempotent side effect. The default path surfaces the + failure without replaying: the connection is re-established, and the caller decides + whether to retry explicitly. + """ + + class TestServer(MCPTool): + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.connect_count = 0 + self.call_tool_mock = AsyncMock(side_effect=MCPError(-32000, "Session terminated")) + + async def connect(self, *, reset: bool = False) -> None: + self.connect_count += 1 + session = Mock(spec=ClientSession) + session.call_tool = self.call_tool_mock + _set_framework_owned_session(self, session) + self.is_connected = True + + def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: + return None # type: ignore[return-value] # pyrefly: ignore[bad-return] # ty: ignore[invalid-return-type] + + server = TestServer(name="test_server") + await server.connect() + + with pytest.raises(ToolExecutionException, match="outcome is unknown") as exc_info: + await server.call_tool("test_tool", param="test_value") + + assert isinstance(exc_info.value.__cause__, MCPError) + assert server.connect_count == 2 + assert server.call_tool_mock.await_count == 1 + + +async def test_mcp_tool_call_not_replayed_on_hint_alone_without_client_trust(): + """A server idempotency hint by itself must not authorize replay. + + ToolAnnotations are server-supplied and untrusted unless the client separately + trusts the server, so with the default trust posture a hinted tool still gets + the unknown-outcome error instead of a replay. + """ + + class TestServer(MCPTool): + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.connect_count = 0 + self.call_tool_mock = AsyncMock(side_effect=MCPError(-32000, "Session terminated")) + + async def connect(self, *, reset: bool = False) -> None: + self.connect_count += 1 + session = Mock(spec=ClientSession) + session.call_tool = self.call_tool_mock + _set_framework_owned_session(self, session) + self.is_connected = True + + def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: + return None # type: ignore[return-value] # pyrefly: ignore[bad-return] # ty: ignore[invalid-return-type] + + server = TestServer(name="test_server") + await server.connect() + server._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotentHint=True) + + with pytest.raises(ToolExecutionException, match="outcome is unknown"): + await server.call_tool("test_tool", param="test_value") + + assert server.connect_count == 2 + assert server.call_tool_mock.await_count == 1 + + +async def test_mcp_regular_tool_call_does_not_duplicate_side_effect_after_disconnect(): + """A non-idempotent tool whose response is lost mid-call executes at most once.""" + from anyio.streams.memory import ClosedResourceError + + class TestServer(MCPTool): + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.state = 0 + + async def connect(self, *, reset: bool = False) -> None: + session = Mock(spec=ClientSession) + + async def call_tool(*args: Any, **kwargs: Any) -> types.CallToolResult: + self.state += 1 + if self.state == 1: + raise ClosedResourceError + return types.CallToolResult(content=[types.TextContent(type="text", text="ok")]) + + session.call_tool = call_tool + _set_framework_owned_session(self, session) + self.is_connected = True + + def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: + return None # type: ignore[return-value] # pyrefly: ignore[bad-return] # ty: ignore[invalid-return-type] + + server = TestServer(name="test_server") + await server.connect() + + with pytest.raises(ToolExecutionException, match="outcome is unknown"): + await server.call_tool("test_tool", param="test_value") + + assert server.state == 1 + + +async def test_connect_discovery_failure_rolls_back_annotations_snapshot(): + """When load_tools succeeds but a later discovery step fails, the rollback in + _connect_on_owner must restore the pre-discovery annotations map along with the + other metadata, or a later replay decision reads hints from the failed session.""" + tool = MCPStdioTool(name="test", command="test-command") + sentinel_annotations = {"prev_tool": types.ToolAnnotations(readOnlyHint=True)} + sentinel_meta = {"prev_tool": {"version": 0}} + sentinel_params = {"prev_tool": {"q"}} + tool._tool_annotations_by_name = dict(sentinel_annotations) + tool._tool_call_meta_by_name = dict(sentinel_meta) + tool._tool_param_names_by_name = dict(sentinel_params) + retained = FunctionTool(name="retained", func=lambda: "ok") + tool._functions.append(retained) + + mock_transport = (MagicMock(), MagicMock()) + mock_context_manager = Mock() + mock_context_manager.__aenter__ = AsyncMock(return_value=mock_transport) + mock_context_manager.__aexit__ = AsyncMock(return_value=None) + tool.get_mcp_client = Mock(return_value=mock_context_manager) # type: ignore[method-assign] + + sdk_client = _mock_sdk_client( + capabilities=types.ServerCapabilities( + tools=types.ToolsCapability(list_changed=False), + prompts=types.PromptsCapability(list_changed=False), + ) + ) + sdk_client.list_tools = AsyncMock( + return_value=types.ListToolsResult( + tools=[ + types.Tool( + name="fresh_tool", + inputSchema={"type": "object", "properties": {}}, + annotations=types.ToolAnnotations(idempotentHint=True), + ) + ] + ) + ) + sdk_client.list_prompts = AsyncMock(side_effect=RuntimeError("prompt boom")) + + with patch("mcp.Client", return_value=sdk_client), pytest.raises(RuntimeError, match="prompt boom"): + await tool.connect() + + assert tool._tool_annotations_by_name == sentinel_annotations + assert tool._tool_call_meta_by_name == sentinel_meta + assert tool._tool_param_names_by_name == sentinel_params + assert tool._functions == [retained] + + async def test_mcp_tool_call_tool_raises_on_is_error(): """Test that call_tool raises ToolExecutionException when MCP returns is_error=True.""" @@ -3987,6 +4141,10 @@ async def test_mcp_connection_reset_integration(): original_exit_stack = tool._exit_stack original_call_tool = tool.session.call_tool + # The query tool is read-only, so a lost response is safe to replay: mark it as such. + remote_name = (func.additional_properties or {}).get("_mcp_remote_name") or func.name + tool._tool_annotations_by_name[remote_name] = types.ToolAnnotations(readOnlyHint=True) + # Simulate connection failure by making call_tool raise ClosedResourceError once call_count = 0 @@ -7015,6 +7173,9 @@ async def test_mcp_tool_connection_properly_invalidated_after_closed_resource_er This test verifies the fix for issue #2884: the tool tries operations optimistically and only reconnects when ClosedResourceError is encountered, avoiding extra latency. + The replay only happens because the client opted into trusting the tool's idempotency + hint here; tools without that hint, or without the client opt-in, raise instead of + retrying (issue #9204). """ from unittest.mock import AsyncMock, MagicMock, patch @@ -7029,7 +7190,9 @@ async def test_mcp_tool_connection_properly_invalidated_after_closed_resource_er command="test_command", args=["arg1"], load_tools=True, + trust_server_idempotency_hints=True, ) + tool._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotentHint=True) # Mock the session mock_session = MagicMock() @@ -7270,11 +7433,14 @@ async def test_mcp_tool_get_prompt_requires_loaded_prompts() -> None: async def test_mcp_tool_call_tool_raises_after_reconnection_still_fails() -> None: from anyio.streams.memory import ClosedResourceError - tool = MCPTool(name="test_tool", load_tools=True) # type: ignore[abstract] # ty: ignore[call-non-callable] + tool = MCPTool( # type: ignore[abstract] # ty: ignore[call-non-callable] + name="test_tool", load_tools=True, trust_server_idempotency_hints=True + ) _set_framework_owned_session( tool, Mock(call_tool=AsyncMock(side_effect=[ClosedResourceError(), ClosedResourceError()])), ) + tool._tool_annotations_by_name["remote_tool"] = types.ToolAnnotations(idempotentHint=True) with ( patch.object(tool, "connect", AsyncMock()) as mock_connect, From 56d0dd857a17c4f43a5e10710fa15e678b6f4ff9 Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:02:55 +0800 Subject: [PATCH 2/9] Python: test(core): opt the reset integration into the trust gate The connection-reset integration test marks the query tool read-only before the simulated drop so the lost response is safe to replay, but the replay also needs the client-owned trust opt-in, which the default posture withholds. Without it the second invocation takes the unknown-outcome path and never reaches the reconnection the test is about. Opt the tool into trust_server_idempotency_hints for that test only, keeping the read-only hint it already sets. --- python/packages/core/tests/core/test_mcp.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/packages/core/tests/core/test_mcp.py b/python/packages/core/tests/core/test_mcp.py index 6d8f41a13ec..3876f0e46af 100644 --- a/python/packages/core/tests/core/test_mcp.py +++ b/python/packages/core/tests/core/test_mcp.py @@ -4122,7 +4122,15 @@ async def test_mcp_connection_reset_integration(): """ url = os.environ.get("LOCAL_MCP_URL") - tool = MCPStreamableHTTPTool(name="integration_test", url=url, approval_mode="never_require") # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + # The query tool below is annotated read-only before the simulated drop; + # replay also needs the client-owned trust opt-in, which the default + # posture deliberately withholds. + tool = MCPStreamableHTTPTool( + name="integration_test", + url=url, + approval_mode="never_require", # type: ignore[arg-type] + trust_server_idempotency_hints=True, + ) # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] async with tool: # Verify initial connection From b6246e38c1402bff447963ec6afb7a21d3db7a34 Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:21:40 +0800 Subject: [PATCH 3/9] Python: test(core): align MCP replay fixtures with SDK and replay gate --- python/packages/core/tests/core/test_mcp.py | 16 ++++++++-------- .../core/tests/core/test_mcp_observability.py | 3 +++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/python/packages/core/tests/core/test_mcp.py b/python/packages/core/tests/core/test_mcp.py index 3876f0e46af..1b9371441e1 100644 --- a/python/packages/core/tests/core/test_mcp.py +++ b/python/packages/core/tests/core/test_mcp.py @@ -2906,7 +2906,7 @@ def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: server = TestServer(name="test_server", trust_server_idempotency_hints=True) await server.connect() - server._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotentHint=True) + server._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotent_hint=True) result = await server.call_tool("test_tool", param="test_value") @@ -2978,7 +2978,7 @@ def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: server = TestServer(name="test_server") await server.connect() - server._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotentHint=True) + server._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotent_hint=True) with pytest.raises(ToolExecutionException, match="outcome is unknown"): await server.call_tool("test_tool", param="test_value") @@ -3026,7 +3026,7 @@ async def test_connect_discovery_failure_rolls_back_annotations_snapshot(): _connect_on_owner must restore the pre-discovery annotations map along with the other metadata, or a later replay decision reads hints from the failed session.""" tool = MCPStdioTool(name="test", command="test-command") - sentinel_annotations = {"prev_tool": types.ToolAnnotations(readOnlyHint=True)} + sentinel_annotations = {"prev_tool": types.ToolAnnotations(read_only_hint=True)} sentinel_meta = {"prev_tool": {"version": 0}} sentinel_params = {"prev_tool": {"q"}} tool._tool_annotations_by_name = dict(sentinel_annotations) @@ -3052,8 +3052,8 @@ async def test_connect_discovery_failure_rolls_back_annotations_snapshot(): tools=[ types.Tool( name="fresh_tool", - inputSchema={"type": "object", "properties": {}}, - annotations=types.ToolAnnotations(idempotentHint=True), + input_schema={"type": "object", "properties": {}}, + annotations=types.ToolAnnotations(idempotent_hint=True), ) ] ) @@ -4151,7 +4151,7 @@ async def test_mcp_connection_reset_integration(): # The query tool is read-only, so a lost response is safe to replay: mark it as such. remote_name = (func.additional_properties or {}).get("_mcp_remote_name") or func.name - tool._tool_annotations_by_name[remote_name] = types.ToolAnnotations(readOnlyHint=True) + tool._tool_annotations_by_name[remote_name] = types.ToolAnnotations(read_only_hint=True) # Simulate connection failure by making call_tool raise ClosedResourceError once call_count = 0 @@ -7200,7 +7200,7 @@ async def test_mcp_tool_connection_properly_invalidated_after_closed_resource_er load_tools=True, trust_server_idempotency_hints=True, ) - tool._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotentHint=True) + tool._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotent_hint=True) # Mock the session mock_session = MagicMock() @@ -7448,7 +7448,7 @@ async def test_mcp_tool_call_tool_raises_after_reconnection_still_fails() -> Non tool, Mock(call_tool=AsyncMock(side_effect=[ClosedResourceError(), ClosedResourceError()])), ) - tool._tool_annotations_by_name["remote_tool"] = types.ToolAnnotations(idempotentHint=True) + tool._tool_annotations_by_name["remote_tool"] = types.ToolAnnotations(idempotent_hint=True) with ( patch.object(tool, "connect", AsyncMock()) as mock_connect, diff --git a/python/packages/core/tests/core/test_mcp_observability.py b/python/packages/core/tests/core/test_mcp_observability.py index 9858823119e..049f6dc30d1 100644 --- a/python/packages/core/tests/core/test_mcp_observability.py +++ b/python/packages/core/tests/core/test_mcp_observability.py @@ -327,6 +327,8 @@ async def test_mcp_tools_call_second_connection_loss_log_respects_sensitive_data """Second connection loss should retain an error log without exporting exception details by default.""" private_error_detail = "founder-secret-token" tool = _make_connected_mcp_tool() + tool.trust_server_idempotency_hints = True + tool._tool_annotations_by_name["get-weather"] = types.ToolAnnotations(idempotent_hint=True) tool.session.call_tool = AsyncMock( # type: ignore[method-assign, union-attr] # ty: ignore[invalid-assignment] side_effect=[ MCPError(-32000, f"session terminated: {private_error_detail}"), @@ -342,6 +344,7 @@ async def test_mcp_tools_call_second_connection_loss_log_respects_sensitive_data ): await tool.call_tool("get-weather") + assert tool.session.call_tool.await_count == 2 assert (private_error_detail in caplog.text) is enable_sensitive_data if not enable_sensitive_data: assert "MCP tool call connection closed after reconnection (MCPError)." in caplog.text From 31d2547929f88ba51ea63a9b0eee4811146ad2a3 Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:47:38 +0800 Subject: [PATCH 4/9] Python: test(core): narrow MCP session before await assertion --- python/packages/core/tests/core/test_mcp_observability.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/packages/core/tests/core/test_mcp_observability.py b/python/packages/core/tests/core/test_mcp_observability.py index 049f6dc30d1..68b0435c80c 100644 --- a/python/packages/core/tests/core/test_mcp_observability.py +++ b/python/packages/core/tests/core/test_mcp_observability.py @@ -344,6 +344,7 @@ async def test_mcp_tools_call_second_connection_loss_log_respects_sensitive_data ): await tool.call_tool("get-weather") + assert tool.session is not None assert tool.session.call_tool.await_count == 2 assert (private_error_detail in caplog.text) is enable_sensitive_data if not enable_sensitive_data: From 53f1de1b8417231426b7300bbe95f52b9db8478f Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:58:32 +0800 Subject: [PATCH 5/9] Python: fix(core): pin MCP replay eligibility to initial catalog --- python/packages/core/AGENTS.md | 2 +- python/packages/core/agent_framework/_mcp.py | 4 +- python/packages/core/tests/core/test_mcp.py | 37 +++++++++++++++++++ .../core/tests/core/test_mcp_observability.py | 2 +- 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index be02f112c46..97323a06883 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -248,7 +248,7 @@ The vector store API is experimental under the shared `VECTOR_STORES` feature ID and reconnect or effective header-identity changes replace a framework-owned Client's per-Client cache. Agent Framework does not expose shared-store or cache-mode configuration. - **MCP Tasks extension** - The current Agent Framework MCP client does not implement the `io.modelcontextprotocol/tasks` runtime. Tools that advertise `execution.taskSupport == "required"` are skipped with a warning; optional and task-free tools continue through ordinary `tools/call`. Tasks client support is tracked in microsoft/agent-framework#9215. -- **Plain `tools/call` replay gating** - On `ClosedResourceError` / session-terminated `MCPError` the request may already have executed server-side, so `_call_tool_with_retries` only replays after reconnect when the server advertised the tool with `idempotentHint` or `readOnlyHint` and the client opted in with `trust_server_idempotency_hints` (`_is_call_replayable`, backed by the `_tool_annotations_by_name` snapshot from `tools/list`); any other tool raises `ToolExecutionException` stating the outcome is unknown once the connection is back up, with the connection error as the inner exception, and leaves the retry decision to the caller. Caller-owned connections keep failing with their existing 'cannot be recreated' error before any of this. Read-only discovery (`tools/list`) and `get_prompt` keep their reconnect-and-retry behavior. +- **Plain `tools/call` replay gating** - On `ClosedResourceError` / session-terminated `MCPError` the request may already have executed server-side, so `_call_tool_with_retries` only replays after reconnect when the server advertised the tool with `idempotentHint` or `readOnlyHint` and the client opted in with `trust_server_idempotency_hints` (`_is_call_replayable`, backed by the `_tool_annotations_by_name` snapshot from `tools/list`); any other tool raises `ToolExecutionException` stating the outcome is unknown once the connection is back up, with the connection error as the inner exception, and leaves the retry decision to the caller. Eligibility is captured before the first dispatch and the refreshed catalog must preserve it: a reconnect cannot upgrade an originally unhinted operation into a replay, and a replacement that removes the hint also blocks replay. Caller-owned connections keep failing with their existing 'cannot be recreated' error before any of this. Read-only discovery (`tools/list`) and `get_prompt` keep their reconnect-and-retry behavior. ### File Access Harness (`_harness/_file_access.py`) diff --git a/python/packages/core/agent_framework/_mcp.py b/python/packages/core/agent_framework/_mcp.py index 3cfbd5ba147..50f78e6260e 100644 --- a/python/packages/core/agent_framework/_mcp.py +++ b/python/packages/core/agent_framework/_mcp.py @@ -3396,6 +3396,8 @@ async def _call_tool_with_retries( from anyio import ClosedResourceError from mcp import MCPError + # A catalog refresh cannot retroactively make the first dispatch safe to repeat. + replay_eligible = self._is_call_replayable(tool_name) for attempt in range(2): span_error_recorded = False try: @@ -3440,7 +3442,7 @@ async def _call_tool_with_retries( if span.is_recording(): set_mcp_span_error(span, type(recovery_error).__name__, str(recovery_error)) raise - if not self._is_call_replayable(tool_name): + if not replay_eligible or not self._is_call_replayable(tool_name): # The request may have reached the server before the connection # dropped, so replaying it could execute a non-idempotent operation # twice. Surface the unknown outcome instead of a replay; the diff --git a/python/packages/core/tests/core/test_mcp.py b/python/packages/core/tests/core/test_mcp.py index f4bf13a0b46..ba00f5b3dfd 100644 --- a/python/packages/core/tests/core/test_mcp.py +++ b/python/packages/core/tests/core/test_mcp.py @@ -2962,6 +2962,43 @@ def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: assert server.call_tool_mock.await_count == 1 +async def test_mcp_tool_call_not_replayed_when_reconnect_adds_hint(): + """A replacement catalog cannot make an already dispatched operation replayable.""" + + class TestServer(MCPTool): + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.connect_count = 0 + self.sessions: list[Any] = [] + + async def connect(self, *, reset: bool = False) -> None: + self.connect_count += 1 + mock_session = Mock(spec=ClientSession) + _set_framework_owned_session(self, mock_session) + self.sessions.append(self.session) + if self.connect_count == 1: + mock_session.call_tool = AsyncMock(side_effect=MCPError(-32000, "Session terminated")) + else: + self._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotent_hint=True) + mock_session.call_tool = AsyncMock( + return_value=types.CallToolResult(content=[types.TextContent(type="text", text="recovered")]) + ) + self.is_connected = True + + def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: + return None # type: ignore[return-value] # pyrefly: ignore[bad-return] # ty: ignore[invalid-return-type] + + server = TestServer(name="test_server", trust_server_idempotency_hints=True) + await server.connect() + + with pytest.raises(ToolExecutionException, match="outcome is unknown"): + await server.call_tool("test_tool", param="test_value") + + assert server.connect_count == 2 + assert server.sessions[0].call_tool.await_count == 1 + assert server.sessions[1].call_tool.await_count == 0 + + async def test_mcp_tool_call_not_replayed_on_hint_alone_without_client_trust(): """A server idempotency hint by itself must not authorize replay. diff --git a/python/packages/core/tests/core/test_mcp_observability.py b/python/packages/core/tests/core/test_mcp_observability.py index 68b0435c80c..4115c5abd44 100644 --- a/python/packages/core/tests/core/test_mcp_observability.py +++ b/python/packages/core/tests/core/test_mcp_observability.py @@ -345,7 +345,7 @@ async def test_mcp_tools_call_second_connection_loss_log_respects_sensitive_data await tool.call_tool("get-weather") assert tool.session is not None - assert tool.session.call_tool.await_count == 2 + assert getattr(tool.session.call_tool, "await_count", 0) == 2 assert (private_error_detail in caplog.text) is enable_sensitive_data if not enable_sensitive_data: assert "MCP tool call connection closed after reconnection (MCPError)." in caplog.text From bc22c21dab17fb75150c0981cfe554ec9d847a57 Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Thu, 5 Nov 2026 01:04:19 +0800 Subject: [PATCH 6/9] test(core): exercise MCP reconnect through active client --- python/packages/core/tests/core/test_mcp.py | 47 ++++++++++++------- .../local_mcp_streamable_http_server.py | 31 ++++++------ 2 files changed, 45 insertions(+), 33 deletions(-) diff --git a/python/packages/core/tests/core/test_mcp.py b/python/packages/core/tests/core/test_mcp.py index ba00f5b3dfd..a6fa29bc8aa 100644 --- a/python/packages/core/tests/core/test_mcp.py +++ b/python/packages/core/tests/core/test_mcp.py @@ -4139,8 +4139,12 @@ async def test_streamable_http_integration(): tool = MCPStreamableHTTPTool(name="integration_test", url=url, approval_mode="never_require") async with tool: - # Test that we can connect and load tools - assert tool.session is not None + # URL construction selects a framework-owned high-level Client. The + # deprecated low-level session property is deliberately empty. + assert tool._connection is not None + assert tool._connection.is_framework_owned + assert tool._connection.client is not None + assert tool._connection.session is None assert isinstance(tool.functions, list) # If there are functions available, try to get information about one @@ -4180,8 +4184,14 @@ async def test_mcp_connection_reset_integration(): ) # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] async with tool: - # Verify initial connection - assert tool.session is not None + # Verify the active framework-owned high-level Client. URL-backed tools + # deliberately do not expose a deprecated low-level session here. + original_connection = tool._connection + assert original_connection is not None + assert original_connection.is_framework_owned + original_client = original_connection.client + assert original_client is not None + assert original_connection.session is None assert tool.is_connected is True assert len(tool.functions) > 0, "The MCP server should have at least one function." @@ -4191,14 +4201,12 @@ async def test_mcp_connection_reset_integration(): assert first_result is not None assert len(first_result) > 0 - # Store the original session and exit stack for comparison - original_session = tool.session + # Capture the active Client and exit stack for comparison after reconnect. original_exit_stack = tool._exit_stack - original_call_tool = tool.session.call_tool + original_call_tool = original_client.call_tool - # The query tool is read-only, so a lost response is safe to replay: mark it as such. - remote_name = (func.additional_properties or {}).get("_mcp_remote_name") or func.name - tool._tool_annotations_by_name[remote_name] = types.ToolAnnotations(read_only_hint=True) + # The local server advertises this query tool as read-only in tools/list. + # Reconnection must discover the same hint before the trusted replay. # Simulate connection failure by making call_tool raise ClosedResourceError once call_count = 0 @@ -4214,16 +4222,21 @@ async def call_tool_with_error(*args, **kwargs): # After reconnection, delegate to the original method return await original_call_tool(*args, **kwargs) - tool.session.call_tool = call_tool_with_error # type: ignore[method-assign] # ty: ignore[invalid-assignment] - - # Invoke the function again - this should trigger automatic reconnection on ClosedResourceError - second_result = _mcp_result_to_text(await func.invoke(query="What is Agent Framework?")) + # Patch the active high-level Client, not the deliberately empty low-level + # session. Reconnection replaces this Client before the trusted replay. + with patch.object(original_client, "call_tool", new=call_tool_with_error): + second_result = _mcp_result_to_text(await func.invoke(query="What is Agent Framework?")) + assert call_count == 1 assert second_result is not None assert len(second_result) > 0 - # Verify we have a new session and exit stack after reconnection - assert tool.session is not None - assert tool.session is not original_session, "Session should be replaced after reconnection" + # Verify reconnection replaced both the framework-owned Client and its exit stack. + new_connection = tool._connection + assert new_connection is not None + assert new_connection.is_framework_owned + assert new_connection.client is not None + assert new_connection.client is not original_client, "Client should be replaced after reconnection" + assert new_connection.session is None assert tool._exit_stack is not original_exit_stack, "Exit stack should be replaced after reconnection" assert tool.is_connected is True diff --git a/python/scripts/local_mcp_streamable_http_server.py b/python/scripts/local_mcp_streamable_http_server.py index 772b7e4161e..ecda5e9ec16 100644 --- a/python/scripts/local_mcp_streamable_http_server.py +++ b/python/scripts/local_mcp_streamable_http_server.py @@ -8,7 +8,8 @@ import asyncio from collections.abc import Sequence -from mcp.server.fastmcp import FastMCP +from mcp import types +from mcp.server.mcpserver import MCPServer from starlette.requests import Request from starlette.responses import JSONResponse, Response @@ -29,14 +30,11 @@ def _normalize_mount_path(path: str) -> str: return normalized.rstrip("/") or "/" -def create_server(*, host: str, port: int, mount_path: str) -> FastMCP: +def create_server(*, mount_path: str) -> MCPServer: """Create the local MCP integration test server.""" - server = FastMCP( + server = MCPServer( name=SERVER_NAME, instructions="Deterministic local MCP server used by Agent Framework integration tests.", - host=host, - port=port, - streamable_http_path=mount_path, log_level="INFO", ) @@ -44,17 +42,16 @@ def create_server(*, host: str, port: int, mount_path: str) -> FastMCP: async def healthz(_request: Request) -> Response: """Return a simple readiness response for CI health checks.""" await asyncio.sleep(0) - return JSONResponse( - { - "status": "ok", - "name": SERVER_NAME, - "mcp_path": mount_path, - } - ) + return JSONResponse({ + "status": "ok", + "name": SERVER_NAME, + "mcp_path": mount_path, + }) @server.tool( name="search_agent_framework_docs", description="Return deterministic Agent Framework documentation text for MCP integration tests.", + annotations=types.ToolAnnotations(read_only_hint=True), ) def search_agent_framework_docs(query: str) -> str: """Return a deterministic response for the MCP integration tests.""" @@ -83,12 +80,14 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def main(argv: Sequence[str] | None = None) -> None: """Start the local MCP streamable HTTP server.""" args = parse_args(argv) - server = create_server( + mount_path = _normalize_mount_path(args.mount_path) + server = create_server(mount_path=mount_path) + server.run( + transport="streamable-http", host=args.host, port=args.port, - mount_path=_normalize_mount_path(args.mount_path), + streamable_http_path=mount_path, ) - server.run(transport="streamable-http") if __name__ == "__main__": From b3018e46a647a54bfa160351b7b67349b9feb6c2 Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:33:51 +0800 Subject: [PATCH 7/9] fix(core): remove duplicate MCP annotation state updates --- python/packages/core/agent_framework/_mcp.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/packages/core/agent_framework/_mcp.py b/python/packages/core/agent_framework/_mcp.py index 2358693a1a4..29cad42b21b 100644 --- a/python/packages/core/agent_framework/_mcp.py +++ b/python/packages/core/agent_framework/_mcp.py @@ -1294,7 +1294,6 @@ def __init__( self._tool_call_meta_by_name: dict[str, dict[str, Any]] = {} self._tool_annotations_by_name: dict[str, types.ToolAnnotations | None] = {} self._tool_param_names_by_name: dict[str, set[str]] = {} - self._tool_annotations_by_name: dict[str, Any] = {} self._global_extra_arg_names, self._tool_extra_arg_names = _normalize_additional_tool_argument_names( additional_tool_argument_names ) @@ -2343,7 +2342,6 @@ def _reset_session_discovery_state(self) -> None: self._tool_call_meta_by_name.clear() self._tool_annotations_by_name.clear() self._tool_param_names_by_name.clear() - self._tool_annotations_by_name.clear() self._progressive_loaded_tool_names.clear() def _reset_session_state(self) -> None: @@ -3354,7 +3352,6 @@ def stage_tool_page(tool_list: types.ListToolsResult) -> None: self._tool_call_meta_by_name = tool_call_meta_by_name self._tool_annotations_by_name = tool_annotations_by_name self._tool_param_names_by_name = tool_param_names_by_name - self._tool_annotations_by_name = tool_annotations_by_name self._progressive_loaded_tool_names.difference_update(existing_tools.keys() - reused_tool_names) async def _cancel_pending_reload_tasks(self) -> None: From ec388cf3da117006f632e6ed225bfb3317640cb9 Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:09:50 +0800 Subject: [PATCH 8/9] fix(core): bind MCP replay eligibility to dispatch identity Capture replay permission after HTTP identity reconciliation and validate it again inside retry admission. A concurrent caller can no longer lend its idempotency hint to another principal's operation. Add a deterministic cross-identity regression and opt recovery-focused tests into replay explicitly. --- python/packages/core/agent_framework/_mcp.py | 88 ++++++++++++++----- .../core/tests/core/test_mcp_http_auth.py | 84 +++++++++++++++++- 2 files changed, 146 insertions(+), 26 deletions(-) diff --git a/python/packages/core/agent_framework/_mcp.py b/python/packages/core/agent_framework/_mcp.py index 7dcb3fc46fd..82bbc1b3001 100644 --- a/python/packages/core/agent_framework/_mcp.py +++ b/python/packages/core/agent_framework/_mcp.py @@ -16,6 +16,7 @@ from copy import copy from dataclasses import dataclass from datetime import date, datetime +from functools import partial from http.cookiejar import CookieJar, DefaultCookiePolicy from inspect import currentframe, isawaitable from typing import TYPE_CHECKING, Any, Literal, Protocol, Self, TypeAlias, TypedDict, TypeVar, cast @@ -3703,13 +3704,20 @@ async def _call_tool( ) async def _invoke_tool( - self, tool_name: str, kwargs: dict[str, Any], trace_context: otel_context.Context + self, + tool_name: str, + kwargs: dict[str, Any], + trace_context: otel_context.Context, + *, + before_dispatch: Callable[[], None] | None = None, ) -> types.CallToolResult: token = otel_context.attach(trace_context) try: filtered_kwargs, meta = self._prepare_call_kwargs(tool_name, kwargs) finally: otel_context.detach(token) + if before_dispatch is not None: + before_dispatch() return await self._require_connection().call_tool(tool_name, filtered_kwargs, meta=meta) async def _call_tool_with_retries( @@ -3724,14 +3732,41 @@ async def _call_tool_with_retries( from anyio import ClosedResourceError from mcp import MCPError - # A catalog refresh cannot retroactively make the first dispatch safe to repeat. - replay_eligible = self._is_call_replayable(tool_name) + replay_eligible = False + lost_connection: Exception | None = None + replay_error: ToolExecutionException | None = None + + def validate_replay_eligibility( + attempt: int, + connection_error: Exception | None, + unknown_outcome: ToolExecutionException | None, + ) -> None: + nonlocal replay_eligible + if attempt == 0: + replay_eligible = self._is_call_replayable(tool_name) + return + if replay_eligible and self._is_call_replayable(tool_name): + return + cause = cast(Exception, connection_error) + error = cast(ToolExecutionException, unknown_outcome) + raise error from cause + for attempt in range(2): failed_connection = self._connection close_generation = self._close_generation span_error_recorded = False try: - result = await self._invoke_tool(tool_name, kwargs, trace_context) + result = await self._invoke_tool( + tool_name, + kwargs, + trace_context, + before_dispatch=partial( + validate_replay_eligibility, + attempt, + lost_connection, + replay_error, + ), + ) _capture_mcp_tool_result(result) if result.is_error: parsed = parser(result) @@ -3748,7 +3783,10 @@ async def _call_tool_with_retries( return parser(result) except ToolExecutionException as tool_ex: if span.is_recording() and not span_error_recorded: - set_mcp_span_error(span, type(tool_ex).__name__, str(tool_ex)) + if tool_ex is replay_error and lost_connection is not None: + set_mcp_span_error(span, type(lost_connection).__name__, str(lost_connection)) + else: + set_mcp_span_error(span, type(tool_ex).__name__, str(tool_ex)) raise except (ClosedResourceError, MCPError) as call_ex: if not self._is_connection_loss(call_ex): @@ -3770,21 +3808,17 @@ async def _call_tool_with_retries( if span.is_recording(): set_mcp_span_error(span, type(recovery_error).__name__, str(recovery_error)) raise - if not replay_eligible or not self._is_call_replayable(tool_name): - # The request may have reached the server before the connection - # dropped, so replaying it could execute a non-idempotent operation - # twice. Surface the unknown outcome instead of a replay; the - # connection is already back up for whoever decides to retry. - if span.is_recording(): - set_mcp_span_error(span, type(call_ex).__name__, str(call_ex)) - raise _mcp_tool_execution_exception( - f"Failed to call tool '{tool_name}' - connection lost and the remote " - "execution outcome is unknown. The connection was re-established but " - "the call was not replayed, because replay requires both a server " - "idempotent/read-only hint and the client-owned " - "trust_server_idempotency_hints opt-in; retry explicitly if safe.", - inner_exception=call_ex, - ) from call_ex + # HTTP admission can reconcile a different identity on retry. Validate the + # original and replacement catalogs only after that reconciliation, before dispatch. + lost_connection = call_ex + replay_error = _mcp_tool_execution_exception( + f"Failed to call tool '{tool_name}' - connection lost and the remote " + "execution outcome is unknown. The connection was re-established but " + "the call was not replayed, because replay requires both a server " + "idempotent/read-only hint and the client-owned " + "trust_server_idempotency_hints opt-in; retry explicitly if safe.", + inner_exception=call_ex, + ) continue # Second attempt also failed, give up. @@ -4998,10 +5032,20 @@ async def _recover_failed_connection( ) async def _invoke_tool( - self, tool_name: str, kwargs: dict[str, Any], trace_context: otel_context.Context + self, + tool_name: str, + kwargs: dict[str, Any], + trace_context: otel_context.Context, + *, + before_dispatch: Callable[[], None] | None = None, ) -> types.CallToolResult: async with self._http_operation(): - return await super()._invoke_tool(tool_name, kwargs, trace_context) + return await super()._invoke_tool( + tool_name, + kwargs, + trace_context, + before_dispatch=before_dispatch, + ) async def _invoke_prompt(self, prompt_name: str, kwargs: dict[str, Any]) -> types.GetPromptResult: async with self._http_operation(): diff --git a/python/packages/core/tests/core/test_mcp_http_auth.py b/python/packages/core/tests/core/test_mcp_http_auth.py index f48fc114b2c..ff23d47db7c 100644 --- a/python/packages/core/tests/core/test_mcp_http_auth.py +++ b/python/packages/core/tests/core/test_mcp_http_auth.py @@ -5,7 +5,7 @@ import asyncio import contextlib import json -from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Collection from contextvars import ContextVar from typing import Any, Literal, TypeAlias, cast from unittest.mock import AsyncMock, Mock, patch @@ -31,7 +31,11 @@ async def mcp_http_client_factory() -> AsyncIterator[MCPHTTPClientFactory]: async def record_request(request: httpx.Request) -> None: requests.append(request) - async def handle(request: httpx.Request, protocol_version: str | None) -> httpx.Response: + async def handle( + request: httpx.Request, + protocol_version: str | None, + replayable_principals: Collection[str] = (), + ) -> httpx.Response: if request.url.path == "/unrelated": return httpx.Response(200) principal = request.headers.get("Authorization", "") @@ -75,6 +79,7 @@ async def handle(request: httpx.Request, protocol_version: str | None) -> httpx. ), }, }, + **({"annotations": {"idempotentHint": True}} if principal in replayable_principals else {}), }, { "name": f"{principal}-only", @@ -123,12 +128,13 @@ def create_client( response_cookies: dict[str, str] | None = None, before_request: Callable[[httpx.Request], Awaitable[None]] | None = None, protocol_version: str | None = None, + replayable_principals: Collection[str] = (), **kwargs: Any, ) -> httpx.AsyncClient: async def handle_with_cookies(request: httpx.Request) -> httpx.Response: if before_request is not None: await before_request(request) - response = await handle(request, protocol_version) + response = await handle(request, protocol_version, replayable_principals) if response_cookies and request.method == "POST": method = json.loads(request.content).get("method") if method in response_cookies: @@ -557,6 +563,7 @@ async def block_recovery() -> None: async with tool, asyncio.timeout(5): with ( patch.object(_connected_client(tool), "call_tool", side_effect=ClosedResourceError()), + patch.object(tool, "_is_call_replayable", return_value=True), patch.object(tool, method, side_effect=block_recovery), patch.object(tool, "request_timeout", timeout), ): @@ -582,6 +589,70 @@ async def block_recovery() -> None: assert tool._header_operations == 0 +async def test_replay_eligibility_follows_the_dispatched_identity_during_concurrent_recovery( + mcp_http_client_factory: MCPHTTPClientFactory, +) -> None: + from anyio import ClosedResourceError + + from agent_framework._mcp import _ClientMCPConnection + + create_client, _, writes = mcp_http_client_factory + client = create_client(replayable_principals={"token-a"}) + tool = _tool(client, "token-a") + tool.trust_server_idempotency_hints = True + b_started = asyncio.Event() + release_b = asyncio.Event() + a_finished = asyncio.Event() + original_call = _ClientMCPConnection.call_tool + original_recovery = tool._recover_failed_connection + b_dispatches = 0 + + async def fail_first_b_dispatch( + connection: Any, + name: str, + arguments: dict[str, Any] | None, + *, + meta: Any, + ) -> Any: + nonlocal b_dispatches + if name == "record" and (arguments or {}).get("marker") == "b": + b_dispatches += 1 + if b_dispatches == 1: + b_started.set() + await release_b.wait() + raise ClosedResourceError + return await original_call(connection, name, arguments, meta=meta) + + async def finish_a_before_b_recovery(**kwargs: Any) -> None: + await a_finished.wait() + await original_recovery(**kwargs) + + async with tool, asyncio.timeout(10): + with ( + patch.object(_ClientMCPConnection, "call_tool", new=fail_first_b_dispatch), + patch.object(tool, "_recover_failed_connection", side_effect=finish_a_before_b_recovery), + ): + b_call = asyncio.create_task(tool.call_tool("record", credential="token-b", marker="b")) + b_outcome: Any = None + try: + await b_started.wait() + a_call = asyncio.create_task(tool.call_tool("record", credential="token-a", marker="a")) + release_b.set() + await a_call + a_finished.set() + (b_outcome,) = await asyncio.gather(b_call, return_exceptions=True) + finally: + release_b.set() + a_finished.set() + await asyncio.gather(b_call, return_exceptions=True) + + assert isinstance(b_outcome, ToolExecutionException) + assert "outcome is unknown" in str(b_outcome) + assert b_dispatches == 1 + assert writes["token-a"] == ["a"] + assert writes["token-b"] == [] + + async def test_recovery_budget_includes_waiting_for_other_active_attempts( mcp_http_client_factory: MCPHTTPClientFactory, ) -> None: @@ -853,6 +924,7 @@ async def fail_replacement_prompt_page(sdk_client: Client, *, cursor: str | None failed_call = AsyncMock(side_effect=ClosedResourceError) with ( patch.object(connection.client, "call_tool", new=failed_call), + patch.object(tool, "_is_call_replayable", return_value=True), patch.object(Client, "list_prompts", new=fail_replacement_prompt_page), ): result = await tool.call_tool("session_1_tool_head") @@ -980,7 +1052,10 @@ async def fail_call(*args: Any, **kwargs: Any) -> None: async with tool: original_connection = tool._connection - with patch.object(_connected_client(tool), "call_tool", side_effect=fail_call): + with ( + patch.object(_connected_client(tool), "call_tool", side_effect=fail_call), + patch.object(tool, "_is_call_replayable", return_value=True), + ): first = asyncio.create_task(tool.call_tool("record", marker="first")) second = asyncio.create_task(tool.call_tool("record", marker="second")) try: @@ -1015,6 +1090,7 @@ async def fail_replacement_once() -> None: with ( patch.object(_connected_client(tool), "call_tool", side_effect=ClosedResourceError()), + patch.object(tool, "_is_call_replayable", return_value=True), patch.object(tool, "_check_connection", side_effect=fail_replacement_once), ): await tool.call_tool("record", marker="recovered") From b6637210da6c7d6df997cd029ee6609ac89e1cda Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Sat, 14 Nov 2026 22:45:04 +0800 Subject: [PATCH 9/9] Python: [BREAKING] stop replaying an MCP tools/call whose response was lost _call_tool_with_retries dispatched the same tools/call a second time after a ClosedResourceError or a session-terminated MCPError. The server can commit the operation before the response is lost, so a non-idempotent tool ran twice and the caller saw a successful result. Dispatch once. On connection loss, recover the connection through the existing recovery helpers and then raise ToolExecutionException stating that the remote outcome is unknown and the call was not retried, keeping the connection error as the cause and on the span, with a redacted warning log. A later, distinct call proceeds on the replacement connection. Caller-owned connections keep their existing "cannot be recreated" error, and tools/list and get_prompt keep their reconnect-and-retry behavior. Server ToolAnnotations stay advisory: idempotentHint and readOnlyHint do not authorize a replay, and no trust or idempotency option is added. Refs #9204 --- python/packages/core/AGENTS.md | 2 +- python/packages/core/README.md | 21 ++ python/packages/core/agent_framework/_mcp.py | 244 ++++++------------ python/packages/core/tests/core/test_mcp.py | 193 ++++++-------- .../core/tests/core/test_mcp_http_auth.py | 51 ++-- .../core/tests/core/test_mcp_observability.py | 21 +- .../local_mcp_streamable_http_server.py | 14 +- 7 files changed, 210 insertions(+), 336 deletions(-) diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index 8fccce685d9..3902a86caa0 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -324,7 +324,7 @@ The vector store API is experimental under the shared `VECTOR_STORES` feature ID and reconnect or effective header-identity changes replace a framework-owned Client's per-Client cache. Agent Framework does not expose shared-store or cache-mode configuration. - **MCP Tasks extension** - The current Agent Framework MCP client does not implement the `io.modelcontextprotocol/tasks` runtime. Tools that advertise `execution.taskSupport == "required"` are skipped with a warning; optional and task-free tools continue through ordinary `tools/call`. Tasks client support is tracked in microsoft/agent-framework#9215. -- **Plain `tools/call` replay gating** - On `ClosedResourceError` / session-terminated `MCPError` the request may already have executed server-side, so `_call_tool_with_retries` only replays after reconnect when the server advertised the tool with `idempotentHint` or `readOnlyHint` and the client opted in with `trust_server_idempotency_hints` (`_is_call_replayable`, backed by the `_tool_annotations_by_name` snapshot from `tools/list`); any other tool raises `ToolExecutionException` stating the outcome is unknown once the connection is back up, with the connection error as the inner exception, and leaves the retry decision to the caller. Eligibility is captured before the first dispatch and the refreshed catalog must preserve it: a reconnect cannot upgrade an originally unhinted operation into a replay, and a replacement that removes the hint also blocks replay. Caller-owned connections keep failing with their existing 'cannot be recreated' error before any of this. Read-only discovery (`tools/list`) and `get_prompt` keep their reconnect-and-retry behavior. +- **Plain `tools/call` never replays after connection loss** - On `ClosedResourceError` / session-terminated `MCPError` the request may already have executed server-side, so `_call_tool_with_retries` dispatches once. It recovers the connection through the existing `_recover_connection_after_loss` helpers, then raises `ToolExecutionException` stating the outcome is unknown and the call was not retried, with the connection error as the inner exception, a redacted WARNING log, and the connection error type on the span. Server `ToolAnnotations` are advisory and never authorize a replay, so there is no trust or idempotency setting. A later, distinct call proceeds on the replacement connection. Caller-owned connections keep failing with their existing 'cannot be recreated' error before any of this. Read-only discovery (`tools/list`) and `get_prompt` keep their reconnect-and-retry behavior. ### File Access Harness (`_harness/_file_access.py`) diff --git a/python/packages/core/README.md b/python/packages/core/README.md index 6095f24a7ed..604b72dfc9d 100644 --- a/python/packages/core/README.md +++ b/python/packages/core/README.md @@ -179,6 +179,27 @@ asyncio.run(main()) You can explore additional agent samples [here](https://github.com/microsoft/agent-framework/tree/main/python/samples/02-agents). +### MCP tool calls after a lost response + +**Breaking change:** a `tools/call` whose connection drops is no longer sent a second +time. The server may have executed the request before the response was lost, so +re-issuing it could run a non-idempotent operation twice. `call_tool` and tool +invocations now raise `ToolExecutionException` stating that the outcome is unknown +and the call was not retried, with the connection error as `__cause__`. Server +`idempotentHint` and `readOnlyHint` annotations are advisory and do not enable a +retry. Discovery (`tools/list`) and `get_prompt` keep their existing retry behavior. + +For framework-owned connections the connection is still recovered before the +exception surfaces, so a later, different call proceeds normally. Caller-supplied +`Client` and `ClientSession` objects are never recreated and keep their existing +"caller-owned and cannot be recreated" error. + +Catch `ToolExecutionException` around calls that must not run twice and reconcile the +outcome in your own code, for example by reading back the record the tool writes, or +by using a server that enforces an idempotency key. Then retry explicitly if it is +safe. This prevents the framework from duplicating an operation; it is not an +exactly-once guarantee. + ### MCP HTTP client ownership and cookies **Breaking change:** HTTP clients created by `MCPStreamableHTTPTool` no longer persist diff --git a/python/packages/core/agent_framework/_mcp.py b/python/packages/core/agent_framework/_mcp.py index 0a1501fff53..0951d3cb1bf 100644 --- a/python/packages/core/agent_framework/_mcp.py +++ b/python/packages/core/agent_framework/_mcp.py @@ -16,7 +16,6 @@ from copy import copy from dataclasses import dataclass from datetime import date, datetime -from functools import partial from http.cookiejar import CookieJar, DefaultCookiePolicy from inspect import currentframe, isawaitable from typing import TYPE_CHECKING, Any, Literal, Protocol, Self, TypeAlias, TypedDict, TypeVar, cast @@ -1301,7 +1300,6 @@ def __init__( max_discovery_items: int | None = _DEFAULT_MCP_DISCOVERY_MAX_ITEMS, max_host_payload_size_bytes: int | None = _DEFAULT_MCP_HOST_PAYLOAD_SIZE_BYTES, tool_result_content: MCPToolResultContentMode = "structured_first", - trust_server_idempotency_hints: bool = False, ) -> None: """Initialize the MCP Tool base. @@ -1359,7 +1357,6 @@ def __init__( self.max_discovery_pages = max_discovery_pages self.max_discovery_items = max_discovery_items self.max_host_payload_size_bytes = max_host_payload_size_bytes - self.trust_server_idempotency_hints = trust_server_idempotency_hints self._exit_stack = AsyncExitStack() self._lifecycle_lock = asyncio.Lock() self._lifecycle_request_lock = asyncio.Lock() @@ -1410,8 +1407,8 @@ def __init__( self._progressive_loader_functions: list[FunctionTool] | None = None self._progressive_loaded_tool_names: set[str] = set() self._tool_call_meta_by_name: dict[str, dict[str, Any]] = {} - self._tool_annotations_by_name: dict[str, types.ToolAnnotations | None] = {} self._tool_param_names_by_name: dict[str, set[str]] = {} + self._tool_annotations_by_name: dict[str, Any] = {} self._global_extra_arg_names, self._tool_extra_arg_names = _normalize_additional_tool_argument_names( additional_tool_argument_names ) @@ -2471,8 +2468,8 @@ def _reset_session_discovery_state(self) -> None: if not isinstance((function.additional_properties or {}).get(_MCP_REMOTE_NAME_KEY), str) ] self._tool_call_meta_by_name.clear() - self._tool_annotations_by_name.clear() self._tool_param_names_by_name.clear() + self._tool_annotations_by_name.clear() self._progressive_loaded_tool_names.clear() def _reset_session_state(self) -> None: @@ -3616,8 +3613,8 @@ def stage_tool_page(tool_list: types.ListToolsResult) -> None: function.approval_mode = tool_approval_updates[function.name] self._functions[:] = current_functions self._tool_call_meta_by_name = tool_call_meta_by_name - self._tool_annotations_by_name = tool_annotations_by_name self._tool_param_names_by_name = tool_param_names_by_name + self._tool_annotations_by_name = tool_annotations_by_name self._progressive_loaded_tool_names.difference_update(existing_tools.keys() - reused_tool_names) async def _cancel_pending_reload_tasks(self) -> None: @@ -3760,7 +3757,9 @@ async def call_tool(self, tool_name: str, **kwargs: Any) -> str | list[Content]: Raises: ToolExecutionException: If the MCP server is not connected, tools are not loaded, - or the tool call fails. + or the tool call fails. A dropped connection raises with the remote execution + outcome unknown and the connection error as the cause: the call is not retried, + because it may already have executed on the server. """ return await self._call_tool(tool_name, kwargs) @@ -3797,20 +3796,13 @@ async def _call_tool( ) async def _invoke_tool( - self, - tool_name: str, - kwargs: dict[str, Any], - trace_context: otel_context.Context, - *, - before_dispatch: Callable[[], None] | None = None, + self, tool_name: str, kwargs: dict[str, Any], trace_context: otel_context.Context ) -> types.CallToolResult: token = otel_context.attach(trace_context) try: filtered_kwargs, meta = self._prepare_call_kwargs(tool_name, kwargs) finally: otel_context.detach(token) - if before_dispatch is not None: - before_dispatch() return await self._require_connection().call_tool(tool_name, filtered_kwargs, meta=meta) async def _call_tool_with_retries( @@ -3821,137 +3813,76 @@ async def _call_tool_with_retries( parser: Callable[..., str | list[Content]], span: otel_trace.Span, ) -> str | list[Content]: - """Execute the MCP tools/call RPC with retry logic.""" + """Execute the MCP tools/call RPC, recovering the connection without replaying the call.""" from anyio import ClosedResourceError from mcp import MCPError - replay_eligible = False - lost_connection: Exception | None = None - replay_error: ToolExecutionException | None = None - - def validate_replay_eligibility( - attempt: int, - connection_error: Exception | None, - unknown_outcome: ToolExecutionException | None, - ) -> None: - nonlocal replay_eligible - if attempt == 0: - replay_eligible = self._is_call_replayable(tool_name) - return - if replay_eligible and self._is_call_replayable(tool_name): - return - cause = cast(Exception, connection_error) - error = cast(ToolExecutionException, unknown_outcome) - raise error from cause - - for attempt in range(2): - failed_connection = self._connection - close_generation = self._close_generation - span_error_recorded = False - try: - result = await self._invoke_tool( - tool_name, - kwargs, - trace_context, - before_dispatch=partial( - validate_replay_eligibility, - attempt, - lost_connection, - replay_error, - ), - ) - _capture_mcp_result(self, result) - if result.is_error: - parsed = parser(result) - text = ( - "\n".join(c.text for c in parsed if c.type == "text" and c.text) - if isinstance(parsed, list) - else str(parsed) - ) - # Per OTel MCP semconv: set error.type="tool_error" for isError results - if span.is_recording(): - set_mcp_span_error(span, "tool_error", text or str(parsed)) - span_error_recorded = True - raise _mcp_tool_execution_exception(text or str(parsed)) - return parser(result) - except ToolExecutionException as tool_ex: - if span.is_recording() and not span_error_recorded: - if tool_ex is replay_error and lost_connection is not None: - set_mcp_span_error(span, type(lost_connection).__name__, str(lost_connection)) - else: - set_mcp_span_error(span, type(tool_ex).__name__, str(tool_ex)) - raise - except (ClosedResourceError, MCPError) as call_ex: - if not self._is_connection_loss(call_ex): - error_message = call_ex.error.message if isinstance(call_ex, MCPError) else str(call_ex) - if span.is_recording(): - set_mcp_span_error(span, type(call_ex).__name__, error_message) - raise _mcp_tool_execution_exception(error_message, inner_exception=call_ex) from call_ex - - if attempt == 0: - try: - await self._recover_connection_after_loss( - operation="tool call", - inner_exception=call_ex, - load_configured=True, - failed_connection=failed_connection, - close_generation=close_generation, - ) - except ToolExecutionException as recovery_error: - if span.is_recording(): - set_mcp_span_error(span, type(recovery_error).__name__, str(recovery_error)) - raise - # HTTP admission can reconcile a different identity on retry. Validate the - # original and replacement catalogs only after that reconciliation, before dispatch. - lost_connection = call_ex - replay_error = _mcp_tool_execution_exception( - f"Failed to call tool '{tool_name}' - connection lost and the remote " - "execution outcome is unknown. The connection was re-established but " - "the call was not replayed, because replay requires both a server " - "idempotent/read-only hint and the client-owned " - "trust_server_idempotency_hints opt-in; retry explicitly if safe.", - inner_exception=call_ex, - ) - continue - - # Second attempt also failed, give up. - _log_mcp_exception( - logging.ERROR, - call_ex, - "MCP connection closed unexpectedly after reconnection: %s", - call_ex, - redacted_message="MCP tool call connection closed after reconnection", + failed_connection = self._connection + close_generation = self._close_generation + span_error_recorded = False + try: + result = await self._invoke_tool(tool_name, kwargs, trace_context) + _capture_mcp_result(self, result) + if result.is_error: + parsed = parser(result) + text = ( + "\n".join(c.text for c in parsed if c.type == "text" and c.text) + if isinstance(parsed, list) + else str(parsed) ) + # Per OTel MCP semconv: set error.type="tool_error" for isError results if span.is_recording(): - set_mcp_span_error(span, type(call_ex).__name__, str(call_ex)) - raise _mcp_tool_execution_exception( - f"Failed to call tool '{tool_name}' - connection lost.", + set_mcp_span_error(span, "tool_error", text or str(parsed)) + span_error_recorded = True + raise _mcp_tool_execution_exception(text or str(parsed)) + return parser(result) + except ToolExecutionException as tool_ex: + if span.is_recording() and not span_error_recorded: + set_mcp_span_error(span, type(tool_ex).__name__, str(tool_ex)) + raise + except (ClosedResourceError, MCPError) as call_ex: + if not self._is_connection_loss(call_ex): + error_message = call_ex.error.message if isinstance(call_ex, MCPError) else str(call_ex) + if span.is_recording(): + set_mcp_span_error(span, type(call_ex).__name__, error_message) + raise _mcp_tool_execution_exception(error_message, inner_exception=call_ex) from call_ex + + # The server may have executed the request before the response was lost, so the + # outcome is unknown and re-issuing it could duplicate a non-idempotent effect. + # Restore availability for the next call and leave the retry decision to the caller. + try: + await self._recover_connection_after_loss( + operation="tool call", inner_exception=call_ex, - ) from call_ex - except Exception as ex: + load_configured=True, + failed_connection=failed_connection, + close_generation=close_generation, + ) + except ToolExecutionException as recovery_error: if span.is_recording(): - set_mcp_span_error(span, type(ex).__name__, str(ex)) - raise _mcp_tool_execution_exception(f"Failed to call tool '{tool_name}'.", inner_exception=ex) from ex - retries_exhausted = _mcp_tool_execution_exception(f"Failed to call tool '{tool_name}' after retries.") - if span.is_recording(): - set_mcp_span_error(span, type(retries_exhausted).__name__, str(retries_exhausted)) - raise retries_exhausted - - def _is_call_replayable(self, tool_name: str) -> bool: - """Return True if *tool_name* may be replayed after connection loss. - - Replay requires both a client-owned trust decision - (``trust_server_idempotency_hints=True``) and a server annotation carrying - ``idempotentHint`` or ``readOnlyHint``. Annotations are supplied by the - server and are untrusted by default, consistent with how the rest of the - package treats ``ToolAnnotations`` as restriction-only hints: without the - client's explicit opt-in, a misannotated or untrusted server could get a - non-idempotent ``tools/call`` executed twice on a lost response. - """ - if not self.trust_server_idempotency_hints: - return False - annotations = self._tool_annotations_by_name.get(tool_name) - return bool(annotations and (annotations.idempotent_hint or annotations.read_only_hint)) + set_mcp_span_error(span, type(recovery_error).__name__, str(recovery_error)) + raise + if span.is_recording(): + set_mcp_span_error(span, type(call_ex).__name__, str(call_ex)) + _log_mcp_exception( + logging.WARNING, + call_ex, + "MCP tool call %r lost its response and was not retried: %s", + tool_name, + call_ex, + redacted_message="MCP tool call outcome is unknown after connection loss; the call was not retried", + ) + raise _mcp_tool_execution_exception( + f"Failed to call tool '{tool_name}' - connection lost and the remote execution " + "outcome is unknown. The call was not retried because it may already have " + "executed on the server. The connection was re-established, so a later call " + "can proceed; reconcile the outcome before retrying this one.", + inner_exception=call_ex, + ) from call_ex + except Exception as ex: + if span.is_recording(): + set_mcp_span_error(span, type(ex).__name__, str(ex)) + raise _mcp_tool_execution_exception(f"Failed to call tool '{tool_name}'.", inner_exception=ex) from ex def _resolved_extra_args(self, tool_name: str) -> set[str]: """Return the user-configured extra argument names allowed for a tool.""" @@ -4197,7 +4128,6 @@ def __init__( max_discovery_items: int | None = _DEFAULT_MCP_DISCOVERY_MAX_ITEMS, max_host_payload_size_bytes: int | None = _DEFAULT_MCP_HOST_PAYLOAD_SIZE_BYTES, tool_result_content: MCPToolResultContentMode = "structured_first", - trust_server_idempotency_hints: bool = False, **kwargs: Any, ) -> None: """Initialize the MCP stdio tool. @@ -4321,11 +4251,6 @@ def __init__( transports. ``None`` disables the limit. tool_result_content: How to choose model-visible text when both ``content`` and ``structuredContent`` are present. See :data:`MCPToolResultContentMode`. - trust_server_idempotency_hints: When ``True``, treat server-declared - ``idempotentHint``/``readOnlyHint`` tool annotations as trustworthy, allowing - a ``tools/call`` whose response was lost to a dropped connection to be retried - once after reconnect. Annotations are server-supplied and untrusted by default, - so the default ``False`` never replays an ambiguous call. kwargs: Additional arguments for framework-owned ``StdioServerParameters`` construction. """ if mcp_client is not None and session is not None: @@ -4384,7 +4309,6 @@ def __init__( max_discovery_items=max_discovery_items, max_host_payload_size_bytes=max_host_payload_size_bytes, tool_result_content=tool_result_content, - trust_server_idempotency_hints=trust_server_idempotency_hints, ) self.command = command self.args = args or [] @@ -4487,7 +4411,6 @@ def __init__( max_discovery_items: int | None = _DEFAULT_MCP_DISCOVERY_MAX_ITEMS, max_host_payload_size_bytes: int | None = _DEFAULT_MCP_HOST_PAYLOAD_SIZE_BYTES, tool_result_content: MCPToolResultContentMode = "structured_first", - trust_server_idempotency_hints: bool = False, **kwargs: Any, ) -> None: """Initialize the MCP streamable HTTP tool. @@ -4685,11 +4608,6 @@ def __init__( transports. ``None`` disables the limit. tool_result_content: How to choose model-visible text when both ``content`` and ``structuredContent`` are present. See :data:`MCPToolResultContentMode`. - trust_server_idempotency_hints: When ``True``, treat server-declared - ``idempotentHint``/``readOnlyHint`` tool annotations as trustworthy, allowing - a ``tools/call`` whose response was lost to a dropped connection to be retried - once after reconnect. Annotations are server-supplied and untrusted by default, - so the default ``False`` never replays an ambiguous call. kwargs: Additional keyword arguments (accepted for backward compatibility but not used). """ if mcp_client is not None and session is not None: @@ -4748,7 +4666,6 @@ def __init__( max_discovery_items=max_discovery_items, max_host_payload_size_bytes=max_host_payload_size_bytes, tool_result_content=tool_result_content, - trust_server_idempotency_hints=trust_server_idempotency_hints, ) self.url = url self.terminate_on_close = terminate_on_close @@ -5126,20 +5043,10 @@ async def _recover_failed_connection( ) async def _invoke_tool( - self, - tool_name: str, - kwargs: dict[str, Any], - trace_context: otel_context.Context, - *, - before_dispatch: Callable[[], None] | None = None, + self, tool_name: str, kwargs: dict[str, Any], trace_context: otel_context.Context ) -> types.CallToolResult: async with self._http_operation(): - return await super()._invoke_tool( - tool_name, - kwargs, - trace_context, - before_dispatch=before_dispatch, - ) + return await super()._invoke_tool(tool_name, kwargs, trace_context) async def _invoke_prompt(self, prompt_name: str, kwargs: dict[str, Any]) -> types.GetPromptResult: async with self._http_operation(): @@ -5360,7 +5267,6 @@ def __init__( max_discovery_items: int | None = _DEFAULT_MCP_DISCOVERY_MAX_ITEMS, max_host_payload_size_bytes: int | None = _DEFAULT_MCP_HOST_PAYLOAD_SIZE_BYTES, tool_result_content: MCPToolResultContentMode = "structured_first", - trust_server_idempotency_hints: bool = False, **kwargs: Any, ) -> None: """Initialize the deprecated MCP WebSocket compatibility wrapper. @@ -5468,11 +5374,6 @@ def __init__( transports. ``None`` disables the limit. tool_result_content: How to choose model-visible text when both ``content`` and ``structuredContent`` are present. See :data:`MCPToolResultContentMode`. - trust_server_idempotency_hints: When ``True``, treat server-declared - ``idempotentHint``/``readOnlyHint`` tool annotations as trustworthy, allowing - a ``tools/call`` whose response was lost to a dropped connection to be retried - once after reconnect. Annotations are server-supplied and untrusted by default, - so the default ``False`` never replays an ambiguous call. kwargs: Deprecated compatibility arguments. They are not used. """ super().__init__( @@ -5500,7 +5401,6 @@ def __init__( max_discovery_items=max_discovery_items, max_host_payload_size_bytes=max_host_payload_size_bytes, tool_result_content=tool_result_content, - trust_server_idempotency_hints=trust_server_idempotency_hints, ) self.url = url self._client_kwargs = kwargs diff --git a/python/packages/core/tests/core/test_mcp.py b/python/packages/core/tests/core/test_mcp.py index 077e907eada..dd88c1511ed 100644 --- a/python/packages/core/tests/core/test_mcp.py +++ b/python/packages/core/tests/core/test_mcp.py @@ -4507,8 +4507,8 @@ def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: await func.invoke(param="test_value") -async def test_mcp_tool_reconnects_after_session_terminated_error(): - """Trusted idempotent tools reconnect once and retry after session termination.""" +async def test_mcp_tool_recovers_connection_after_session_terminated_error_without_replay(): + """Session termination reconnects once, but the ambiguous call is not re-issued.""" class TestServer(MCPTool): def __init__(self, **kwargs: Any) -> None: @@ -4533,44 +4533,6 @@ async def _reconnect_without_loading(self) -> None: def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: return None # type: ignore[return-value] # pyrefly: ignore[bad-return] # ty: ignore[invalid-return-type] - server = TestServer(name="test_server", trust_server_idempotency_hints=True) - await server._reconnect_without_loading() - server._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotent_hint=True) - - result = await server.call_tool("test_tool", param="test_value") - - assert _mcp_result_to_text(result) == "recovered" - assert server.connect_count == 2 - assert server.sessions[0].call_tool.await_count == 1 - assert server.sessions[1].call_tool.await_count == 1 - - -async def test_mcp_tool_call_not_replayed_without_idempotency_hint(): - """Tools without an idempotency hint raise an unknown-outcome error instead of replaying. - - The connection can drop after the server executed the request, so re-issuing the same - tools/call could duplicate a non-idempotent side effect. The default path surfaces the - failure without replaying: the connection is re-established, and the caller decides - whether to retry explicitly. - """ - - class TestServer(MCPTool): - def __init__(self, **kwargs: Any) -> None: - super().__init__(**kwargs) - self.connect_count = 0 - self.call_tool_mock = AsyncMock(side_effect=MCPError(-32000, "Session terminated")) - - async def _reconnect_without_loading(self) -> None: - self.connect_count += 1 - session = Mock(spec=ClientSession) - session.call_tool = self.call_tool_mock - _set_framework_owned_session(self, session) - self.is_connected = True - self._tools_loaded = self._prompts_loaded = True - - def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: - return None # type: ignore[return-value] # pyrefly: ignore[bad-return] # ty: ignore[invalid-return-type] - server = TestServer(name="test_server") await server._reconnect_without_loading() @@ -4579,53 +4541,36 @@ def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: assert isinstance(exc_info.value.__cause__, MCPError) assert server.connect_count == 2 - assert server.call_tool_mock.await_count == 1 - - -async def test_mcp_tool_call_not_replayed_when_reconnect_adds_hint(): - """A replacement catalog cannot make an already dispatched operation replayable.""" - - class TestServer(MCPTool): - def __init__(self, **kwargs: Any) -> None: - super().__init__(**kwargs) - self.connect_count = 0 - self.sessions: list[Any] = [] - - async def _reconnect_without_loading(self) -> None: - self.connect_count += 1 - mock_session = Mock(spec=ClientSession) - _set_framework_owned_session(self, mock_session) - self.sessions.append(self.session) - if self.connect_count == 1: - mock_session.call_tool = AsyncMock(side_effect=MCPError(-32000, "Session terminated")) - else: - self._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotent_hint=True) - mock_session.call_tool = AsyncMock( - return_value=types.CallToolResult(content=[types.TextContent(type="text", text="recovered")]) - ) - self.is_connected = True - self._tools_loaded = self._prompts_loaded = True - - def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: - return None # type: ignore[return-value] # pyrefly: ignore[bad-return] # ty: ignore[invalid-return-type] - - server = TestServer(name="test_server", trust_server_idempotency_hints=True) - await server._reconnect_without_loading() + assert server.sessions[0].call_tool.await_count == 1 + assert server.sessions[1].call_tool.await_count == 0 - with pytest.raises(ToolExecutionException, match="outcome is unknown"): - await server.call_tool("test_tool", param="test_value") + # Availability is restored for the next operation, which is a distinct call. + result = await server.call_tool("other_tool", param="test_value") + assert _mcp_result_to_text(result) == "recovered" assert server.connect_count == 2 assert server.sessions[0].call_tool.await_count == 1 - assert server.sessions[1].call_tool.await_count == 0 + assert server.sessions[1].call_tool.await_count == 1 -async def test_mcp_tool_call_not_replayed_on_hint_alone_without_client_trust(): - """A server idempotency hint by itself must not authorize replay. +@pytest.mark.parametrize( + "annotations", + [ + None, + types.ToolAnnotations(idempotent_hint=True), + types.ToolAnnotations(read_only_hint=True), + types.ToolAnnotations(idempotent_hint=True, read_only_hint=True), + ], + ids=["unhinted", "idempotent_hint", "read_only_hint", "both_hints"], +) +async def test_mcp_tool_call_is_not_replayed_regardless_of_server_annotations( + annotations: types.ToolAnnotations | None, +): + """Server annotations are advisory and never authorize replay of an ambiguous call. - ToolAnnotations are server-supplied and untrusted unless the client separately - trusts the server, so with the default trust posture a hinted tool still gets - the unknown-outcome error instead of a replay. + The connection can drop after the server executed the request, so re-issuing the same + tools/call could duplicate a non-idempotent side effect. A server can also advertise a + hint incorrectly, so the failure surfaces without replay for every annotation posture. """ class TestServer(MCPTool): @@ -4647,28 +4592,35 @@ def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: server = TestServer(name="test_server") await server._reconnect_without_loading() - server._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotent_hint=True) + server._tool_annotations_by_name["test_tool"] = annotations - with pytest.raises(ToolExecutionException, match="outcome is unknown"): + with pytest.raises(ToolExecutionException, match="outcome is unknown") as exc_info: await server.call_tool("test_tool", param="test_value") + assert isinstance(exc_info.value.__cause__, MCPError) assert server.connect_count == 2 assert server.call_tool_mock.await_count == 1 async def test_mcp_regular_tool_call_does_not_duplicate_side_effect_after_disconnect(): - """A non-idempotent tool whose response is lost mid-call executes at most once.""" + """A non-idempotent tool whose response is lost mid-call executes at most once. + + The replacement connection would answer successfully, so a replay would hide the + duplicate. The caller gets the unknown-outcome failure and the next call still works. + """ from anyio.streams.memory import ClosedResourceError class TestServer(MCPTool): def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self.state = 0 + self.dispatched: list[str] = [] async def _reconnect_without_loading(self) -> None: session = Mock(spec=ClientSession) - async def call_tool(*args: Any, **kwargs: Any) -> types.CallToolResult: + async def call_tool(name: str, *args: Any, **kwargs: Any) -> types.CallToolResult: + self.dispatched.append(name) self.state += 1 if self.state == 1: raise ClosedResourceError @@ -4685,16 +4637,22 @@ def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: server = TestServer(name="test_server") await server._reconnect_without_loading() - with pytest.raises(ToolExecutionException, match="outcome is unknown"): + with pytest.raises(ToolExecutionException, match="outcome is unknown") as exc_info: await server.call_tool("test_tool", param="test_value") + assert isinstance(exc_info.value.__cause__, ClosedResourceError) assert server.state == 1 + assert server.dispatched == ["test_tool"] + + assert _mcp_result_to_text(await server.call_tool("other_tool", param="test_value")) == "ok" + assert server.state == 2 + assert server.dispatched == ["test_tool", "other_tool"] async def test_connect_discovery_failure_rolls_back_annotations_snapshot(): """When load_tools succeeds but a later discovery step fails, the rollback in _connect_on_owner must restore the pre-discovery annotations map along with the - other metadata, or a later replay decision reads hints from the failed session.""" + other metadata, or security labels are derived from the failed session's hints.""" tool = MCPStdioTool(name="test", command="test-command") sentinel_annotations = {"prev_tool": types.ToolAnnotations(read_only_hint=True)} sentinel_meta = {"prev_tool": {"version": 0}} @@ -6068,14 +6026,10 @@ async def test_mcp_connection_reset_integration(): """ url = os.environ.get("LOCAL_MCP_URL") - # The query tool below is annotated read-only before the simulated drop; - # replay also needs the client-owned trust opt-in, which the default - # posture deliberately withholds. tool = MCPStreamableHTTPTool( name="integration_test", url=url, approval_mode="never_require", # type: ignore[arg-type] - trust_server_idempotency_hints=True, ) # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] async with tool: @@ -6100,9 +6054,6 @@ async def test_mcp_connection_reset_integration(): original_exit_stack = tool._exit_stack original_call_tool = original_client.call_tool - # The local server advertises this query tool as read-only in tools/list. - # Reconnection must discover the same hint before the trusted replay. - # Simulate connection failure by making call_tool raise ClosedResourceError once call_count = 0 @@ -6118,12 +6069,13 @@ async def call_tool_with_error(*args, **kwargs): return await original_call_tool(*args, **kwargs) # Patch the active high-level Client, not the deliberately empty low-level - # session. Reconnection replaces this Client before the trusted replay. - with patch.object(original_client, "call_tool", new=call_tool_with_error): - second_result = _mcp_result_to_text(await func.invoke(query="What is Agent Framework?")) + # session. The lost response is reported without replaying the call. + with ( + patch.object(original_client, "call_tool", new=call_tool_with_error), + pytest.raises(ToolExecutionException, match="outcome is unknown"), + ): + await func.invoke(query="What is Agent Framework?") assert call_count == 1 - assert second_result is not None - assert len(second_result) > 0 # Verify reconnection replaced both the framework-owned Client and its exit stack. new_connection = tool._connection @@ -6138,9 +6090,11 @@ async def call_tool_with_error(*args, **kwargs): # Verify tools are still available after reconnection assert len(tool.functions) > 0 - # Both results should include text (we don't compare content as it may vary) - assert len(first_result) > 0 + # A later, distinct call succeeds on the replacement connection. + second_result = _mcp_result_to_text(await func.invoke(query="What is Agent Framework?")) + assert second_result is not None assert len(second_result) > 0 + assert len(first_result) > 0 async def test_mcp_tool_message_handler_notification(): @@ -9393,9 +9347,8 @@ async def test_mcp_tool_connection_properly_invalidated_after_closed_resource_er This test verifies the fix for issue #2884: the tool tries operations optimistically and only reconnects when ClosedResourceError is encountered, avoiding extra latency. - The replay only happens because the client opted into trusting the tool's idempotency - hint here; tools without that hint, or without the client opt-in, raise instead of - retrying (issue #9204). + Recovery restores the connection for the next operation; the call whose response was + lost is reported rather than re-issued (issue #9204). """ from unittest.mock import AsyncMock, MagicMock, patch @@ -9410,9 +9363,7 @@ async def test_mcp_tool_connection_properly_invalidated_after_closed_resource_er command="test_command", args=["arg1"], load_tools=True, - trust_server_idempotency_hints=True, ) - tool._tool_annotations_by_name["test_tool"] = types.ToolAnnotations(idempotent_hint=True) # Mock the session mock_session = MagicMock() @@ -9442,7 +9393,7 @@ async def restore_session(): result = await tool.call_tool("test_tool", arg1="value1") assert result is not None - # Test Case 1: Connection closed unexpectedly, should reconnect and retry + # Test Case 1: Connection closed unexpectedly, should reconnect without replay # Simulate ClosedResourceError on first call, then succeed call_count = 0 @@ -9456,13 +9407,18 @@ async def call_tool_with_error(*args, **kwargs): mock_session.call_tool = call_tool_with_error # This call should trigger reconnection after ClosedResourceError - result = await tool.call_tool("test_tool", arg1="value2") - assert result is not None + with pytest.raises(ToolExecutionException, match="outcome is unknown"): + await tool.call_tool("test_tool", arg1="value2") + assert call_count == 1 assert mock_connect.call_count >= 1 mock_connect.assert_awaited_once_with() # Verify _exit_stack.aclose was called during reconnection original_exit_stack.aclose.assert_called() # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + # The replacement connection serves the next call. + assert await tool.call_tool("test_tool", arg1="value2") is not None + assert call_count == 2 + # Test Case 2: Reconnection failure # Reset counters call_count = 0 @@ -9644,28 +9600,25 @@ async def test_mcp_tool_get_prompt_requires_loaded_prompts() -> None: await tool.get_prompt("remote_prompt") -async def test_mcp_tool_call_tool_raises_after_reconnection_still_fails() -> None: +async def test_mcp_tool_call_tool_reports_unknown_outcome_once_after_connection_loss() -> None: from anyio.streams.memory import ClosedResourceError - tool = MCPTool( # type: ignore[abstract] # ty: ignore[call-non-callable] - name="test_tool", load_tools=True, trust_server_idempotency_hints=True - ) - _set_framework_owned_session( - tool, - Mock(call_tool=AsyncMock(side_effect=[ClosedResourceError(), ClosedResourceError()])), - ) + call_tool = AsyncMock(side_effect=[ClosedResourceError(), ClosedResourceError()]) + tool = MCPTool(name="test_tool", load_tools=True) # type: ignore[abstract] # ty: ignore[call-non-callable] + _set_framework_owned_session(tool, Mock(call_tool=call_tool)) tool._tools_loaded = tool._prompts_loaded = True - tool._tool_annotations_by_name["remote_tool"] = types.ToolAnnotations(idempotent_hint=True) with ( patch.object(tool, "_reconnect_without_loading", AsyncMock()) as mock_connect, - patch.object(logger, "error") as mock_error, - pytest.raises(ToolExecutionException, match="connection lost"), + patch.object(logger, "warning") as mock_warning, + pytest.raises(ToolExecutionException, match="outcome is unknown") as exc_info, ): await tool.call_tool("remote_tool") + assert isinstance(exc_info.value.__cause__, ClosedResourceError) + assert call_tool.await_count == 1 mock_connect.assert_awaited_once_with() - mock_error.assert_called_once() + mock_warning.assert_called_once() async def test_mcp_tool_get_prompt_raises_after_reconnection_still_fails() -> None: diff --git a/python/packages/core/tests/core/test_mcp_http_auth.py b/python/packages/core/tests/core/test_mcp_http_auth.py index ff23d47db7c..e0a42fe3f84 100644 --- a/python/packages/core/tests/core/test_mcp_http_auth.py +++ b/python/packages/core/tests/core/test_mcp_http_auth.py @@ -5,7 +5,7 @@ import asyncio import contextlib import json -from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Collection +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable from contextvars import ContextVar from typing import Any, Literal, TypeAlias, cast from unittest.mock import AsyncMock, Mock, patch @@ -31,11 +31,7 @@ async def mcp_http_client_factory() -> AsyncIterator[MCPHTTPClientFactory]: async def record_request(request: httpx.Request) -> None: requests.append(request) - async def handle( - request: httpx.Request, - protocol_version: str | None, - replayable_principals: Collection[str] = (), - ) -> httpx.Response: + async def handle(request: httpx.Request, protocol_version: str | None) -> httpx.Response: if request.url.path == "/unrelated": return httpx.Response(200) principal = request.headers.get("Authorization", "") @@ -79,7 +75,6 @@ async def handle( ), }, }, - **({"annotations": {"idempotentHint": True}} if principal in replayable_principals else {}), }, { "name": f"{principal}-only", @@ -128,13 +123,12 @@ def create_client( response_cookies: dict[str, str] | None = None, before_request: Callable[[httpx.Request], Awaitable[None]] | None = None, protocol_version: str | None = None, - replayable_principals: Collection[str] = (), **kwargs: Any, ) -> httpx.AsyncClient: async def handle_with_cookies(request: httpx.Request) -> httpx.Response: if before_request is not None: await before_request(request) - response = await handle(request, protocol_version, replayable_principals) + response = await handle(request, protocol_version) if response_cookies and request.method == "POST": method = json.loads(request.content).get("method") if method in response_cookies: @@ -563,7 +557,6 @@ async def block_recovery() -> None: async with tool, asyncio.timeout(5): with ( patch.object(_connected_client(tool), "call_tool", side_effect=ClosedResourceError()), - patch.object(tool, "_is_call_replayable", return_value=True), patch.object(tool, method, side_effect=block_recovery), patch.object(tool, "request_timeout", timeout), ): @@ -575,8 +568,9 @@ async def block_recovery() -> None: await asyncio.sleep(0) assert not call.done() release.set() - await call - assert writes["token-a"] == ["retry"] + with pytest.raises(ToolExecutionException, match="outcome is unknown"): + await call + assert writes["token-a"] == [] else: with pytest.raises(ToolExecutionException, match="Timed out recovering") as error: await call @@ -589,7 +583,7 @@ async def block_recovery() -> None: assert tool._header_operations == 0 -async def test_replay_eligibility_follows_the_dispatched_identity_during_concurrent_recovery( +async def test_ambiguous_call_is_not_replayed_after_a_concurrent_identity_change( mcp_http_client_factory: MCPHTTPClientFactory, ) -> None: from anyio import ClosedResourceError @@ -597,9 +591,8 @@ async def test_replay_eligibility_follows_the_dispatched_identity_during_concurr from agent_framework._mcp import _ClientMCPConnection create_client, _, writes = mcp_http_client_factory - client = create_client(replayable_principals={"token-a"}) + client = create_client() tool = _tool(client, "token-a") - tool.trust_server_idempotency_hints = True b_started = asyncio.Event() release_b = asyncio.Event() a_finished = asyncio.Event() @@ -924,13 +917,11 @@ async def fail_replacement_prompt_page(sdk_client: Client, *, cursor: str | None failed_call = AsyncMock(side_effect=ClosedResourceError) with ( patch.object(connection.client, "call_tool", new=failed_call), - patch.object(tool, "_is_call_replayable", return_value=True), patch.object(Client, "list_prompts", new=fail_replacement_prompt_page), + pytest.raises(ToolExecutionException, match="outcome is unknown"), ): - result = await tool.call_tool("session_1_tool_head") - assert isinstance(result, list) - assert len(result) == 1 and result[0].text == "ok" - failed_call.assert_awaited_once() + await tool.call_tool("session_1_tool_head") + failed_call.assert_awaited_once() assert server.generation == 3 assert {function.name for function in tool.functions} == { "session_3_tool_head", @@ -1052,21 +1043,21 @@ async def fail_call(*args: Any, **kwargs: Any) -> None: async with tool: original_connection = tool._connection - with ( - patch.object(_connected_client(tool), "call_tool", side_effect=fail_call), - patch.object(tool, "_is_call_replayable", return_value=True), - ): + with patch.object(_connected_client(tool), "call_tool", side_effect=fail_call): first = asyncio.create_task(tool.call_tool("record", marker="first")) second = asyncio.create_task(tool.call_tool("record", marker="second")) + outcomes: list[Any] = [] try: await asyncio.wait_for(arrivals.get(), timeout=5) await asyncio.wait_for(arrivals.get(), timeout=5) finally: fail.set() - await asyncio.wait_for(asyncio.gather(first, second), timeout=5) + outcomes = await asyncio.wait_for(asyncio.gather(first, second, return_exceptions=True), timeout=5) + assert all(isinstance(outcome, ToolExecutionException) for outcome in outcomes) + assert all("outcome is unknown" in str(outcome) for outcome in outcomes) assert tool._connection is not original_connection assert len(_requests_for_method(requests, "initialize")) == 2 - assert sorted(writes["token-a"]) == ["first", "second"] + assert writes["token-a"] == [] async def test_liveness_failure_during_recovery_discovery_recovers_the_replacement_connection( @@ -1090,13 +1081,17 @@ async def fail_replacement_once() -> None: with ( patch.object(_connected_client(tool), "call_tool", side_effect=ClosedResourceError()), - patch.object(tool, "_is_call_replayable", return_value=True), patch.object(tool, "_check_connection", side_effect=fail_replacement_once), + pytest.raises(ToolExecutionException, match="outcome is unknown"), ): await tool.call_tool("record", marker="recovered") assert replacement_failed assert len(_requests_for_method(requests, "initialize")) == 3 - assert writes["token-a"] == ["recovered"] + assert writes["token-a"] == [] + + # The recovered replacement connection serves a later, distinct call. + assert await tool.call_tool("record", marker="later") is not None + assert writes["token-a"] == ["later"] async def test_cancelling_recovery_propagates_and_releases_the_exclusive_barrier( diff --git a/python/packages/core/tests/core/test_mcp_observability.py b/python/packages/core/tests/core/test_mcp_observability.py index efe1e0fb423..c58f82b98a6 100644 --- a/python/packages/core/tests/core/test_mcp_observability.py +++ b/python/packages/core/tests/core/test_mcp_observability.py @@ -319,16 +319,14 @@ async def test_mcp_tools_call_mcp_error_sets_error_type( @pytest.mark.parametrize("enable_sensitive_data", [True, False], indirect=True) -async def test_mcp_tools_call_second_connection_loss_log_respects_sensitive_data( +async def test_mcp_tools_call_unknown_outcome_log_respects_sensitive_data( span_exporter: InMemorySpanExporter, enable_sensitive_data: bool, caplog: pytest.LogCaptureFixture, ) -> None: - """Second connection loss should retain an error log without exporting exception details by default.""" + """An ambiguous connection loss keeps a redacted warning and the original span error type.""" private_error_detail = "founder-secret-token" tool = _make_connected_mcp_tool() - tool.trust_server_idempotency_hints = True - tool._tool_annotations_by_name["get-weather"] = types.ToolAnnotations(idempotent_hint=True) tool.session.call_tool = AsyncMock( # type: ignore[method-assign, union-attr] # ty: ignore[invalid-assignment] side_effect=[ MCPError(-32000, f"session terminated: {private_error_detail}"), @@ -340,16 +338,23 @@ async def test_mcp_tools_call_second_connection_loss_log_respects_sensitive_data span_exporter.clear() with ( - caplog.at_level(logging.ERROR, logger="agent_framework._mcp"), - pytest.raises(ToolExecutionException, match="connection lost"), + caplog.at_level(logging.WARNING, logger="agent_framework._mcp"), + pytest.raises(ToolExecutionException, match="outcome is unknown"), ): await tool.call_tool("get-weather") assert tool.session is not None - assert getattr(tool.session.call_tool, "await_count", 0) == 2 + assert getattr(tool.session.call_tool, "await_count", 0) == 1 assert (private_error_detail in caplog.text) is enable_sensitive_data if not enable_sensitive_data: - assert "MCP tool call connection closed after reconnection (MCPError)." in caplog.text + assert ( + "MCP tool call outcome is unknown after connection loss; the call was not retried (MCPError)." + in caplog.text + ) + call_spans = [span for span in span_exporter.get_finished_spans() if "tools/call" in span.name] + assert len(call_spans) == 1 + assert call_spans[0].attributes.get(OtelAttr.ERROR_TYPE) == "MCPError" # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + _assert_error_detail_policy(call_spans[0], private_error_detail, enable_sensitive_data) @pytest.mark.parametrize("enable_sensitive_data", [True, False], indirect=True) diff --git a/python/scripts/local_mcp_streamable_http_server.py b/python/scripts/local_mcp_streamable_http_server.py index ecda5e9ec16..ea879cac9c3 100644 --- a/python/scripts/local_mcp_streamable_http_server.py +++ b/python/scripts/local_mcp_streamable_http_server.py @@ -8,7 +8,6 @@ import asyncio from collections.abc import Sequence -from mcp import types from mcp.server.mcpserver import MCPServer from starlette.requests import Request from starlette.responses import JSONResponse, Response @@ -42,16 +41,17 @@ def create_server(*, mount_path: str) -> MCPServer: async def healthz(_request: Request) -> Response: """Return a simple readiness response for CI health checks.""" await asyncio.sleep(0) - return JSONResponse({ - "status": "ok", - "name": SERVER_NAME, - "mcp_path": mount_path, - }) + return JSONResponse( + { + "status": "ok", + "name": SERVER_NAME, + "mcp_path": mount_path, + } + ) @server.tool( name="search_agent_framework_docs", description="Return deterministic Agent Framework documentation text for MCP integration tests.", - annotations=types.ToolAnnotations(read_only_hint=True), ) def search_agent_framework_docs(query: str) -> str: """Return a deterministic response for the MCP integration tests."""