diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index e6fa66d8221..3902a86caa0 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -324,6 +324,7 @@ The vector store API is experimental under the shared `VECTOR_STORES` feature ID and reconnect or effective header-identity changes replace a framework-owned Client's per-Client cache. Agent Framework does not expose shared-store or cache-mode configuration. - **MCP Tasks extension** - The current Agent Framework MCP client does not implement the `io.modelcontextprotocol/tasks` runtime. Tools that advertise `execution.taskSupport == "required"` are skipped with a warning; optional and task-free tools continue through ordinary `tools/call`. Tasks client support is tracked in microsoft/agent-framework#9215. +- **Plain `tools/call` never replays after connection loss** - On `ClosedResourceError` / session-terminated `MCPError` the request may already have executed server-side, so `_call_tool_with_retries` dispatches once. It recovers the connection through the existing `_recover_connection_after_loss` helpers, then raises `ToolExecutionException` stating the outcome is unknown and the call was not retried, with the connection error as the inner exception, a redacted WARNING log, and the connection error type on the span. Server `ToolAnnotations` are advisory and never authorize a replay, so there is no trust or idempotency setting. A later, distinct call proceeds on the replacement connection. Caller-owned connections keep failing with their existing 'cannot be recreated' error before any of this. Read-only discovery (`tools/list`) and `get_prompt` keep their reconnect-and-retry behavior. ### File Access Harness (`_harness/_file_access.py`) diff --git a/python/packages/core/README.md b/python/packages/core/README.md index 6095f24a7ed..604b72dfc9d 100644 --- a/python/packages/core/README.md +++ b/python/packages/core/README.md @@ -179,6 +179,27 @@ asyncio.run(main()) You can explore additional agent samples [here](https://github.com/microsoft/agent-framework/tree/main/python/samples/02-agents). +### MCP tool calls after a lost response + +**Breaking change:** a `tools/call` whose connection drops is no longer sent a second +time. The server may have executed the request before the response was lost, so +re-issuing it could run a non-idempotent operation twice. `call_tool` and tool +invocations now raise `ToolExecutionException` stating that the outcome is unknown +and the call was not retried, with the connection error as `__cause__`. Server +`idempotentHint` and `readOnlyHint` annotations are advisory and do not enable a +retry. Discovery (`tools/list`) and `get_prompt` keep their existing retry behavior. + +For framework-owned connections the connection is still recovered before the +exception surfaces, so a later, different call proceeds normally. Caller-supplied +`Client` and `ClientSession` objects are never recreated and keep their existing +"caller-owned and cannot be recreated" error. + +Catch `ToolExecutionException` around calls that must not run twice and reconcile the +outcome in your own code, for example by reading back the record the tool writes, or +by using a server that enforces an idempotency key. Then retry explicitly if it is +safe. This prevents the framework from duplicating an operation; it is not an +exactly-once guarantee. + ### MCP HTTP client ownership and cookies **Breaking change:** HTTP clients created by `MCPStreamableHTTPTool` no longer persist diff --git a/python/packages/core/agent_framework/_mcp.py b/python/packages/core/agent_framework/_mcp.py index 67046efc169..0951d3cb1bf 100644 --- a/python/packages/core/agent_framework/_mcp.py +++ b/python/packages/core/agent_framework/_mcp.py @@ -3757,7 +3757,9 @@ async def call_tool(self, tool_name: str, **kwargs: Any) -> str | list[Content]: Raises: ToolExecutionException: If the MCP server is not connected, tools are not loaded, - or the tool call fails. + or the tool call fails. A dropped connection raises with the remote execution + outcome unknown and the connection error as the cause: the call is not retried, + because it may already have executed on the server. """ return await self._call_tool(tool_name, kwargs) @@ -3811,78 +3813,76 @@ async def _call_tool_with_retries( parser: Callable[..., str | list[Content]], span: otel_trace.Span, ) -> str | list[Content]: - """Execute the MCP tools/call RPC with retry logic.""" + """Execute the MCP tools/call RPC, recovering the connection without replaying the call.""" from anyio import ClosedResourceError from mcp import MCPError - for attempt in range(2): - failed_connection = self._connection - close_generation = self._close_generation - span_error_recorded = False - try: - result = await self._invoke_tool(tool_name, kwargs, trace_context) - _capture_mcp_result(self, result) - if result.is_error: - parsed = parser(result) - text = ( - "\n".join(c.text for c in parsed if c.type == "text" and c.text) - if isinstance(parsed, list) - else str(parsed) - ) - # Per OTel MCP semconv: set error.type="tool_error" for isError results - if span.is_recording(): - set_mcp_span_error(span, "tool_error", text or str(parsed)) - span_error_recorded = True - raise _mcp_tool_execution_exception(text or str(parsed)) - return parser(result) - except ToolExecutionException as tool_ex: - if span.is_recording() and not span_error_recorded: - set_mcp_span_error(span, type(tool_ex).__name__, str(tool_ex)) - raise - except (ClosedResourceError, MCPError) as call_ex: - if not self._is_connection_loss(call_ex): - error_message = call_ex.error.message if isinstance(call_ex, MCPError) else str(call_ex) - if span.is_recording(): - set_mcp_span_error(span, type(call_ex).__name__, error_message) - raise _mcp_tool_execution_exception(error_message, inner_exception=call_ex) from call_ex - - if attempt == 0: - try: - await self._recover_connection_after_loss( - operation="tool call", - inner_exception=call_ex, - load_configured=True, - failed_connection=failed_connection, - close_generation=close_generation, - ) - except ToolExecutionException as recovery_error: - if span.is_recording(): - set_mcp_span_error(span, type(recovery_error).__name__, str(recovery_error)) - raise - continue - - # Second attempt also failed, give up. - _log_mcp_exception( - logging.ERROR, - call_ex, - "MCP connection closed unexpectedly after reconnection: %s", - call_ex, - redacted_message="MCP tool call connection closed after reconnection", + failed_connection = self._connection + close_generation = self._close_generation + span_error_recorded = False + try: + result = await self._invoke_tool(tool_name, kwargs, trace_context) + _capture_mcp_result(self, result) + if result.is_error: + parsed = parser(result) + text = ( + "\n".join(c.text for c in parsed if c.type == "text" and c.text) + if isinstance(parsed, list) + else str(parsed) ) + # Per OTel MCP semconv: set error.type="tool_error" for isError results if span.is_recording(): - set_mcp_span_error(span, type(call_ex).__name__, str(call_ex)) - raise _mcp_tool_execution_exception( - f"Failed to call tool '{tool_name}' - connection lost.", + set_mcp_span_error(span, "tool_error", text or str(parsed)) + span_error_recorded = True + raise _mcp_tool_execution_exception(text or str(parsed)) + return parser(result) + except ToolExecutionException as tool_ex: + if span.is_recording() and not span_error_recorded: + set_mcp_span_error(span, type(tool_ex).__name__, str(tool_ex)) + raise + except (ClosedResourceError, MCPError) as call_ex: + if not self._is_connection_loss(call_ex): + error_message = call_ex.error.message if isinstance(call_ex, MCPError) else str(call_ex) + if span.is_recording(): + set_mcp_span_error(span, type(call_ex).__name__, error_message) + raise _mcp_tool_execution_exception(error_message, inner_exception=call_ex) from call_ex + + # The server may have executed the request before the response was lost, so the + # outcome is unknown and re-issuing it could duplicate a non-idempotent effect. + # Restore availability for the next call and leave the retry decision to the caller. + try: + await self._recover_connection_after_loss( + operation="tool call", inner_exception=call_ex, - ) from call_ex - except Exception as ex: + load_configured=True, + failed_connection=failed_connection, + close_generation=close_generation, + ) + except ToolExecutionException as recovery_error: if span.is_recording(): - set_mcp_span_error(span, type(ex).__name__, str(ex)) - raise _mcp_tool_execution_exception(f"Failed to call tool '{tool_name}'.", inner_exception=ex) from ex - retries_exhausted = _mcp_tool_execution_exception(f"Failed to call tool '{tool_name}' after retries.") - if span.is_recording(): - set_mcp_span_error(span, type(retries_exhausted).__name__, str(retries_exhausted)) - raise retries_exhausted + set_mcp_span_error(span, type(recovery_error).__name__, str(recovery_error)) + raise + if span.is_recording(): + set_mcp_span_error(span, type(call_ex).__name__, str(call_ex)) + _log_mcp_exception( + logging.WARNING, + call_ex, + "MCP tool call %r lost its response and was not retried: %s", + tool_name, + call_ex, + redacted_message="MCP tool call outcome is unknown after connection loss; the call was not retried", + ) + raise _mcp_tool_execution_exception( + f"Failed to call tool '{tool_name}' - connection lost and the remote execution " + "outcome is unknown. The call was not retried because it may already have " + "executed on the server. The connection was re-established, so a later call " + "can proceed; reconcile the outcome before retrying this one.", + inner_exception=call_ex, + ) from call_ex + except Exception as ex: + if span.is_recording(): + set_mcp_span_error(span, type(ex).__name__, str(ex)) + raise _mcp_tool_execution_exception(f"Failed to call tool '{tool_name}'.", inner_exception=ex) from ex def _resolved_extra_args(self, tool_name: str) -> set[str]: """Return the user-configured extra argument names allowed for a tool.""" diff --git a/python/packages/core/tests/core/test_mcp.py b/python/packages/core/tests/core/test_mcp.py index ca2a47f333f..dd88c1511ed 100644 --- a/python/packages/core/tests/core/test_mcp.py +++ b/python/packages/core/tests/core/test_mcp.py @@ -16,7 +16,7 @@ from textwrap import dedent from types import SimpleNamespace, TracebackType from typing import Any, cast -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest from httpx2 import AsyncClient, MockTransport, Request, Response @@ -4507,8 +4507,8 @@ def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: await func.invoke(param="test_value") -async def test_mcp_tool_reconnects_after_session_terminated_error(): - """Session termination errors should reconnect once and retry the tool call.""" +async def test_mcp_tool_recovers_connection_after_session_terminated_error_without_replay(): + """Session termination reconnects once, but the ambiguous call is not re-issued.""" class TestServer(MCPTool): def __init__(self, **kwargs: Any) -> None: @@ -4536,7 +4536,16 @@ def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: server = TestServer(name="test_server") await server._reconnect_without_loading() - result = await server.call_tool("test_tool", param="test_value") + with pytest.raises(ToolExecutionException, match="outcome is unknown") as exc_info: + await server.call_tool("test_tool", param="test_value") + + assert isinstance(exc_info.value.__cause__, MCPError) + assert server.connect_count == 2 + assert server.sessions[0].call_tool.await_count == 1 + assert server.sessions[1].call_tool.await_count == 0 + + # Availability is restored for the next operation, which is a distinct call. + result = await server.call_tool("other_tool", param="test_value") assert _mcp_result_to_text(result) == "recovered" assert server.connect_count == 2 @@ -4544,6 +4553,150 @@ def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: assert server.sessions[1].call_tool.await_count == 1 +@pytest.mark.parametrize( + "annotations", + [ + None, + types.ToolAnnotations(idempotent_hint=True), + types.ToolAnnotations(read_only_hint=True), + types.ToolAnnotations(idempotent_hint=True, read_only_hint=True), + ], + ids=["unhinted", "idempotent_hint", "read_only_hint", "both_hints"], +) +async def test_mcp_tool_call_is_not_replayed_regardless_of_server_annotations( + annotations: types.ToolAnnotations | None, +): + """Server annotations are advisory and never authorize replay of an ambiguous call. + + The connection can drop after the server executed the request, so re-issuing the same + tools/call could duplicate a non-idempotent side effect. A server can also advertise a + hint incorrectly, so the failure surfaces without replay for every annotation posture. + """ + + class TestServer(MCPTool): + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.connect_count = 0 + self.call_tool_mock = AsyncMock(side_effect=MCPError(-32000, "Session terminated")) + + async def _reconnect_without_loading(self) -> None: + self.connect_count += 1 + session = Mock(spec=ClientSession) + session.call_tool = self.call_tool_mock + _set_framework_owned_session(self, session) + self.is_connected = True + self._tools_loaded = self._prompts_loaded = True + + def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: + return None # type: ignore[return-value] # pyrefly: ignore[bad-return] # ty: ignore[invalid-return-type] + + server = TestServer(name="test_server") + await server._reconnect_without_loading() + server._tool_annotations_by_name["test_tool"] = annotations + + with pytest.raises(ToolExecutionException, match="outcome is unknown") as exc_info: + await server.call_tool("test_tool", param="test_value") + + assert isinstance(exc_info.value.__cause__, MCPError) + assert server.connect_count == 2 + assert server.call_tool_mock.await_count == 1 + + +async def test_mcp_regular_tool_call_does_not_duplicate_side_effect_after_disconnect(): + """A non-idempotent tool whose response is lost mid-call executes at most once. + + The replacement connection would answer successfully, so a replay would hide the + duplicate. The caller gets the unknown-outcome failure and the next call still works. + """ + from anyio.streams.memory import ClosedResourceError + + class TestServer(MCPTool): + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.state = 0 + self.dispatched: list[str] = [] + + async def _reconnect_without_loading(self) -> None: + session = Mock(spec=ClientSession) + + async def call_tool(name: str, *args: Any, **kwargs: Any) -> types.CallToolResult: + self.dispatched.append(name) + self.state += 1 + if self.state == 1: + raise ClosedResourceError + return types.CallToolResult(content=[types.TextContent(type="text", text="ok")]) + + session.call_tool = call_tool + _set_framework_owned_session(self, session) + self.is_connected = True + self._tools_loaded = self._prompts_loaded = True + + def get_mcp_client(self) -> _AsyncGeneratorContextManager[Any, None]: + return None # type: ignore[return-value] # pyrefly: ignore[bad-return] # ty: ignore[invalid-return-type] + + server = TestServer(name="test_server") + await server._reconnect_without_loading() + + with pytest.raises(ToolExecutionException, match="outcome is unknown") as exc_info: + await server.call_tool("test_tool", param="test_value") + + assert isinstance(exc_info.value.__cause__, ClosedResourceError) + assert server.state == 1 + assert server.dispatched == ["test_tool"] + + assert _mcp_result_to_text(await server.call_tool("other_tool", param="test_value")) == "ok" + assert server.state == 2 + assert server.dispatched == ["test_tool", "other_tool"] + + +async def test_connect_discovery_failure_rolls_back_annotations_snapshot(): + """When load_tools succeeds but a later discovery step fails, the rollback in + _connect_on_owner must restore the pre-discovery annotations map along with the + other metadata, or security labels are derived from the failed session's hints.""" + tool = MCPStdioTool(name="test", command="test-command") + sentinel_annotations = {"prev_tool": types.ToolAnnotations(read_only_hint=True)} + sentinel_meta = {"prev_tool": {"version": 0}} + sentinel_params = {"prev_tool": {"q"}} + tool._tool_annotations_by_name = dict(sentinel_annotations) + tool._tool_call_meta_by_name = dict(sentinel_meta) + tool._tool_param_names_by_name = dict(sentinel_params) + retained = FunctionTool(name="retained", func=lambda: "ok") + tool._functions.append(retained) + + mock_transport = (MagicMock(), MagicMock()) + mock_context_manager = Mock() + mock_context_manager.__aenter__ = AsyncMock(return_value=mock_transport) + mock_context_manager.__aexit__ = AsyncMock(return_value=None) + tool.get_mcp_client = Mock(return_value=mock_context_manager) # type: ignore[method-assign] + + sdk_client = _mock_sdk_client( + capabilities=types.ServerCapabilities( + tools=types.ToolsCapability(list_changed=False), + prompts=types.PromptsCapability(list_changed=False), + ) + ) + sdk_client.list_tools = AsyncMock( + return_value=types.ListToolsResult( + tools=[ + types.Tool( + name="fresh_tool", + input_schema={"type": "object", "properties": {}}, + annotations=types.ToolAnnotations(idempotent_hint=True), + ) + ] + ) + ) + sdk_client.list_prompts = AsyncMock(side_effect=RuntimeError("prompt boom")) + + with patch("mcp.Client", return_value=sdk_client), pytest.raises(RuntimeError, match="prompt boom"): + await tool.connect() + + assert tool._tool_annotations_by_name == sentinel_annotations + assert tool._tool_call_meta_by_name == sentinel_meta + assert tool._tool_param_names_by_name == sentinel_params + assert tool._functions == [retained] + + async def test_mcp_tool_call_tool_raises_on_is_error(): """Test that call_tool raises ToolExecutionException when MCP returns is_error=True.""" @@ -5839,8 +5992,12 @@ async def test_streamable_http_integration(): tool = MCPStreamableHTTPTool(name="integration_test", url=url, approval_mode="never_require") async with tool: - # Test that we can connect and load tools - assert tool.session is not None + # URL construction selects a framework-owned high-level Client. The + # deprecated low-level session property is deliberately empty. + assert tool._connection is not None + assert tool._connection.is_framework_owned + assert tool._connection.client is not None + assert tool._connection.session is None assert isinstance(tool.functions, list) # If there are functions available, try to get information about one @@ -5869,11 +6026,21 @@ async def test_mcp_connection_reset_integration(): """ url = os.environ.get("LOCAL_MCP_URL") - tool = MCPStreamableHTTPTool(name="integration_test", url=url, approval_mode="never_require") # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + tool = MCPStreamableHTTPTool( + name="integration_test", + url=url, + approval_mode="never_require", # type: ignore[arg-type] + ) # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] async with tool: - # Verify initial connection - assert tool.session is not None + # Verify the active framework-owned high-level Client. URL-backed tools + # deliberately do not expose a deprecated low-level session here. + original_connection = tool._connection + assert original_connection is not None + assert original_connection.is_framework_owned + original_client = original_connection.client + assert original_client is not None + assert original_connection.session is None assert tool.is_connected is True assert len(tool.functions) > 0, "The MCP server should have at least one function." @@ -5883,10 +6050,9 @@ async def test_mcp_connection_reset_integration(): assert first_result is not None assert len(first_result) > 0 - # Store the original session and exit stack for comparison - original_session = tool.session + # Capture the active Client and exit stack for comparison after reconnect. original_exit_stack = tool._exit_stack - original_call_tool = tool.session.call_tool + original_call_tool = original_client.call_tool # Simulate connection failure by making call_tool raise ClosedResourceError once call_count = 0 @@ -5902,25 +6068,33 @@ async def call_tool_with_error(*args, **kwargs): # After reconnection, delegate to the original method return await original_call_tool(*args, **kwargs) - tool.session.call_tool = call_tool_with_error # type: ignore[method-assign] # ty: ignore[invalid-assignment] - - # Invoke the function again - this should trigger automatic reconnection on ClosedResourceError - second_result = _mcp_result_to_text(await func.invoke(query="What is Agent Framework?")) - assert second_result is not None - assert len(second_result) > 0 - - # Verify we have a new session and exit stack after reconnection - assert tool.session is not None - assert tool.session is not original_session, "Session should be replaced after reconnection" + # Patch the active high-level Client, not the deliberately empty low-level + # session. The lost response is reported without replaying the call. + with ( + patch.object(original_client, "call_tool", new=call_tool_with_error), + pytest.raises(ToolExecutionException, match="outcome is unknown"), + ): + await func.invoke(query="What is Agent Framework?") + assert call_count == 1 + + # Verify reconnection replaced both the framework-owned Client and its exit stack. + new_connection = tool._connection + assert new_connection is not None + assert new_connection.is_framework_owned + assert new_connection.client is not None + assert new_connection.client is not original_client, "Client should be replaced after reconnection" + assert new_connection.session is None assert tool._exit_stack is not original_exit_stack, "Exit stack should be replaced after reconnection" assert tool.is_connected is True # Verify tools are still available after reconnection assert len(tool.functions) > 0 - # Both results should include text (we don't compare content as it may vary) - assert len(first_result) > 0 + # A later, distinct call succeeds on the replacement connection. + second_result = _mcp_result_to_text(await func.invoke(query="What is Agent Framework?")) + assert second_result is not None assert len(second_result) > 0 + assert len(first_result) > 0 async def test_mcp_tool_message_handler_notification(): @@ -9173,6 +9347,8 @@ async def test_mcp_tool_connection_properly_invalidated_after_closed_resource_er This test verifies the fix for issue #2884: the tool tries operations optimistically and only reconnects when ClosedResourceError is encountered, avoiding extra latency. + Recovery restores the connection for the next operation; the call whose response was + lost is reported rather than re-issued (issue #9204). """ from unittest.mock import AsyncMock, MagicMock, patch @@ -9217,7 +9393,7 @@ async def restore_session(): result = await tool.call_tool("test_tool", arg1="value1") assert result is not None - # Test Case 1: Connection closed unexpectedly, should reconnect and retry + # Test Case 1: Connection closed unexpectedly, should reconnect without replay # Simulate ClosedResourceError on first call, then succeed call_count = 0 @@ -9231,13 +9407,18 @@ async def call_tool_with_error(*args, **kwargs): mock_session.call_tool = call_tool_with_error # This call should trigger reconnection after ClosedResourceError - result = await tool.call_tool("test_tool", arg1="value2") - assert result is not None + with pytest.raises(ToolExecutionException, match="outcome is unknown"): + await tool.call_tool("test_tool", arg1="value2") + assert call_count == 1 assert mock_connect.call_count >= 1 mock_connect.assert_awaited_once_with() # Verify _exit_stack.aclose was called during reconnection original_exit_stack.aclose.assert_called() # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + # The replacement connection serves the next call. + assert await tool.call_tool("test_tool", arg1="value2") is not None + assert call_count == 2 + # Test Case 2: Reconnection failure # Reset counters call_count = 0 @@ -9419,25 +9600,25 @@ async def test_mcp_tool_get_prompt_requires_loaded_prompts() -> None: await tool.get_prompt("remote_prompt") -async def test_mcp_tool_call_tool_raises_after_reconnection_still_fails() -> None: +async def test_mcp_tool_call_tool_reports_unknown_outcome_once_after_connection_loss() -> None: from anyio.streams.memory import ClosedResourceError + call_tool = AsyncMock(side_effect=[ClosedResourceError(), ClosedResourceError()]) tool = MCPTool(name="test_tool", load_tools=True) # type: ignore[abstract] # ty: ignore[call-non-callable] - _set_framework_owned_session( - tool, - Mock(call_tool=AsyncMock(side_effect=[ClosedResourceError(), ClosedResourceError()])), - ) + _set_framework_owned_session(tool, Mock(call_tool=call_tool)) tool._tools_loaded = tool._prompts_loaded = True with ( patch.object(tool, "_reconnect_without_loading", AsyncMock()) as mock_connect, - patch.object(logger, "error") as mock_error, - pytest.raises(ToolExecutionException, match="connection lost"), + patch.object(logger, "warning") as mock_warning, + pytest.raises(ToolExecutionException, match="outcome is unknown") as exc_info, ): await tool.call_tool("remote_tool") + assert isinstance(exc_info.value.__cause__, ClosedResourceError) + assert call_tool.await_count == 1 mock_connect.assert_awaited_once_with() - mock_error.assert_called_once() + mock_warning.assert_called_once() async def test_mcp_tool_get_prompt_raises_after_reconnection_still_fails() -> None: diff --git a/python/packages/core/tests/core/test_mcp_http_auth.py b/python/packages/core/tests/core/test_mcp_http_auth.py index f48fc114b2c..e0a42fe3f84 100644 --- a/python/packages/core/tests/core/test_mcp_http_auth.py +++ b/python/packages/core/tests/core/test_mcp_http_auth.py @@ -568,8 +568,9 @@ async def block_recovery() -> None: await asyncio.sleep(0) assert not call.done() release.set() - await call - assert writes["token-a"] == ["retry"] + with pytest.raises(ToolExecutionException, match="outcome is unknown"): + await call + assert writes["token-a"] == [] else: with pytest.raises(ToolExecutionException, match="Timed out recovering") as error: await call @@ -582,6 +583,69 @@ async def block_recovery() -> None: assert tool._header_operations == 0 +async def test_ambiguous_call_is_not_replayed_after_a_concurrent_identity_change( + mcp_http_client_factory: MCPHTTPClientFactory, +) -> None: + from anyio import ClosedResourceError + + from agent_framework._mcp import _ClientMCPConnection + + create_client, _, writes = mcp_http_client_factory + client = create_client() + tool = _tool(client, "token-a") + b_started = asyncio.Event() + release_b = asyncio.Event() + a_finished = asyncio.Event() + original_call = _ClientMCPConnection.call_tool + original_recovery = tool._recover_failed_connection + b_dispatches = 0 + + async def fail_first_b_dispatch( + connection: Any, + name: str, + arguments: dict[str, Any] | None, + *, + meta: Any, + ) -> Any: + nonlocal b_dispatches + if name == "record" and (arguments or {}).get("marker") == "b": + b_dispatches += 1 + if b_dispatches == 1: + b_started.set() + await release_b.wait() + raise ClosedResourceError + return await original_call(connection, name, arguments, meta=meta) + + async def finish_a_before_b_recovery(**kwargs: Any) -> None: + await a_finished.wait() + await original_recovery(**kwargs) + + async with tool, asyncio.timeout(10): + with ( + patch.object(_ClientMCPConnection, "call_tool", new=fail_first_b_dispatch), + patch.object(tool, "_recover_failed_connection", side_effect=finish_a_before_b_recovery), + ): + b_call = asyncio.create_task(tool.call_tool("record", credential="token-b", marker="b")) + b_outcome: Any = None + try: + await b_started.wait() + a_call = asyncio.create_task(tool.call_tool("record", credential="token-a", marker="a")) + release_b.set() + await a_call + a_finished.set() + (b_outcome,) = await asyncio.gather(b_call, return_exceptions=True) + finally: + release_b.set() + a_finished.set() + await asyncio.gather(b_call, return_exceptions=True) + + assert isinstance(b_outcome, ToolExecutionException) + assert "outcome is unknown" in str(b_outcome) + assert b_dispatches == 1 + assert writes["token-a"] == ["a"] + assert writes["token-b"] == [] + + async def test_recovery_budget_includes_waiting_for_other_active_attempts( mcp_http_client_factory: MCPHTTPClientFactory, ) -> None: @@ -854,11 +918,10 @@ async def fail_replacement_prompt_page(sdk_client: Client, *, cursor: str | None with ( patch.object(connection.client, "call_tool", new=failed_call), patch.object(Client, "list_prompts", new=fail_replacement_prompt_page), + pytest.raises(ToolExecutionException, match="outcome is unknown"), ): - result = await tool.call_tool("session_1_tool_head") - assert isinstance(result, list) - assert len(result) == 1 and result[0].text == "ok" - failed_call.assert_awaited_once() + await tool.call_tool("session_1_tool_head") + failed_call.assert_awaited_once() assert server.generation == 3 assert {function.name for function in tool.functions} == { "session_3_tool_head", @@ -983,15 +1046,18 @@ async def fail_call(*args: Any, **kwargs: Any) -> None: with patch.object(_connected_client(tool), "call_tool", side_effect=fail_call): first = asyncio.create_task(tool.call_tool("record", marker="first")) second = asyncio.create_task(tool.call_tool("record", marker="second")) + outcomes: list[Any] = [] try: await asyncio.wait_for(arrivals.get(), timeout=5) await asyncio.wait_for(arrivals.get(), timeout=5) finally: fail.set() - await asyncio.wait_for(asyncio.gather(first, second), timeout=5) + outcomes = await asyncio.wait_for(asyncio.gather(first, second, return_exceptions=True), timeout=5) + assert all(isinstance(outcome, ToolExecutionException) for outcome in outcomes) + assert all("outcome is unknown" in str(outcome) for outcome in outcomes) assert tool._connection is not original_connection assert len(_requests_for_method(requests, "initialize")) == 2 - assert sorted(writes["token-a"]) == ["first", "second"] + assert writes["token-a"] == [] async def test_liveness_failure_during_recovery_discovery_recovers_the_replacement_connection( @@ -1016,11 +1082,16 @@ async def fail_replacement_once() -> None: with ( patch.object(_connected_client(tool), "call_tool", side_effect=ClosedResourceError()), patch.object(tool, "_check_connection", side_effect=fail_replacement_once), + pytest.raises(ToolExecutionException, match="outcome is unknown"), ): await tool.call_tool("record", marker="recovered") assert replacement_failed assert len(_requests_for_method(requests, "initialize")) == 3 - assert writes["token-a"] == ["recovered"] + assert writes["token-a"] == [] + + # The recovered replacement connection serves a later, distinct call. + assert await tool.call_tool("record", marker="later") is not None + assert writes["token-a"] == ["later"] async def test_cancelling_recovery_propagates_and_releases_the_exclusive_barrier( diff --git a/python/packages/core/tests/core/test_mcp_observability.py b/python/packages/core/tests/core/test_mcp_observability.py index 449a09873f6..c58f82b98a6 100644 --- a/python/packages/core/tests/core/test_mcp_observability.py +++ b/python/packages/core/tests/core/test_mcp_observability.py @@ -319,12 +319,12 @@ async def test_mcp_tools_call_mcp_error_sets_error_type( @pytest.mark.parametrize("enable_sensitive_data", [True, False], indirect=True) -async def test_mcp_tools_call_second_connection_loss_log_respects_sensitive_data( +async def test_mcp_tools_call_unknown_outcome_log_respects_sensitive_data( span_exporter: InMemorySpanExporter, enable_sensitive_data: bool, caplog: pytest.LogCaptureFixture, ) -> None: - """Second connection loss should retain an error log without exporting exception details by default.""" + """An ambiguous connection loss keeps a redacted warning and the original span error type.""" private_error_detail = "founder-secret-token" tool = _make_connected_mcp_tool() tool.session.call_tool = AsyncMock( # type: ignore[method-assign, union-attr] # ty: ignore[invalid-assignment] @@ -338,14 +338,23 @@ async def test_mcp_tools_call_second_connection_loss_log_respects_sensitive_data span_exporter.clear() with ( - caplog.at_level(logging.ERROR, logger="agent_framework._mcp"), - pytest.raises(ToolExecutionException, match="connection lost"), + caplog.at_level(logging.WARNING, logger="agent_framework._mcp"), + pytest.raises(ToolExecutionException, match="outcome is unknown"), ): await tool.call_tool("get-weather") + assert tool.session is not None + assert getattr(tool.session.call_tool, "await_count", 0) == 1 assert (private_error_detail in caplog.text) is enable_sensitive_data if not enable_sensitive_data: - assert "MCP tool call connection closed after reconnection (MCPError)." in caplog.text + assert ( + "MCP tool call outcome is unknown after connection loss; the call was not retried (MCPError)." + in caplog.text + ) + call_spans = [span for span in span_exporter.get_finished_spans() if "tools/call" in span.name] + assert len(call_spans) == 1 + assert call_spans[0].attributes.get(OtelAttr.ERROR_TYPE) == "MCPError" # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + _assert_error_detail_policy(call_spans[0], private_error_detail, enable_sensitive_data) @pytest.mark.parametrize("enable_sensitive_data", [True, False], indirect=True) diff --git a/python/scripts/local_mcp_streamable_http_server.py b/python/scripts/local_mcp_streamable_http_server.py index 772b7e4161e..ea879cac9c3 100644 --- a/python/scripts/local_mcp_streamable_http_server.py +++ b/python/scripts/local_mcp_streamable_http_server.py @@ -8,7 +8,7 @@ import asyncio from collections.abc import Sequence -from mcp.server.fastmcp import FastMCP +from mcp.server.mcpserver import MCPServer from starlette.requests import Request from starlette.responses import JSONResponse, Response @@ -29,14 +29,11 @@ def _normalize_mount_path(path: str) -> str: return normalized.rstrip("/") or "/" -def create_server(*, host: str, port: int, mount_path: str) -> FastMCP: +def create_server(*, mount_path: str) -> MCPServer: """Create the local MCP integration test server.""" - server = FastMCP( + server = MCPServer( name=SERVER_NAME, instructions="Deterministic local MCP server used by Agent Framework integration tests.", - host=host, - port=port, - streamable_http_path=mount_path, log_level="INFO", ) @@ -83,12 +80,14 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def main(argv: Sequence[str] | None = None) -> None: """Start the local MCP streamable HTTP server.""" args = parse_args(argv) - server = create_server( + mount_path = _normalize_mount_path(args.mount_path) + server = create_server(mount_path=mount_path) + server.run( + transport="streamable-http", host=args.host, port=args.port, - mount_path=_normalize_mount_path(args.mount_path), + streamable_http_path=mount_path, ) - server.run(transport="streamable-http") if __name__ == "__main__":