From 065ceb16892649b89e3cebaba01d4d93a1f555c0 Mon Sep 17 00:00:00 2001 From: karthik-0306 Date: Sat, 14 Nov 2026 22:32:40 +0530 Subject: [PATCH 1/3] Python: Add FailoverChatClient for per-model-call failover across chat clients --- python/packages/core/AGENTS.md | 12 + .../packages/core/agent_framework/__init__.py | 2 + .../core/agent_framework/__init__.pyi | 2 + .../core/agent_framework/_failover.py | 333 ++++++++++ .../core/agent_framework/_feature_stage.py | 1 + .../packages/core/tests/core/test_failover.py | 569 ++++++++++++++++++ .../samples/02-agents/chat_client/README.md | 3 +- .../chat_client/chat_client_failover.py | 90 +++ 8 files changed, 1011 insertions(+), 1 deletion(-) create mode 100644 python/packages/core/agent_framework/_failover.py create mode 100644 python/packages/core/tests/core/test_failover.py create mode 100644 python/samples/02-agents/chat_client/chat_client_failover.py diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index e6fa66d8221..029d0c1bd27 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -71,6 +71,18 @@ agent_framework/ for streaming and non-streaming requests. Provides `__aenter__()` returning `self`, `__aexit__()` delegating to `close()`, and a default no-op async `close()`. Connectors override cleanup for owned connections and credentials; injected resources remain caller-owned. `SupportsChatGetResponse` does not require lifecycle methods. +- **`FailoverChatClient`** (`_failover.py`, experimental under `CHAT_CLIENT_FAILOVER`) - Tries an ordered list of + chat clients for each model call. It owns the single function-invocation loop and calls each candidate below that + candidate's `FunctionInvocationLayer` (the same entry the candidate's own loop uses per model call), so candidate + chat middleware and telemetry run per attempt while tools run once, in the failover client's loop. A failure after + earlier model calls requested tools therefore never re-executes them. Every model call starts again with the first + client. Any `Exception` fails over unless `should_failover` returns `False`; cancellation is never caught, and + `ResponseInvalidatedException` and `MiddlewareException` always propagate. Streaming fails over only before an + update with contents is released; content-less updates are held until then and discarded with a failed attempt. + History must be client-managed: `store=True` or a service continuation option (the union of candidates' + `SERVICE_CONTINUATION_OPTION_NAMES`) raises `ChatClientInvalidRequestException` before any call, and candidates + with `STORES_BY_DEFAULT` receive `store=False` per attempt when `store` is unset. Candidates are caller-owned and + not closed. ### Types (`_types.py`) diff --git a/python/packages/core/agent_framework/__init__.py b/python/packages/core/agent_framework/__init__.py index e695e8d1e69..ffbba04421c 100644 --- a/python/packages/core/agent_framework/__init__.py +++ b/python/packages/core/agent_framework/__init__.py @@ -113,6 +113,7 @@ "tool_called_check", "tool_calls_present", ), + "._failover": ("FailoverChatClient",), "._feature_stage": ("ExperimentalFeature", "ReleaseCandidateFeature"), "._harness._agent": ("DEFAULT_HARNESS_INSTRUCTIONS", "create_harness_agent"), "._harness._background_agents": ( @@ -514,6 +515,7 @@ "Executor", "ExpectedToolCall", "ExperimentalFeature", + "FailoverChatClient", "FanInEdgeGroup", "FanOutEdgeGroup", "FieldTypes", diff --git a/python/packages/core/agent_framework/__init__.pyi b/python/packages/core/agent_framework/__init__.pyi index a46b7dcdc94..4af8217ac70 100644 --- a/python/packages/core/agent_framework/__init__.pyi +++ b/python/packages/core/agent_framework/__init__.pyi @@ -75,6 +75,7 @@ from ._evaluation import ( tool_called_check, tool_calls_present, ) +from ._failover import FailoverChatClient from ._feature_stage import ExperimentalFeature, ReleaseCandidateFeature from ._harness._agent import DEFAULT_HARNESS_INSTRUCTIONS, create_harness_agent from ._harness._background_agents import ( @@ -469,6 +470,7 @@ __all__ = [ "Executor", "ExpectedToolCall", "ExperimentalFeature", + "FailoverChatClient", "FanInEdgeGroup", "FanOutEdgeGroup", "FieldTypes", diff --git a/python/packages/core/agent_framework/_failover.py b/python/packages/core/agent_framework/_failover.py new file mode 100644 index 00000000000..ab77d1eed4f --- /dev/null +++ b/python/packages/core/agent_framework/_failover.py @@ -0,0 +1,333 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Ordered failover across chat clients for each model call of the function-invocation loop.""" + +from __future__ import annotations + +import logging +import sys +from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, Generic, TypedDict, cast + +from ._clients import BaseChatClient, SupportsChatGetResponse +from ._feature_stage import ExperimentalFeature, experimental +from ._middleware import ChatMiddlewareLayer, MiddlewareException +from ._sessions import is_local_history_conversation_id +from ._tools import FunctionInvocationLayer +from ._types import ChatResponse, ChatResponseUpdate, Message, ResponseStream +from .exceptions import ChatClientInvalidRequestException, ResponseInvalidatedException + +if sys.version_info >= (3, 13): + from typing import TypeVar # pragma: no cover +else: + from typing_extensions import TypeVar # pragma: no cover + +if TYPE_CHECKING: + from ._compaction import CompactionStrategy, TokenizerProtocol + from ._middleware import ChatAndFunctionMiddlewareTypes + from ._tools import FunctionInvocationConfiguration + from ._types import ChatOptions # referenced by the OptionsCoT default + +logger = logging.getLogger("agent_framework") + +OptionsCoT = TypeVar( + "OptionsCoT", + bound=TypedDict, # type: ignore[valid-type] + default="ChatOptions[None]", + covariant=True, +) + +FailoverPredicate = Callable[[Exception], bool] +"""Decide whether a failed chat client attempt may be retried on the next chat client.""" + + +@experimental(feature_id=ExperimentalFeature.CHAT_CLIENT_FAILOVER) +class FailoverChatClient( + FunctionInvocationLayer[OptionsCoT], + ChatMiddlewareLayer[OptionsCoT], + BaseChatClient[OptionsCoT], + Generic[OptionsCoT], +): + """Send each model call to an ordered list of chat clients until one succeeds. + + ``FailoverChatClient`` owns the single function-invocation loop for a request and fails over per model call, + not per run. When a chat client fails after earlier model calls already requested tools, the next chat client + receives the same conversation, including the tool results produced so far, so no tool runs again. Wrapping + complete chat clients in a plain ``try``/``except`` does not have this property: each client runs its own tool + loop, so a failure on a later model call restarts that loop on the next client and re-executes tools. + + Each attempt goes through the selected client's chat middleware and telemetry layers but not its + function-invocation layer. Configure tools, function middleware, and ``function_invocation_configuration`` on + the ``FailoverChatClient`` (or the agent that uses it). Chat middleware configured on the failover client, or + passed by an agent, runs once per model call around all attempts. + + Failover rules: + + - Clients are tried in order for every model call. The first client is tried first again on the next model + call, so a recovered primary resumes serving requests. + - Any ``Exception`` raised by an attempt triggers failover unless ``should_failover`` returns ``False``. + Cancellation is never caught. ``ResponseInvalidatedException`` and middleware control-flow exceptions always + propagate unchanged, because handing an invalidated provider response to another client could run tool calls + the provider already cancelled. + - A streaming attempt fails over only before it releases an update that carries content. Updates without + content (for example, response-created metadata) are held back until content arrives, so an attempt that fails + after only sending metadata is discarded. Once content has been released, a later error propagates. + - When the last client fails, its exception propagates unchanged. + + Conversation history must be client-managed because history stored by one service cannot be read by another. + Requests that set ``store=True`` or carry a service-managed continuation (such as ``conversation_id`` or + ``previous_response_id``) are rejected with ``ChatClientInvalidRequestException`` before any client is called. + Clients that store history by default (``STORES_BY_DEFAULT``) receive ``store=False`` for their attempt unless + the request sets ``store`` explicitly. Agents using this client keep history locally, as they do for any + client that does not store history by default. + + The chat clients are caller-owned: ``close()`` and ``async with`` do not close them. + + Examples: + .. code-block:: python + + from agent_framework import Agent, FailoverChatClient + from agent_framework.anthropic import AnthropicClient + from agent_framework.openai import OpenAIChatClient + + client = FailoverChatClient([ + OpenAIChatClient(model="gpt-5.4-mini"), + AnthropicClient(model="claude-sonnet-5-5"), + ]) + agent = Agent(client=client, instructions="You are a support assistant.", tools=[issue_refund]) + response = await agent.run("Refund order 123.") + """ + + def __init__( + self, + clients: Sequence[SupportsChatGetResponse[Any]], + *, + should_failover: FailoverPredicate | None = None, + middleware: Sequence[ChatAndFunctionMiddlewareTypes] | None = None, + function_invocation_configuration: FunctionInvocationConfiguration | None = None, + compaction_strategy: CompactionStrategy | None = None, + tokenizer: TokenizerProtocol | None = None, + additional_properties: dict[str, Any] | None = None, + ) -> None: + """Initialize a FailoverChatClient. + + Args: + clients: The chat clients to try, in order. The sequence is copied; the clients remain caller-owned. + + Keyword Args: + should_failover: Optional predicate called with the exception from a failed attempt. Return ``True`` to + try the next client or ``False`` to propagate the exception. It is not called for the last client, + for cancellation, for ``ResponseInvalidatedException``, or for middleware control-flow exceptions. + When omitted, every other ``Exception`` triggers failover. + middleware: Chat and function middleware for this client. Chat middleware runs once per model call + around all attempts; function middleware runs around tool executions. + function_invocation_configuration: Configuration for the function-invocation loop owned by this client. + compaction_strategy: Optional compaction strategy applied before each model call, before any attempt. + tokenizer: Optional tokenizer used by token-aware compaction strategies. + additional_properties: Additional properties for the client. + + Raises: + ValueError: If ``clients`` is empty or contains the same client more than once. + TypeError: If an entry does not provide ``get_response``. + """ + candidates = tuple(clients) + if not candidates: + raise ValueError("FailoverChatClient requires at least one chat client.") + if len({id(candidate) for candidate in candidates}) != len(candidates): + raise ValueError("FailoverChatClient requires distinct chat clients; the same client appears twice.") + for candidate in candidates: + if not callable(getattr(candidate, "get_response", None)): + raise TypeError( + f"FailoverChatClient clients must provide get_response(); got {type(candidate).__name__}." + ) + self._clients = candidates + self._should_failover = should_failover + continuation_option_names = {"conversation_id"} + for candidate in candidates: + declared = getattr(candidate, "SERVICE_CONTINUATION_OPTION_NAMES", ()) + if isinstance(declared, Sequence) and not isinstance(declared, (str, bytes, bytearray)): + continuation_option_names.update( + name for name in cast(Sequence[Any], declared) if isinstance(name, str) + ) + self._continuation_option_names = tuple(sorted(continuation_option_names)) + super().__init__( + middleware=middleware, + function_invocation_configuration=function_invocation_configuration, + compaction_strategy=compaction_strategy, + tokenizer=tokenizer, + additional_properties=additional_properties, + ) + + @property + def clients(self) -> tuple[SupportsChatGetResponse[Any], ...]: + """The chat clients tried for each model call, in order.""" + return self._clients + + def _inner_get_response( + self, + *, + messages: Sequence[Message], + stream: bool, + options: Mapping[str, Any], + **kwargs: Any, + ) -> Awaitable[ChatResponse] | ResponseStream[ChatResponseUpdate, ChatResponse]: + if stream: + return self._stream_with_failover(messages=messages, options=options, client_kwargs=kwargs) + return self._get_response_with_failover(messages=messages, options=options, client_kwargs=kwargs) + + async def _get_response_with_failover( + self, + *, + messages: Sequence[Message], + options: Mapping[str, Any], + client_kwargs: Mapping[str, Any], + ) -> ChatResponse: + self._ensure_client_managed_history(options) + last_index = len(self._clients) - 1 + for index, client in enumerate(self._clients): + try: + return await cast( + "Awaitable[ChatResponse]", + self._call_client( + client, messages=messages, stream=False, options=options, client_kwargs=client_kwargs + ), + ) + except Exception as exception: + if index == last_index or not self._is_failover_allowed(exception): + raise + self._log_failover(index, client, exception) + raise AssertionError("unreachable") # pragma: no cover + + def _stream_with_failover( + self, + *, + messages: Sequence[Message], + options: Mapping[str, Any], + client_kwargs: Mapping[str, Any], + ) -> ResponseStream[ChatResponseUpdate, ChatResponse]: + response_format = options.get("response_format") + final_responses: list[ChatResponse] = [] + + async def _updates() -> AsyncIterator[ChatResponseUpdate]: + self._ensure_client_managed_history(options) + last_index = len(self._clients) - 1 + for index, client in enumerate(self._clients): + is_last = index == last_index + try: + attempt = await cast( + "ResponseStream[ChatResponseUpdate, ChatResponse]", + self._call_client( + client, messages=messages, stream=True, options=options, client_kwargs=client_kwargs + ), + ) + except Exception as exception: + if is_last or not self._is_failover_allowed(exception): + raise + self._log_failover(index, client, exception) + continue + + held: list[ChatResponseUpdate] = [] + released = False + failed = False + async with attempt: + iterator = attempt.__aiter__() + while True: + try: + update = await iterator.__anext__() + except StopAsyncIteration: + break + except Exception as exception: + if released or is_last or not self._is_failover_allowed(exception): + raise + self._log_failover(index, client, exception) + failed = True + break + if not released and not update.contents: + held.append(update) + continue + if not released: + released = True + for held_update in held: + yield held_update + held.clear() + yield update + if failed: + continue + # The attempt finalizes when its iteration ends, so a finalization error has already been + # handled above; this returns the finalized response. + final_response = await attempt.get_final_response() + for held_update in held: + yield held_update + final_responses.append(final_response) + return + + def _finalize(updates: Sequence[ChatResponseUpdate]) -> ChatResponse: + if final_responses: + return final_responses[-1] + # Reached only when a direct consumer of this stream stops before an attempt completes. + return ChatResponse.from_updates(updates, output_format_type=response_format) + + return ResponseStream(_updates(), finalizer=_finalize) + + def _call_client( + self, + client: SupportsChatGetResponse[Any], + *, + messages: Sequence[Message], + stream: bool, + options: Mapping[str, Any], + client_kwargs: Mapping[str, Any], + ) -> Awaitable[ChatResponse[Any]] | ResponseStream[ChatResponseUpdate, ChatResponse[Any]]: + """Run one model call on ``client`` without its function-invocation loop.""" + attempt_options = dict(options) + if getattr(client, "STORES_BY_DEFAULT", False) is True and attempt_options.get("store") is None: + attempt_options["store"] = False + target: Any = client + if isinstance(client, FunctionInvocationLayer): + # This is the same call a client's own function-invocation loop makes for each model call: it enters the + # layer below FunctionInvocationLayer, keeping the client's chat middleware and telemetry. + target = super(FunctionInvocationLayer, client) + get_response = cast( + "Callable[..., Awaitable[ChatResponse[Any]] | ResponseStream[ChatResponseUpdate, ChatResponse[Any]]]", + target.get_response, + ) + return get_response( + messages=messages, + stream=stream, + options=attempt_options, + client_kwargs=dict(client_kwargs), + ) + + def _ensure_client_managed_history(self, options: Mapping[str, Any]) -> None: + if options.get("store") is True: + raise ChatClientInvalidRequestException( + "FailoverChatClient requires client-managed history so another chat client can continue the " + "conversation; store=True is not supported. Leave store unset or set store=False." + ) + for name in self._continuation_option_names: + value = options.get(name) + if value is None or value == "": + continue + if name == "conversation_id" and isinstance(value, str) and is_local_history_conversation_id(value): + continue + raise ChatClientInvalidRequestException( + f"FailoverChatClient cannot continue a service-managed conversation ('{name}' is set) because the " + "other chat clients cannot read history stored by that service. Use client-managed history instead." + ) + + def _is_failover_allowed(self, exception: Exception) -> bool: + if isinstance(exception, (ResponseInvalidatedException, MiddlewareException)): + return False + if self._should_failover is None: + return True + return bool(self._should_failover(exception)) + + def _log_failover(self, index: int, client: SupportsChatGetResponse[Any], exception: Exception) -> None: + logger.warning( + "Chat client %d of %d (%s) failed with %s; trying the next chat client.", + index + 1, + len(self._clients), + type(client).__name__, + type(exception).__name__, + exc_info=exception, + ) diff --git a/python/packages/core/agent_framework/_feature_stage.py b/python/packages/core/agent_framework/_feature_stage.py index b4988db3c19..c1be125d2c7 100644 --- a/python/packages/core/agent_framework/_feature_stage.py +++ b/python/packages/core/agent_framework/_feature_stage.py @@ -52,6 +52,7 @@ class ExperimentalFeature(str, Enum): AGENT_TRIGGERED_COMPACTION = "AGENT_TRIGGERED_COMPACTION" AGENT_HOOKS = "AGENT_HOOKS" + CHAT_CLIENT_FAILOVER = "CHAT_CLIENT_FAILOVER" COMPUTER_USE = "COMPUTER_USE" DECLARATIVE_AGENTS = "DECLARATIVE_AGENTS" EVALS = "EVALS" diff --git a/python/packages/core/tests/core/test_failover.py b/python/packages/core/tests/core/test_failover.py new file mode 100644 index 00000000000..97b57526237 --- /dev/null +++ b/python/packages/core/tests/core/test_failover.py @@ -0,0 +1,569 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +from collections.abc import Awaitable, Mapping, Sequence +from typing import Any, ClassVar, cast + +import pytest + +from agent_framework import ( + Agent, + BaseChatClient, + ChatContext, + ChatMiddleware, + ChatMiddlewareLayer, + ChatOptions, + ChatResponse, + ChatResponseUpdate, + Content, + FailoverChatClient, + FunctionInvocationContext, + FunctionInvocationLayer, + FunctionMiddleware, + Message, + ResponseStream, + tool, +) +from agent_framework._middleware import MiddlewareException +from agent_framework.exceptions import ( + ChatClientException, + ChatClientInvalidRequestException, + ResponseInvalidatedException, +) + +pytestmark = pytest.mark.filterwarnings("ignore:.*CHAT_CLIENT_FAILOVER.*:FutureWarning") + +# A step is one scripted model call: a response, an exception, or (streaming) a list of updates and exceptions. +Step = ChatResponse | BaseException | Sequence[ChatResponseUpdate | BaseException] + + +class _RawScriptedClient(BaseChatClient[Any]): + """Chat client without middleware or a tool loop (like ``Raw*`` provider clients) and scripted model calls.""" + + def __init__( + self, + steps: Sequence[Step] = (), + *, + raise_on_call: BaseException | None = None, + finalize_error: Exception | None = None, + ) -> None: + super().__init__() + self._steps = list(steps) + self._raise_on_call = raise_on_call + self._finalize_error = finalize_error + self.calls: list[dict[str, Any]] = [] + self.closed = False + + async def close(self) -> None: + self.closed = True + + def _inner_get_response( + self, *, messages: Sequence[Message], stream: bool, options: Mapping[str, Any], **kwargs: Any + ) -> Awaitable[ChatResponse] | ResponseStream[ChatResponseUpdate, ChatResponse]: + self.calls.append({"messages": list(messages), "options": dict(options)}) + if self._raise_on_call is not None: + raise self._raise_on_call + step: Step = self._steps.pop(0) if self._steps else _text_response("done") + + if stream: + items = _updates_for(step) if isinstance(step, ChatResponse) else step + + async def _updates(): + if isinstance(items, BaseException): + raise items + for item in items: + if isinstance(item, BaseException): + raise item + yield item + + if self._finalize_error is None: + return self._build_response_stream(_updates()) + finalize_error = self._finalize_error + + def _failing_finalizer(_: Sequence[ChatResponseUpdate]) -> ChatResponse: + raise finalize_error + + return ResponseStream(_updates(), finalizer=_failing_finalizer) + + async def _response() -> ChatResponse: + if isinstance(step, BaseException): + raise step + assert isinstance(step, ChatResponse) + return step + + return _response() + + +class _ScriptedClient(FunctionInvocationLayer[Any], ChatMiddlewareLayer[Any], _RawScriptedClient): + """Chat client with the same layering as shipped clients and scripted model calls.""" + + def __init__( + self, + steps: Sequence[Step] = (), + *, + raise_on_call: BaseException | None = None, + finalize_error: Exception | None = None, + middleware: Sequence[Any] | None = None, + ) -> None: + super().__init__(middleware=middleware, steps=steps, raise_on_call=raise_on_call, finalize_error=finalize_error) + + +class _StoringScriptedClient(_ScriptedClient): + STORES_BY_DEFAULT: ClassVar[bool] = True + + +class _ResponsesLikeScriptedClient(_StoringScriptedClient): + SERVICE_CONTINUATION_OPTION_NAMES: ClassVar[tuple[str, ...]] = ("conversation_id", "previous_response_id") + + +def _text_response(text: str) -> ChatResponse: + return ChatResponse(messages=Message(role="assistant", contents=[text])) + + +def _tool_call_response(call_id: str = "call-1") -> ChatResponse: + call = Content.from_function_call(call_id=call_id, name="send_email", arguments={"to": "cfo@contoso.com"}) + return ChatResponse(messages=Message(role="assistant", contents=[call])) + + +def _updates_for(response: ChatResponse) -> list[ChatResponseUpdate]: + return [ChatResponseUpdate(contents=list(message.contents), role="assistant") for message in response.messages] + + +def _rate_limited(name: str) -> ChatClientException: + return ChatClientException(f"{name}: 429 Too Many Requests", log_level=None) + + +def _function_results(messages: Sequence[Message]) -> list[Content]: + return [content for message in messages for content in message.contents if content.type == "function_result"] + + +@pytest.fixture +def sent_emails() -> list[str]: + return [] + + +@pytest.fixture +def send_email(sent_emails: list[str]) -> Any: + @tool(approval_mode="never_require") + def send_email(to: str) -> str: + """Send an email.""" + sent_emails.append(to) + return f"sent to {to}" + + return send_email + + +async def test_failover_after_tool_call_does_not_run_the_tool_again(send_email: Any, sent_emails: list[str]) -> None: + primary = _ScriptedClient([_tool_call_response(), _rate_limited("primary")]) + fallback = _ScriptedClient([_text_response("done via fallback")]) + client = FailoverChatClient([primary, fallback]) + + response = await client.get_response( + [Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]} + ) + + assert response.text == "done via fallback" + assert sent_emails == ["cfo@contoso.com"] + assert len(primary.calls) == 2 + assert len(fallback.calls) == 1 + # The fallback continues the same model call, so it sees the tool result produced before the failure. + assert [result.call_id for result in _function_results(fallback.calls[0]["messages"])] == ["call-1"] + + +async def test_streaming_failover_after_tool_call_does_not_run_the_tool_again( + send_email: Any, sent_emails: list[str] +) -> None: + primary = _ScriptedClient([_updates_for(_tool_call_response()), _rate_limited("primary")]) + fallback = _ScriptedClient([_updates_for(_text_response("done via fallback"))]) + client = FailoverChatClient([primary, fallback]) + + stream = client.get_response( + [Message(role="user", contents=["Email the CFO."])], stream=True, options={"tools": [send_email]} + ) + async for _ in stream: + pass + response = await stream.get_final_response() + + assert response.text == "done via fallback" + assert sent_emails == ["cfo@contoso.com"] + assert len(fallback.calls) == 1 + assert [result.call_id for result in _function_results(fallback.calls[0]["messages"])] == ["call-1"] + + +async def test_each_model_call_starts_with_the_first_client(send_email: Any, sent_emails: list[str]) -> None: + primary = _ScriptedClient([_rate_limited("primary"), _text_response("primary recovered")]) + fallback = _ScriptedClient([_tool_call_response()]) + client = FailoverChatClient([primary, fallback]) + + response = await client.get_response( + [Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]} + ) + + assert response.text == "primary recovered" + assert sent_emails == ["cfo@contoso.com"] + assert len(primary.calls) == 2 + assert len(fallback.calls) == 1 + + +async def test_successful_first_client_does_not_call_the_others() -> None: + primary = _ScriptedClient([_text_response("primary")]) + fallback = _ScriptedClient() + client = FailoverChatClient([primary, fallback]) + + response = await client.get_response([Message(role="user", contents=["hi"])]) + + assert response.text == "primary" + assert fallback.calls == [] + + +async def test_last_client_error_propagates_unchanged() -> None: + last_error = _rate_limited("fallback") + primary = _ScriptedClient([_rate_limited("primary")]) + fallback = _ScriptedClient([last_error]) + client = FailoverChatClient([primary, fallback]) + + with pytest.raises(ChatClientException) as raised: + await client.get_response([Message(role="user", contents=["hi"])]) + + assert raised.value is last_error + assert len(primary.calls) == 1 + assert len(fallback.calls) == 1 + + +async def test_should_failover_can_stop_failover() -> None: + first_error = ValueError("bad request") + seen: list[Exception] = [] + + def should_failover(exception: Exception) -> bool: + seen.append(exception) + return False + + primary = _ScriptedClient([first_error]) + fallback = _ScriptedClient() + client = FailoverChatClient([primary, fallback], should_failover=should_failover) + + with pytest.raises(ValueError) as raised: + await client.get_response([Message(role="user", contents=["hi"])]) + + assert raised.value is first_error + assert seen == [first_error] + assert fallback.calls == [] + + +@pytest.mark.parametrize( + "error", + [ + ResponseInvalidatedException("provider invalidated the response", log_level=None), + MiddlewareException("middleware stopped the call", log_level=None), + ], + ids=["response-invalidated", "middleware-control-flow"], +) +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_protected_errors_never_fail_over(error: Exception, stream: bool) -> None: + primary = _ScriptedClient([error]) + fallback = _ScriptedClient() + client = FailoverChatClient([primary, fallback], should_failover=lambda _: True) + + with pytest.raises(type(error)) as raised: + if stream: + async for _ in client.get_response([Message(role="user", contents=["hi"])], stream=True): + pass + else: + await client.get_response([Message(role="user", contents=["hi"])]) + + assert raised.value is error + assert fallback.calls == [] + + +async def test_cancellation_is_not_failed_over() -> None: + primary = _ScriptedClient([asyncio.CancelledError()]) + fallback = _ScriptedClient() + client = FailoverChatClient([primary, fallback]) + + with pytest.raises(asyncio.CancelledError): + await client.get_response([Message(role="user", contents=["hi"])]) + + assert fallback.calls == [] + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_error_raised_when_starting_the_call_fails_over(stream: bool) -> None: + primary = _ScriptedClient(raise_on_call=_rate_limited("primary")) + fallback = _ScriptedClient([_text_response("fallback")]) + client = FailoverChatClient([primary, fallback]) + + if stream: + response_stream = client.get_response([Message(role="user", contents=["hi"])], stream=True) + async for _ in response_stream: + pass + response = await response_stream.get_final_response() + else: + response = await client.get_response([Message(role="user", contents=["hi"])]) + + assert response.text == "fallback" + + +async def test_streaming_error_after_content_is_not_failed_over() -> None: + error = _rate_limited("primary") + primary = _ScriptedClient([ + [ChatResponseUpdate(contents=[Content.from_text("Your refund")], role="assistant"), error] + ]) + fallback = _ScriptedClient() + client = FailoverChatClient([primary, fallback]) + + received: list[str] = [] + with pytest.raises(ChatClientException) as raised: + async for update in client.get_response([Message(role="user", contents=["hi"])], stream=True): + received.append(update.text) + + assert raised.value is error + assert received == ["Your refund"] + assert fallback.calls == [] + + +async def test_streaming_metadata_from_a_failed_attempt_is_discarded() -> None: + primary = _ScriptedClient([ + [ChatResponseUpdate(response_id="primary-response", role="assistant"), _rate_limited("p")] + ]) + fallback = _ScriptedClient([ + [ + ChatResponseUpdate(response_id="fallback-response", role="assistant"), + ChatResponseUpdate(contents=[Content.from_text("done")], response_id="fallback-response", role="assistant"), + ] + ]) + client = FailoverChatClient([primary, fallback]) + + stream = client.get_response([Message(role="user", contents=["hi"])], stream=True) + updates = [update async for update in stream] + response = await stream.get_final_response() + + assert {update.response_id for update in updates} == {"fallback-response"} + assert response.text == "done" + assert response.response_id == "fallback-response" + + +async def test_streaming_attempt_without_content_updates_succeeds() -> None: + primary = _ScriptedClient([[ChatResponseUpdate(response_id="primary-response", role="assistant")]]) + client = FailoverChatClient([primary, _ScriptedClient()]) + + stream = client.get_response([Message(role="user", contents=["hi"])], stream=True) + updates = [update async for update in stream] + + assert [update.response_id for update in updates] == ["primary-response"] + assert (await stream.get_final_response()).response_id == "primary-response" + + +@pytest.mark.parametrize( + "options", + [ + {"store": True}, + {"conversation_id": "conv_123"}, + {"previous_response_id": "resp_123"}, + ], + ids=["store-true", "service-conversation", "candidate-declared-continuation"], +) +async def test_service_managed_history_is_rejected_before_any_call(options: dict[str, Any]) -> None: + primary = _ResponsesLikeScriptedClient() + fallback = _ScriptedClient() + client = FailoverChatClient([primary, fallback]) + + with pytest.raises(ChatClientInvalidRequestException, match="client-managed history|service-managed"): + await client.get_response([Message(role="user", contents=["hi"])], options=cast("ChatOptions[Any]", options)) + + assert primary.calls == [] + assert fallback.calls == [] + + +async def test_clients_that_store_by_default_receive_store_false() -> None: + primary = _StoringScriptedClient([_rate_limited("primary")]) + fallback = _ScriptedClient([_text_response("fallback")]) + client = FailoverChatClient([primary, fallback]) + + await client.get_response([Message(role="user", contents=["hi"])], options={"temperature": 0.2}) + + assert primary.calls[0]["options"]["store"] is False + assert "store" not in fallback.calls[0]["options"] + assert fallback.calls[0]["options"]["temperature"] == 0.2 + + +async def test_attempts_use_client_chat_middleware_but_not_client_tool_loop( + send_email: Any, sent_emails: list[str] +) -> None: + attempts: list[str] = [] + client_tool_calls: list[str] = [] + failover_tool_calls: list[str] = [] + + class _RecordAttempt(ChatMiddleware): + def __init__(self, name: str) -> None: + self.name = name + + async def process(self, context: ChatContext, call_next: Any) -> None: + attempts.append(self.name) + await call_next() + + class _RecordToolCall(FunctionMiddleware): + def __init__(self, calls: list[str]) -> None: + self.calls = calls + + async def process(self, context: FunctionInvocationContext, call_next: Any) -> None: + self.calls.append(context.function.name) + await call_next() + + primary = _ScriptedClient( + [_tool_call_response(), _rate_limited("primary")], + middleware=[_RecordAttempt("primary"), _RecordToolCall(client_tool_calls)], + ) + fallback = _ScriptedClient([_text_response("done")], middleware=[_RecordAttempt("fallback")]) + client = FailoverChatClient( + [primary, fallback], middleware=[_RecordAttempt("failover"), _RecordToolCall(failover_tool_calls)] + ) + + await client.get_response([Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]}) + + # Failover-level chat middleware runs once per model call, around every attempt of that call. + assert attempts == ["failover", "primary", "failover", "primary", "fallback"] + assert client_tool_calls == [] + assert failover_tool_calls == ["send_email"] + assert sent_emails == ["cfo@contoso.com"] + + +async def test_agent_keeps_history_client_side_across_failover(send_email: Any, sent_emails: list[str]) -> None: + primary = _StoringScriptedClient([_tool_call_response(), _rate_limited("primary"), _text_response("second")]) + fallback = _ScriptedClient([_text_response("first")]) + agent = Agent(client=FailoverChatClient([primary, fallback]), tools=[send_email]) + session = agent.create_session() + + first = await agent.run("Email the CFO.", session=session) + second = await agent.run("Thanks!", session=session) + + assert first.text == "first" + assert second.text == "second" + assert sent_emails == ["cfo@contoso.com"] + # The second run reaches the primary with the whole conversation, including the turn the fallback answered. + second_run_texts = [message.text for message in primary.calls[-1]["messages"]] + assert "Email the CFO." in second_run_texts + assert "first" in second_run_texts + assert "Thanks!" in second_run_texts + assert all(call["options"]["store"] is False for call in primary.calls) + + +async def test_nested_failover_client_is_a_valid_candidate(send_email: Any, sent_emails: list[str]) -> None: + regional = FailoverChatClient([ + _ScriptedClient([_tool_call_response(), _rate_limited("east")]), + _ScriptedClient([_rate_limited("west")]), + ]) + other_provider = _ScriptedClient([_text_response("other provider")]) + client = FailoverChatClient([regional, other_provider]) + + response = await client.get_response( + [Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]} + ) + + assert response.text == "other provider" + assert sent_emails == ["cfo@contoso.com"] + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_clients_without_a_tool_loop_are_supported(stream: bool, send_email: Any, sent_emails: list[str]) -> None: + primary = _RawScriptedClient([_tool_call_response(), _rate_limited("primary")]) + fallback = _RawScriptedClient([_text_response("done via fallback")]) + client = FailoverChatClient([primary, fallback]) + messages = [Message(role="user", contents=["Email the CFO."])] + + if stream: + response_stream = client.get_response(messages, stream=True, options={"tools": [send_email]}) + async for _ in response_stream: + pass + response = await response_stream.get_final_response() + else: + response = await client.get_response(messages, options={"tools": [send_email]}) + + assert response.text == "done via fallback" + assert sent_emails == ["cfo@contoso.com"] + + +async def test_streaming_finalization_error_before_content_fails_over() -> None: + primary = _ScriptedClient( + [[ChatResponseUpdate(response_id="primary-response", role="assistant")]], + finalize_error=_rate_limited("primary"), + ) + fallback = _ScriptedClient([_text_response("fallback")]) + client = FailoverChatClient([primary, fallback]) + + stream = client.get_response([Message(role="user", contents=["hi"])], stream=True) + updates = [update async for update in stream] + + assert {update.response_id for update in updates} == {None} + assert (await stream.get_final_response()).text == "fallback" + + +async def test_streaming_error_opening_the_last_client_propagates() -> None: + last_error = _rate_limited("fallback") + client = FailoverChatClient([ + _ScriptedClient(raise_on_call=_rate_limited("primary")), + _ScriptedClient(raise_on_call=last_error), + ]) + + with pytest.raises(ChatClientException) as raised: + async for _ in client.get_response([Message(role="user", contents=["hi"])], stream=True): + pass + + assert raised.value is last_error + + +async def test_streaming_consumer_can_stop_early() -> None: + primary = _ScriptedClient([ + [ + ChatResponseUpdate(contents=[Content.from_text("first")], role="assistant"), + ChatResponseUpdate(contents=[Content.from_text(" second")], role="assistant"), + ] + ]) + client = FailoverChatClient([primary, _ScriptedClient()]) + + stream = client.get_response([Message(role="user", contents=["hi"])], stream=True) + async for _ in stream: + break + await stream.close() + + assert (await stream.get_final_response()).text == "first" + + +async def test_local_history_conversation_id_is_not_a_service_continuation() -> None: + from agent_framework._sessions import LOCAL_HISTORY_CONVERSATION_ID + + client = FailoverChatClient([_ScriptedClient([_text_response("ok")])]) + + response = await client.get_response( + [Message(role="user", contents=["hi"])], options={"conversation_id": LOCAL_HISTORY_CONVERSATION_ID} + ) + + assert response.text == "ok" + + +async def test_close_leaves_clients_open() -> None: + primary = _ScriptedClient() + fallback = _ScriptedClient() + + async with FailoverChatClient([primary, fallback]) as client: + assert client.clients == (primary, fallback) + + assert not primary.closed + assert not fallback.closed + + +def test_constructor_validates_clients() -> None: + primary = _ScriptedClient() + + with pytest.raises(ValueError, match="at least one"): + FailoverChatClient([]) + with pytest.raises(ValueError, match="distinct"): + FailoverChatClient([primary, primary]) + not_a_client: Any = object() + with pytest.raises(TypeError, match="get_response"): + FailoverChatClient([primary, not_a_client]) + + +def test_failover_client_is_experimental() -> None: + from agent_framework import ExperimentalFeature + + assert getattr(FailoverChatClient, "__feature_id__", None) == ExperimentalFeature.CHAT_CLIENT_FAILOVER.value diff --git a/python/samples/02-agents/chat_client/README.md b/python/samples/02-agents/chat_client/README.md index abb80337130..fa6eddc8251 100644 --- a/python/samples/02-agents/chat_client/README.md +++ b/python/samples/02-agents/chat_client/README.md @@ -7,6 +7,7 @@ This folder contains examples for direct chat client usage patterns. | File | Description | |------|-------------| | [`built_in_chat_clients.py`](built_in_chat_clients.py) | Consolidated sample for built-in chat clients. Passes a client name to `main()`, which creates the selected client with `get_client()`. | +| [`chat_client_failover.py`](chat_client_failover.py) | Demonstrates `FailoverChatClient`, which retries each model call on the next chat client when one fails, without re-running tools that already ran. Uses `FoundryChatClient` with a deliberately invalid primary deployment so the failover is visible. | | [`chat_response_cancellation.py`](chat_response_cancellation.py) | Demonstrates how to cancel chat responses during streaming, showing proper cancellation handling and cleanup. | | [`custom_chat_client.py`](custom_chat_client.py) | Demonstrates how to create custom chat clients by extending the `BaseChatClient` class. Shows a `EchoingChatClient` implementation and how to integrate it with `Agent` using the `as_agent()` method. | | [`require_per_service_call_history_persistence.py`](require_per_service_call_history_persistence.py) | Compares two otherwise identical `FoundryChatClient` agents with `store=False`; the only difference is whether `require_per_service_call_history_persistence` is enabled, and only the run without it stores the synthesized tool result when middleware terminates the loop early. | @@ -36,7 +37,7 @@ Example: uv run samples/02-agents/chat_client/built_in_chat_clients.py ``` -The `require_per_service_call_history_persistence.py` sample uses `FoundryChatClient`, so set the usual Foundry settings first and sign in with the Azure CLI: +The `require_per_service_call_history_persistence.py` and `chat_client_failover.py` samples use `FoundryChatClient`, so set the usual Foundry settings first and sign in with the Azure CLI: ```bash export FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" diff --git a/python/samples/02-agents/chat_client/chat_client_failover.py b/python/samples/02-agents/chat_client/chat_client_failover.py new file mode 100644 index 00000000000..ec81656e3ef --- /dev/null +++ b/python/samples/02-agents/chat_client/chat_client_failover.py @@ -0,0 +1,90 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import logging +import os +from typing import Annotated + +from agent_framework import Agent, FailoverChatClient, tool +from agent_framework.foundry import FoundryChatClient +from azure.identity import AzureCliCredential +from dotenv import load_dotenv +from pydantic import Field + +# Load environment variables from .env file +load_dotenv() + +""" +Chat Client Failover Example + +This sample sends each model call to an ordered list of chat clients with `FailoverChatClient`. When a +client fails (for example with a 429, a 5xx, or a regional outage), the same model call is retried on the +next client. Failover happens per model call inside the tool loop, so tools that already ran are not run +again, and the next client sees the tool results produced so far. + +To show failover without waiting for a real outage, the primary client points at a deployment name that does +not exist, so every call to it fails. In production the clients would be, for example, a deployment in a +second region or a model from another provider. + +Conversation history stays on the client side: `FailoverChatClient` sends `store=False` to clients that store +history in the service by default, because history stored by one service cannot be read by another. + +Environment variables: + FOUNDRY_PROJECT_ENDPOINT - Microsoft Foundry project endpoint. + FOUNDRY_MODEL - Model deployment name used by the fallback client, for example `gpt-4o`. +""" + +refunds_issued: list[str] = [] + + +# NOTE: approval_mode="never_require" is for sample brevity. +@tool(approval_mode="never_require") +def issue_refund(order_id: Annotated[str, Field(description="The order to refund.")]) -> str: + """Issue a refund for an order.""" + refunds_issued.append(order_id) + return f"Refund issued for order {order_id}." + + +async def main() -> None: + # Show the warning that FailoverChatClient logs for each failover. + logging.basicConfig(level=logging.WARNING, format="%(levelname)s %(message)s") + + credential = AzureCliCredential() + project_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + + # 1. The primary client fails on every call because the deployment does not exist. + primary = FoundryChatClient( + project_endpoint=project_endpoint, + model="deployment-that-does-not-exist", + credential=credential, + ) + # 2. The fallback client uses a working deployment. + fallback = FoundryChatClient( + project_endpoint=project_endpoint, + model=os.environ["FOUNDRY_MODEL"], + credential=credential, + ) + + # 3. Tools, function middleware, and agent middleware are configured once, on the failover client or agent. + agent = Agent( + client=FailoverChatClient([primary, fallback]), + name="SupportAgent", + instructions="You are a customer support agent. Use the tools you have to resolve requests.", + tools=[issue_refund], + ) + + response = await agent.run("Please refund order 123 and confirm when it is done.") + print(f"Agent: {response.text}") + print(f"Refunds issued: {refunds_issued}") + + +if __name__ == "__main__": + asyncio.run(main()) + +""" +Sample output (abridged): + +WARNING Chat client 1 of 2 (FoundryChatClient) failed with ...; trying the next chat client. +Agent: I've issued a refund for order 123. +Refunds issued: ['123'] +""" From f14de8ee93721379ec3787bf6e5dad8fab04146f Mon Sep 17 00:00:00 2001 From: karthik-0306 Date: Sun, 15 Nov 2026 00:09:21 +0530 Subject: [PATCH 2/3] Python: Address review feedback on FailoverChatClient Validate options before approval resume, propagate nested continuation names, omit other providers' reasoning per attempt, forward the caller session to candidate middleware, and close sample resources. --- python/packages/core/AGENTS.md | 14 +- .../core/agent_framework/_failover.py | 262 ++++++++++++++++-- .../packages/core/tests/core/test_failover.py | 182 +++++++++++- .../chat_client/chat_client_failover.py | 55 ++-- 4 files changed, 466 insertions(+), 47 deletions(-) diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index 029d0c1bd27..3e9eb418174 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -80,9 +80,17 @@ agent_framework/ `ResponseInvalidatedException` and `MiddlewareException` always propagate. Streaming fails over only before an update with contents is released; content-less updates are held until then and discarded with a failed attempt. History must be client-managed: `store=True` or a service continuation option (the union of candidates' - `SERVICE_CONTINUATION_OPTION_NAMES`) raises `ChatClientInvalidRequestException` before any call, and candidates - with `STORES_BY_DEFAULT` receive `store=False` per attempt when `store` is unset. Candidates are caller-owned and - not closed. + `SERVICE_CONTINUATION_OPTION_NAMES`, including a nested failover client's computed union) raises + `ChatClientInvalidRequestException` in `get_response` before the function-invocation loop resolves approvals, so + no approved tool runs first; the per-model-call check remains for middleware-modified options. Candidates with + `STORES_BY_DEFAULT` receive `store=False` per attempt when `store` is unset. The caller's session is carried past + the failover client's own chat middleware layer through a request-local client kwarg, so candidate chat middleware + receives it. Reasoning is provider-specific: the client records a hashed client-type/model/service-URL source on + each response message that contains reasoning (`additional_properties["agent_framework.failover"]`, first writer + wins so nested clients keep the innermost producer) and omits that reasoning, plus the group's compaction + `has_reasoning` flag, from the per-attempt copy sent to a candidate that does not accept that source. Calls and + results are always sent, the caller's transcript is not modified, and unrecorded reasoning is sent unchanged. + Candidates are caller-owned and not closed. ### Types (`_types.py`) diff --git a/python/packages/core/agent_framework/_failover.py b/python/packages/core/agent_framework/_failover.py index ab77d1eed4f..a519e332e0f 100644 --- a/python/packages/core/agent_framework/_failover.py +++ b/python/packages/core/agent_framework/_failover.py @@ -4,16 +4,19 @@ from __future__ import annotations +import copy +import hashlib import logging import sys from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Generic, TypedDict, cast +from typing import TYPE_CHECKING, Any, Generic, Literal, TypedDict, cast, overload from ._clients import BaseChatClient, SupportsChatGetResponse +from ._compaction import GROUP_ANNOTATION_KEY, GROUP_HAS_REASONING_KEY, GROUP_ID_KEY from ._feature_stage import ExperimentalFeature, experimental from ._middleware import ChatMiddlewareLayer, MiddlewareException from ._sessions import is_local_history_conversation_id -from ._tools import FunctionInvocationLayer +from ._tools import FunctionInvocationLayer, ResponseModelBoundT from ._types import ChatResponse, ChatResponseUpdate, Message, ResponseStream from .exceptions import ChatClientInvalidRequestException, ResponseInvalidatedException @@ -26,7 +29,7 @@ from ._compaction import CompactionStrategy, TokenizerProtocol from ._middleware import ChatAndFunctionMiddlewareTypes from ._tools import FunctionInvocationConfiguration - from ._types import ChatOptions # referenced by the OptionsCoT default + from ._types import ChatOptions logger = logging.getLogger("agent_framework") @@ -40,6 +43,52 @@ FailoverPredicate = Callable[[Exception], bool] """Decide whether a failed chat client attempt may be retried on the next chat client.""" +_FAILOVER_METADATA_KEY = "agent_framework.failover" +"""Message ``additional_properties`` namespace recording which candidate produced a message's reasoning.""" + +_CANDIDATE_SESSION_KEY = "_agent_framework_failover_session" +"""Request-local client kwarg that carries the caller's session past this client's chat middleware layer.""" + + +def _client_source(client: SupportsChatGetResponse[Any]) -> str: + """Return a stable, opaque identity for the service a client calls. + + The identity combines the client type, model, and service URL, so two deployments of the same model in + different regions are different sources. It is hashed so endpoints are not copied into persisted history. + """ + service_url = getattr(client, "service_url", None) + try: + url = service_url() if callable(service_url) else None + except Exception: + url = None + identity = f"{type(client).__module__}.{type(client).__qualname__}|{getattr(client, 'model', None)}|{url}" + return hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16] + + +def _message_source(message: Message) -> str | None: + metadata = message.additional_properties.get(_FAILOVER_METADATA_KEY) + if isinstance(metadata, Mapping): + source = cast(Mapping[str, Any], metadata).get("source") + if isinstance(source, str): + return source + return None + + +def _group_annotation(message: Message) -> Mapping[str, Any] | None: + annotation = message.additional_properties.get(GROUP_ANNOTATION_KEY) + return cast(Mapping[str, Any], annotation) if isinstance(annotation, Mapping) else None + + +def _without_reasoning(message: Message) -> Message: + """Return a copy of ``message`` without reasoning contents or a group reasoning flag.""" + projected = copy.copy(message) + projected.contents = [content for content in message.contents if content.type != "text_reasoning"] + projected.additional_properties = dict(message.additional_properties) + annotation = _group_annotation(message) + if annotation is not None and annotation.get(GROUP_HAS_REASONING_KEY) is True: + projected.additional_properties[GROUP_ANNOTATION_KEY] = {**annotation, GROUP_HAS_REASONING_KEY: False} + return projected + @experimental(feature_id=ExperimentalFeature.CHAT_CLIENT_FAILOVER) class FailoverChatClient( @@ -59,7 +108,8 @@ class FailoverChatClient( Each attempt goes through the selected client's chat middleware and telemetry layers but not its function-invocation layer. Configure tools, function middleware, and ``function_invocation_configuration`` on the ``FailoverChatClient`` (or the agent that uses it). Chat middleware configured on the failover client, or - passed by an agent, runs once per model call around all attempts. + passed by an agent, runs once per model call around all attempts. Each client's own chat middleware runs for + each of its attempts and receives the caller's session. Failover rules: @@ -76,10 +126,17 @@ class FailoverChatClient( Conversation history must be client-managed because history stored by one service cannot be read by another. Requests that set ``store=True`` or carry a service-managed continuation (such as ``conversation_id`` or - ``previous_response_id``) are rejected with ``ChatClientInvalidRequestException`` before any client is called. - Clients that store history by default (``STORES_BY_DEFAULT``) receive ``store=False`` for their attempt unless - the request sets ``store`` explicitly. Agents using this client keep history locally, as they do for any - client that does not store history by default. + ``previous_response_id``) are rejected with ``ChatClientInvalidRequestException`` before the function-invocation + loop starts, so no approved tool runs for a request that cannot be served. Clients that store history by default + (``STORES_BY_DEFAULT``) receive ``store=False`` for their attempt unless the request sets ``store`` explicitly. + Agents using this client keep history locally, as they do for any client that does not store history by default. + + Reasoning is provider-specific: a provider can only replay reasoning it produced itself (for example, OpenAI + requires its encrypted reasoning payload). This client records which client produced each response's reasoning + and omits that reasoning from the copy of the conversation sent to a different client, together with the + compaction flag that marks its group as containing reasoning. Function calls and results are always sent, and the + caller's transcript is never modified. Reasoning in history that this client did not record (for example, from + an earlier streamed run) is sent unchanged. The chat clients are caller-owned: ``close()`` and ``async with`` do not close them. @@ -142,14 +199,36 @@ def __init__( ) self._clients = candidates self._should_failover = should_failover + continuation_option_names = {"conversation_id"} for candidate in candidates: - declared = getattr(candidate, "SERVICE_CONTINUATION_OPTION_NAMES", ()) + # A nested failover client must contribute its own candidates' names, not the inherited default. + declared = ( + candidate._continuation_option_names + if isinstance(candidate, FailoverChatClient) + else getattr(candidate, "SERVICE_CONTINUATION_OPTION_NAMES", ()) + ) if isinstance(declared, Sequence) and not isinstance(declared, (str, bytes, bytearray)): continuation_option_names.update( name for name in cast(Sequence[Any], declared) if isinstance(name, str) ) self._continuation_option_names = tuple(sorted(continuation_option_names)) + + # Reasoning recorded with one of a candidate's sources is replayed to that candidate. A nested failover + # client records its own candidates' sources, so it accepts all of them. + candidate_sources: list[str | None] = [] + accepted_sources: list[frozenset[str]] = [] + for candidate in candidates: + if isinstance(candidate, FailoverChatClient): + candidate_sources.append(None) + accepted_sources.append(candidate._all_sources) + else: + source = _client_source(candidate) + candidate_sources.append(source) + accepted_sources.append(frozenset({source})) + self._candidate_sources = tuple(candidate_sources) + self._accepted_sources = tuple(accepted_sources) + self._all_sources: frozenset[str] = frozenset[str]().union(*accepted_sources) super().__init__( middleware=middleware, function_invocation_configuration=function_invocation_configuration, @@ -163,6 +242,103 @@ def clients(self) -> tuple[SupportsChatGetResponse[Any], ...]: """The chat clients tried for each model call, in order.""" return self._clients + @overload + def get_response( + self, + messages: Sequence[Message], + *, + stream: Literal[False] = ..., + options: ChatOptions[ResponseModelBoundT], + middleware: Sequence[ChatAndFunctionMiddlewareTypes] | None = None, + compaction_strategy: CompactionStrategy | None = None, + tokenizer: TokenizerProtocol | None = None, + function_invocation_kwargs: Mapping[str, Any] | None = None, + client_kwargs: Mapping[str, Any] | None = None, + ) -> Awaitable[ChatResponse[ResponseModelBoundT]]: ... + + @overload + def get_response( + self, + messages: Sequence[Message], + *, + stream: Literal[False] = ..., + options: OptionsCoT | ChatOptions[None] | None = None, + middleware: Sequence[ChatAndFunctionMiddlewareTypes] | None = None, + compaction_strategy: CompactionStrategy | None = None, + tokenizer: TokenizerProtocol | None = None, + function_invocation_kwargs: Mapping[str, Any] | None = None, + client_kwargs: Mapping[str, Any] | None = None, + ) -> Awaitable[ChatResponse[Any]]: ... + + @overload + def get_response( + self, + messages: Sequence[Message], + *, + stream: Literal[True], + options: OptionsCoT | ChatOptions[Any] | None = None, + middleware: Sequence[ChatAndFunctionMiddlewareTypes] | None = None, + compaction_strategy: CompactionStrategy | None = None, + tokenizer: TokenizerProtocol | None = None, + function_invocation_kwargs: Mapping[str, Any] | None = None, + client_kwargs: Mapping[str, Any] | None = None, + ) -> ResponseStream[ChatResponseUpdate, ChatResponse[Any]]: ... + + def get_response( + self, + messages: Sequence[Message], + *, + stream: bool = False, + options: OptionsCoT | ChatOptions[Any] | None = None, + middleware: Sequence[ChatAndFunctionMiddlewareTypes] | None = None, + compaction_strategy: CompactionStrategy | None = None, + tokenizer: TokenizerProtocol | None = None, + function_invocation_kwargs: Mapping[str, Any] | None = None, + client_kwargs: Mapping[str, Any] | None = None, + ) -> Awaitable[ChatResponse[Any]] | ResponseStream[ChatResponseUpdate, ChatResponse[Any]]: + """Get a response, failing over to the next chat client for each model call that fails. + + Args: + messages: The messages to send. + + Keyword Args: + stream: Whether to stream the response. + options: Chat options forwarded to each attempted client. + middleware: Additional chat and function middleware for this request. + compaction_strategy: Optional per-call compaction override. + tokenizer: Optional per-call tokenizer override. + function_invocation_kwargs: Keyword arguments forwarded only to tool invocations. + client_kwargs: Additional client keyword arguments forwarded to each attempted client. + + Returns: + When ``stream`` is ``False``, an awaitable response; otherwise a response stream. + + Raises: + ChatClientInvalidRequestException: If ``options`` set ``store=True`` or carry a service-managed + continuation. This is raised before any approved tool runs or any client is called. + """ + # Reject service-managed history before the inherited loop resolves approvals, which can run tools + # without a model call. The per-model-call check in _inner_get_response still covers middleware changes. + self._ensure_client_managed_history(cast("Mapping[str, Any] | None", options) or {}) + forwarded_client_kwargs = dict(client_kwargs) if client_kwargs is not None else {} + if (session := forwarded_client_kwargs.get("session")) is not None: + # This client's chat middleware layer consumes "session"; carry it to the candidates' chat layers. + forwarded_client_kwargs[_CANDIDATE_SESSION_KEY] = session + get_response = cast("Callable[..., Any]", super().get_response) + return cast( + "Awaitable[ChatResponse[Any]] | ResponseStream[ChatResponseUpdate, ChatResponse[Any]]", + get_response( + messages, + stream=stream, + options=options, + middleware=middleware, + compaction_strategy=compaction_strategy, + tokenizer=tokenizer, + function_invocation_kwargs=function_invocation_kwargs, + client_kwargs=forwarded_client_kwargs, + ), + ) + def _inner_get_response( self, *, @@ -171,9 +347,12 @@ def _inner_get_response( options: Mapping[str, Any], **kwargs: Any, ) -> Awaitable[ChatResponse] | ResponseStream[ChatResponseUpdate, ChatResponse]: + client_kwargs = dict(kwargs) + if (session := client_kwargs.pop(_CANDIDATE_SESSION_KEY, None)) is not None: + client_kwargs["session"] = session if stream: - return self._stream_with_failover(messages=messages, options=options, client_kwargs=kwargs) - return self._get_response_with_failover(messages=messages, options=options, client_kwargs=kwargs) + return self._stream_with_failover(messages=messages, options=options, client_kwargs=client_kwargs) + return self._get_response_with_failover(messages=messages, options=options, client_kwargs=client_kwargs) async def _get_response_with_failover( self, @@ -186,16 +365,19 @@ async def _get_response_with_failover( last_index = len(self._clients) - 1 for index, client in enumerate(self._clients): try: - return await cast( + response = await cast( "Awaitable[ChatResponse]", self._call_client( - client, messages=messages, stream=False, options=options, client_kwargs=client_kwargs + index, messages=messages, stream=False, options=options, client_kwargs=client_kwargs ), ) except Exception as exception: if index == last_index or not self._is_failover_allowed(exception): raise self._log_failover(index, client, exception) + continue + self._record_reasoning_source(index, response) + return response raise AssertionError("unreachable") # pragma: no cover def _stream_with_failover( @@ -217,7 +399,7 @@ async def _updates() -> AsyncIterator[ChatResponseUpdate]: attempt = await cast( "ResponseStream[ChatResponseUpdate, ChatResponse]", self._call_client( - client, messages=messages, stream=True, options=options, client_kwargs=client_kwargs + index, messages=messages, stream=True, options=options, client_kwargs=client_kwargs ), ) except Exception as exception: @@ -258,6 +440,7 @@ async def _updates() -> AsyncIterator[ChatResponseUpdate]: final_response = await attempt.get_final_response() for held_update in held: yield held_update + self._record_reasoning_source(index, final_response) final_responses.append(final_response) return @@ -271,14 +454,15 @@ def _finalize(updates: Sequence[ChatResponseUpdate]) -> ChatResponse: def _call_client( self, - client: SupportsChatGetResponse[Any], + index: int, *, messages: Sequence[Message], stream: bool, options: Mapping[str, Any], client_kwargs: Mapping[str, Any], ) -> Awaitable[ChatResponse[Any]] | ResponseStream[ChatResponseUpdate, ChatResponse[Any]]: - """Run one model call on ``client`` without its function-invocation loop.""" + """Run one model call on the client at ``index`` without its function-invocation loop.""" + client = self._clients[index] attempt_options = dict(options) if getattr(client, "STORES_BY_DEFAULT", False) is True and attempt_options.get("store") is None: attempt_options["store"] = False @@ -292,12 +476,56 @@ def _call_client( target.get_response, ) return get_response( - messages=messages, + messages=self._messages_for_client(index, messages), stream=stream, options=attempt_options, client_kwargs=dict(client_kwargs), ) + def _record_reasoning_source(self, index: int, response: ChatResponse[Any]) -> None: + """Record which client produced the reasoning in ``response``, keeping a nested client's record.""" + source = self._candidate_sources[index] + if source is None: + return + for message in response.messages: + if any(content.type == "text_reasoning" for content in message.contents): + message.additional_properties.setdefault(_FAILOVER_METADATA_KEY, {"source": source}) + + def _messages_for_client(self, index: int, messages: Sequence[Message]) -> Sequence[Message]: + """Return the conversation to send to the client at ``index``. + + Reasoning recorded as produced by a different client is omitted, along with the compaction flag that marks + its group as containing reasoning. The caller's messages are not modified; unchanged conversations are + returned as-is. + """ + accepted = self._accepted_sources[index] + projected: list[Message] | None = None + foreign_groups: set[Any] = set() + for position, message in enumerate(messages): + source = _message_source(message) + if source is None or source in accepted: + continue + if not any(content.type == "text_reasoning" for content in message.contents): + continue + if projected is None: + projected = list(messages) + annotation = _group_annotation(message) + if annotation is not None and annotation.get(GROUP_ID_KEY) is not None: + foreign_groups.add(annotation[GROUP_ID_KEY]) + projected[position] = _without_reasoning(message) + if projected is None: + return messages + if foreign_groups: + for position, message in enumerate(projected): + annotation = _group_annotation(message) + if ( + annotation is not None + and annotation.get(GROUP_ID_KEY) in foreign_groups + and annotation.get(GROUP_HAS_REASONING_KEY) is True + ): + projected[position] = _without_reasoning(message) + return [message for message in projected if message.contents] + def _ensure_client_managed_history(self, options: Mapping[str, Any]) -> None: if options.get("store") is True: raise ChatClientInvalidRequestException( diff --git a/python/packages/core/tests/core/test_failover.py b/python/packages/core/tests/core/test_failover.py index 97b57526237..daf71cf15f8 100644 --- a/python/packages/core/tests/core/test_failover.py +++ b/python/packages/core/tests/core/test_failover.py @@ -46,8 +46,10 @@ def __init__( *, raise_on_call: BaseException | None = None, finalize_error: Exception | None = None, + model: str | None = None, ) -> None: super().__init__() + self.model = model self._steps = list(steps) self._raise_on_call = raise_on_call self._finalize_error = finalize_error @@ -104,8 +106,15 @@ def __init__( raise_on_call: BaseException | None = None, finalize_error: Exception | None = None, middleware: Sequence[Any] | None = None, + model: str | None = None, ) -> None: - super().__init__(middleware=middleware, steps=steps, raise_on_call=raise_on_call, finalize_error=finalize_error) + super().__init__( + middleware=middleware, + steps=steps, + raise_on_call=raise_on_call, + finalize_error=finalize_error, + model=model, + ) class _StoringScriptedClient(_ScriptedClient): @@ -567,3 +576,174 @@ def test_failover_client_is_experimental() -> None: from agent_framework import ExperimentalFeature assert getattr(FailoverChatClient, "__feature_id__", None) == ExperimentalFeature.CHAT_CLIENT_FAILOVER.value + + +# region Review regressions + + +def _reasoning_tool_call_response(call_id: str = "call-1") -> ChatResponse: + # Plain-text thinking without a provider replay payload, as local providers such as Ollama emit it. + reasoning = Content.from_text_reasoning(text="I should email the CFO.") + call = Content.from_function_call(call_id=call_id, name="send_email", arguments={"to": "cfo@contoso.com"}) + return ChatResponse(messages=Message(role="assistant", contents=[reasoning, call])) + + +def _content_types(messages: Sequence[Message]) -> list[str]: + return [content.type for message in messages for content in message.contents] + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_approval_resume_rejects_service_managed_history_before_running_tools(stream: bool) -> None: + executed: list[str] = [] + + @tool(approval_mode="always_require") + def delete_records(table: str) -> str: + """Delete records.""" + executed.append(table) + return "deleted" + + call = Content.from_function_call(call_id="call-1", name="delete_records", arguments={"table": "orders"}) + primary = _ScriptedClient([ChatResponse(messages=Message(role="assistant", contents=[call]))]) + agent = Agent(client=FailoverChatClient([primary, _ScriptedClient()]), tools=[delete_records]) + session = agent.create_session() + first = await agent.run("Delete the orders table.", session=session) + approval = first.user_input_requests[0].to_function_approval_response(approved=True) + + with pytest.raises(ChatClientInvalidRequestException, match="client-managed history"): + if stream: + await agent.run(approval, session=session, stream=True, options={"store": True}).get_final_response() + else: + await agent.run(approval, session=session, options={"store": True}) + + assert executed == [] + + +async def test_nested_failover_rejects_inner_continuation_options_before_any_call() -> None: + inner_primary = _ResponsesLikeScriptedClient() + inner_fallback = _ScriptedClient() + outer_fallback = _ScriptedClient() + client = FailoverChatClient([FailoverChatClient([inner_primary, inner_fallback]), outer_fallback]) + + with pytest.raises(ChatClientInvalidRequestException, match="previous_response_id"): + await client.get_response( + [Message(role="user", contents=["hi"])], + options=cast("ChatOptions[Any]", {"previous_response_id": "resp_123"}), + ) + + assert inner_primary.calls == [] + assert inner_fallback.calls == [] + assert outer_fallback.calls == [] + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_candidate_chat_middleware_receives_the_caller_session(stream: bool) -> None: + seen_sessions: list[Any] = [] + + class _RecordSession(ChatMiddleware): + async def process(self, context: ChatContext, call_next: Any) -> None: + seen_sessions.append(context.session) + await call_next() + + primary = _ScriptedClient([_rate_limited("primary")], middleware=[_RecordSession()]) + fallback = _ScriptedClient([_text_response("ok")], middleware=[_RecordSession()]) + agent = Agent(client=FailoverChatClient([primary, fallback])) + session = agent.create_session() + + if stream: + await agent.run("hi", session=session, stream=True).get_final_response() + else: + await agent.run("hi", session=session) + + assert seen_sessions == [session, session] + + +async def test_reasoning_from_another_client_is_not_sent_to_the_fallback( + send_email: Any, sent_emails: list[str] +) -> None: + primary = _ScriptedClient([_reasoning_tool_call_response(), _rate_limited("primary")], model="local-model") + fallback = _ScriptedClient([_text_response("done")], model="hosted-model") + client = FailoverChatClient([primary, fallback]) + + response = await client.get_response( + [Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]} + ) + + assert response.text == "done" + assert sent_emails == ["cfo@contoso.com"] + # The client that produced the reasoning still receives it on its next model call. + assert "text_reasoning" in _content_types(primary.calls[1]["messages"]) + # A different client receives the tool call and result, but not reasoning it cannot replay. + fallback_types = _content_types(fallback.calls[0]["messages"]) + assert "text_reasoning" not in fallback_types + assert {"function_call", "function_result"} <= set(fallback_types) + # The caller-visible transcript is unchanged. + assert "text_reasoning" in _content_types(response.messages) + + +async def test_nested_failover_keeps_reasoning_for_the_client_that_produced_it( + send_email: Any, sent_emails: list[str] +) -> None: + producer = _ScriptedClient([_reasoning_tool_call_response(), _text_response("producer answered")], model="a") + regional = FailoverChatClient([producer, _ScriptedClient(model="b")]) + client = FailoverChatClient([regional, _ScriptedClient(model="c")]) + + response = await client.get_response( + [Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]} + ) + + assert response.text == "producer answered" + assert sent_emails == ["cfo@contoso.com"] + assert "text_reasoning" in _content_types(producer.calls[1]["messages"]) + + +@pytest.mark.parametrize("annotate", [False, True], ids=["plain-history", "compaction-annotated-history"]) +async def test_openai_fallback_accepts_history_with_another_providers_reasoning( + annotate: bool, send_email: Any, sent_emails: list[str] +) -> None: + pytest.importorskip("agent_framework_openai") + from unittest.mock import MagicMock, patch + + from openai.types.responses import Response + + from agent_framework import CharacterEstimatorTokenizer + from agent_framework.openai import OpenAIChatClient + + sdk_response = Response.model_validate({ + "id": "resp_fallback", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-test", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Done.", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + }) + # The client reads responses through the SDK's raw-response wrapper, as in the OpenAI package tests. + raw_response = MagicMock(headers={}) + raw_response.parse.return_value = sdk_response + + fallback = OpenAIChatClient(model="gpt-test", api_key="test-key") + primary = _ScriptedClient([_reasoning_tool_call_response(), _rate_limited("primary")], model="local-model") + client = FailoverChatClient([primary, fallback], tokenizer=CharacterEstimatorTokenizer() if annotate else None) + + with patch.object(fallback.client.responses, "create", return_value=raw_response) as create: + response = await client.get_response( + [Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]} + ) + + assert response.text == "Done." + assert sent_emails == ["cfo@contoso.com"] + # The real OpenAI request preparation accepted the conversation and sent it to the service once. + assert create.call_count == 1 + sent_item_types = [item.get("type") for item in create.call_args.kwargs["input"]] + assert "reasoning" not in sent_item_types + assert {"function_call", "function_call_output"} <= set(sent_item_types) diff --git a/python/samples/02-agents/chat_client/chat_client_failover.py b/python/samples/02-agents/chat_client/chat_client_failover.py index ec81656e3ef..0907d5e0681 100644 --- a/python/samples/02-agents/chat_client/chat_client_failover.py +++ b/python/samples/02-agents/chat_client/chat_client_failover.py @@ -7,7 +7,7 @@ from agent_framework import Agent, FailoverChatClient, tool from agent_framework.foundry import FoundryChatClient -from azure.identity import AzureCliCredential +from azure.identity.aio import AzureCliCredential from dotenv import load_dotenv from pydantic import Field @@ -49,33 +49,36 @@ async def main() -> None: # Show the warning that FailoverChatClient logs for each failover. logging.basicConfig(level=logging.WARNING, format="%(levelname)s %(message)s") - credential = AzureCliCredential() project_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] - # 1. The primary client fails on every call because the deployment does not exist. - primary = FoundryChatClient( - project_endpoint=project_endpoint, - model="deployment-that-does-not-exist", - credential=credential, - ) - # 2. The fallback client uses a working deployment. - fallback = FoundryChatClient( - project_endpoint=project_endpoint, - model=os.environ["FOUNDRY_MODEL"], - credential=credential, - ) - - # 3. Tools, function middleware, and agent middleware are configured once, on the failover client or agent. - agent = Agent( - client=FailoverChatClient([primary, fallback]), - name="SupportAgent", - instructions="You are a customer support agent. Use the tools you have to resolve requests.", - tools=[issue_refund], - ) - - response = await agent.run("Please refund order 123 and confirm when it is done.") - print(f"Agent: {response.text}") - print(f"Refunds issued: {refunds_issued}") + # FailoverChatClient does not close the clients it is given, so the sample owns the credential and both + # clients and closes them on success, failure, or cancellation. + async with ( + AzureCliCredential() as credential, + # 1. The primary client fails on every call because the deployment does not exist. + FoundryChatClient( + project_endpoint=project_endpoint, + model="deployment-that-does-not-exist", + credential=credential, + ) as primary, + # 2. The fallback client uses a working deployment. + FoundryChatClient( + project_endpoint=project_endpoint, + model=os.environ["FOUNDRY_MODEL"], + credential=credential, + ) as fallback, + ): + # 3. Tools, function middleware, and agent middleware are configured once, on the failover client or agent. + agent = Agent( + client=FailoverChatClient([primary, fallback]), + name="SupportAgent", + instructions="You are a customer support agent. Use the tools you have to resolve requests.", + tools=[issue_refund], + ) + + response = await agent.run("Please refund order 123 and confirm when it is done.") + print(f"Agent: {response.text}") + print(f"Refunds issued: {refunds_issued}") if __name__ == "__main__": From 7602faca748ee481d565b50a5f22d3ee5002349e Mon Sep 17 00:00:00 2001 From: karthik-0306 Date: Sun, 15 Nov 2026 00:25:08 +0530 Subject: [PATCH 3/3] Python: Keep failover reasoning provenance across streamed runs and forward sessions to nested failover clients --- python/packages/core/AGENTS.md | 8 ++- .../core/agent_framework/_failover.py | 69 ++++++++++++------- .../packages/core/tests/core/test_failover.py | 49 +++++++++++++ 3 files changed, 97 insertions(+), 29 deletions(-) diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index 3e9eb418174..faae31274e7 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -86,9 +86,11 @@ agent_framework/ `STORES_BY_DEFAULT` receive `store=False` per attempt when `store` is unset. The caller's session is carried past the failover client's own chat middleware layer through a request-local client kwarg, so candidate chat middleware receives it. Reasoning is provider-specific: the client records a hashed client-type/model/service-URL source on - each response message that contains reasoning (`additional_properties["agent_framework.failover"]`, first writer - wins so nested clients keep the innermost producer) and omits that reasoning, plus the group's compaction - `has_reasoning` flag, from the per-attempt copy sent to a candidate that does not accept that source. Calls and + each `text_reasoning` content, in streamed updates and final responses + (`Content.additional_properties["agent_framework.failover"]`; content-level so it survives streamed-history + aggregation and persistence; first writer wins so nested clients keep the innermost producer) and omits that + reasoning, plus the group's compaction `has_reasoning` flag, from the per-attempt copy sent to a candidate that + does not accept that source. Calls and results are always sent, the caller's transcript is not modified, and unrecorded reasoning is sent unchanged. Candidates are caller-owned and not closed. diff --git a/python/packages/core/agent_framework/_failover.py b/python/packages/core/agent_framework/_failover.py index a519e332e0f..d1643cf78f9 100644 --- a/python/packages/core/agent_framework/_failover.py +++ b/python/packages/core/agent_framework/_failover.py @@ -17,7 +17,7 @@ from ._middleware import ChatMiddlewareLayer, MiddlewareException from ._sessions import is_local_history_conversation_id from ._tools import FunctionInvocationLayer, ResponseModelBoundT -from ._types import ChatResponse, ChatResponseUpdate, Message, ResponseStream +from ._types import ChatResponse, ChatResponseUpdate, Content, Message, ResponseStream from .exceptions import ChatClientInvalidRequestException, ResponseInvalidatedException if sys.version_info >= (3, 13): @@ -44,7 +44,11 @@ """Decide whether a failed chat client attempt may be retried on the next chat client.""" _FAILOVER_METADATA_KEY = "agent_framework.failover" -"""Message ``additional_properties`` namespace recording which candidate produced a message's reasoning.""" +"""Reasoning ``Content.additional_properties`` namespace recording which candidate produced the reasoning. + +It is stored on the content rather than the message because streamed messages are rebuilt from updates, which keeps +content metadata but not message metadata. +""" _CANDIDATE_SESSION_KEY = "_agent_framework_failover_session" """Request-local client kwarg that carries the caller's session past this client's chat middleware layer.""" @@ -65,8 +69,8 @@ def _client_source(client: SupportsChatGetResponse[Any]) -> str: return hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16] -def _message_source(message: Message) -> str | None: - metadata = message.additional_properties.get(_FAILOVER_METADATA_KEY) +def _content_source(content: Content) -> str | None: + metadata = content.additional_properties.get(_FAILOVER_METADATA_KEY) if isinstance(metadata, Mapping): source = cast(Mapping[str, Any], metadata).get("source") if isinstance(source, str): @@ -74,15 +78,22 @@ def _message_source(message: Message) -> str | None: return None +def _is_foreign_reasoning(content: Content, accepted: frozenset[str]) -> bool: + if content.type != "text_reasoning": + return False + source = _content_source(content) + return source is not None and source not in accepted + + def _group_annotation(message: Message) -> Mapping[str, Any] | None: annotation = message.additional_properties.get(GROUP_ANNOTATION_KEY) return cast(Mapping[str, Any], annotation) if isinstance(annotation, Mapping) else None -def _without_reasoning(message: Message) -> Message: - """Return a copy of ``message`` without reasoning contents or a group reasoning flag.""" +def _without_reasoning(message: Message, accepted: frozenset[str]) -> Message: + """Return a copy of ``message`` without foreign reasoning contents or a group reasoning flag.""" projected = copy.copy(message) - projected.contents = [content for content in message.contents if content.type != "text_reasoning"] + projected.contents = [content for content in message.contents if not _is_foreign_reasoning(content, accepted)] projected.additional_properties = dict(message.additional_properties) annotation = _group_annotation(message) if annotation is not None and annotation.get(GROUP_HAS_REASONING_KEY) is True: @@ -132,11 +143,12 @@ class FailoverChatClient( Agents using this client keep history locally, as they do for any client that does not store history by default. Reasoning is provider-specific: a provider can only replay reasoning it produced itself (for example, OpenAI - requires its encrypted reasoning payload). This client records which client produced each response's reasoning - and omits that reasoning from the copy of the conversation sent to a different client, together with the - compaction flag that marks its group as containing reasoning. Function calls and results are always sent, and the - caller's transcript is never modified. Reasoning in history that this client did not record (for example, from - an earlier streamed run) is sent unchanged. + requires its encrypted reasoning payload). This client records on each reasoning content, in streamed and + non-streamed responses, which client produced it. That record is kept in the conversation history, so it also + applies to later runs. Reasoning produced by a different client is omitted from the copy of the conversation sent + to a client, together with the compaction flag that marks its group as containing reasoning. Function calls and + results are always sent, and the caller's transcript is never modified. Reasoning without a record (for example, + history created before this client was used) is sent unchanged. The chat clients are caller-owned: ``close()`` and ``async with`` do not close them. @@ -376,7 +388,8 @@ async def _get_response_with_failover( raise self._log_failover(index, client, exception) continue - self._record_reasoning_source(index, response) + for message in response.messages: + self._record_reasoning_source(index, message.contents) return response raise AssertionError("unreachable") # pragma: no cover @@ -424,6 +437,8 @@ async def _updates() -> AsyncIterator[ChatResponseUpdate]: self._log_failover(index, client, exception) failed = True break + # Record on the streamed contents too: callers rebuild streamed history from updates. + self._record_reasoning_source(index, update.contents) if not released and not update.contents: held.append(update) continue @@ -440,7 +455,8 @@ async def _updates() -> AsyncIterator[ChatResponseUpdate]: final_response = await attempt.get_final_response() for held_update in held: yield held_update - self._record_reasoning_source(index, final_response) + for message in final_response.messages: + self._record_reasoning_source(index, message.contents) final_responses.append(final_response) return @@ -466,6 +482,10 @@ def _call_client( attempt_options = dict(options) if getattr(client, "STORES_BY_DEFAULT", False) is True and attempt_options.get("store") is None: attempt_options["store"] = False + attempt_client_kwargs = dict(client_kwargs) + if isinstance(client, FailoverChatClient) and (session := attempt_client_kwargs.get("session")) is not None: + # A nested failover client is entered below its own get_response, so hand it the session the same way. + attempt_client_kwargs[_CANDIDATE_SESSION_KEY] = session target: Any = client if isinstance(client, FunctionInvocationLayer): # This is the same call a client's own function-invocation loop makes for each model call: it enters the @@ -479,17 +499,17 @@ def _call_client( messages=self._messages_for_client(index, messages), stream=stream, options=attempt_options, - client_kwargs=dict(client_kwargs), + client_kwargs=attempt_client_kwargs, ) - def _record_reasoning_source(self, index: int, response: ChatResponse[Any]) -> None: - """Record which client produced the reasoning in ``response``, keeping a nested client's record.""" + def _record_reasoning_source(self, index: int, contents: Sequence[Content]) -> None: + """Record which client produced each reasoning content, keeping a nested client's record.""" source = self._candidate_sources[index] if source is None: return - for message in response.messages: - if any(content.type == "text_reasoning" for content in message.contents): - message.additional_properties.setdefault(_FAILOVER_METADATA_KEY, {"source": source}) + for content in contents: + if content.type == "text_reasoning": + content.additional_properties.setdefault(_FAILOVER_METADATA_KEY, {"source": source}) def _messages_for_client(self, index: int, messages: Sequence[Message]) -> Sequence[Message]: """Return the conversation to send to the client at ``index``. @@ -502,17 +522,14 @@ def _messages_for_client(self, index: int, messages: Sequence[Message]) -> Seque projected: list[Message] | None = None foreign_groups: set[Any] = set() for position, message in enumerate(messages): - source = _message_source(message) - if source is None or source in accepted: - continue - if not any(content.type == "text_reasoning" for content in message.contents): + if not any(_is_foreign_reasoning(content, accepted) for content in message.contents): continue if projected is None: projected = list(messages) annotation = _group_annotation(message) if annotation is not None and annotation.get(GROUP_ID_KEY) is not None: foreign_groups.add(annotation[GROUP_ID_KEY]) - projected[position] = _without_reasoning(message) + projected[position] = _without_reasoning(message, accepted) if projected is None: return messages if foreign_groups: @@ -523,7 +540,7 @@ def _messages_for_client(self, index: int, messages: Sequence[Message]) -> Seque and annotation.get(GROUP_ID_KEY) in foreign_groups and annotation.get(GROUP_HAS_REASONING_KEY) is True ): - projected[position] = _without_reasoning(message) + projected[position] = _without_reasoning(message, accepted) return [message for message in projected if message.contents] def _ensure_client_managed_history(self, options: Mapping[str, Any]) -> None: diff --git a/python/packages/core/tests/core/test_failover.py b/python/packages/core/tests/core/test_failover.py index daf71cf15f8..2ef74bfc5cb 100644 --- a/python/packages/core/tests/core/test_failover.py +++ b/python/packages/core/tests/core/test_failover.py @@ -657,6 +657,30 @@ async def process(self, context: ChatContext, call_next: Any) -> None: assert seen_sessions == [session, session] +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_nested_failover_candidates_receive_the_caller_session(stream: bool) -> None: + seen_sessions: list[Any] = [] + + class _RecordSession(ChatMiddleware): + async def process(self, context: ChatContext, call_next: Any) -> None: + seen_sessions.append(context.session) + await call_next() + + regional = FailoverChatClient([ + _ScriptedClient([_rate_limited("east")], middleware=[_RecordSession()]), + _ScriptedClient([_text_response("ok")], middleware=[_RecordSession()]), + ]) + agent = Agent(client=FailoverChatClient([regional, _ScriptedClient()])) + session = agent.create_session() + + if stream: + await agent.run("hi", session=session, stream=True).get_final_response() + else: + await agent.run("hi", session=session) + + assert seen_sessions == [session, session] + + async def test_reasoning_from_another_client_is_not_sent_to_the_fallback( send_email: Any, sent_emails: list[str] ) -> None: @@ -680,6 +704,31 @@ async def test_reasoning_from_another_client_is_not_sent_to_the_fallback( assert "text_reasoning" in _content_types(response.messages) +@pytest.mark.parametrize("first_run_stream", [False, True], ids=["non-streamed-history", "streamed-history"]) +async def test_reasoning_recorded_in_an_earlier_run_is_not_sent_to_another_client( + first_run_stream: bool, send_email: Any, sent_emails: list[str] +) -> None: + primary = _ScriptedClient( + [_reasoning_tool_call_response(), _text_response("first answer"), _rate_limited("primary")], + model="local-model", + ) + fallback = _ScriptedClient([_text_response("second answer")], model="hosted-model") + agent = Agent(client=FailoverChatClient([primary, fallback]), tools=[send_email]) + session = agent.create_session() + + if first_run_stream: + await agent.run("Email the CFO.", session=session, stream=True).get_final_response() + else: + await agent.run("Email the CFO.", session=session) + second = await agent.run("Thanks.", session=session) + + assert second.text == "second answer" + assert sent_emails == ["cfo@contoso.com"] + fallback_types = _content_types(fallback.calls[0]["messages"]) + assert "text_reasoning" not in fallback_types + assert {"function_call", "function_result"} <= set(fallback_types) + + async def test_nested_failover_keeps_reasoning_for_the_client_that_produced_it( send_email: Any, sent_emails: list[str] ) -> None: