diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index e6fa66d8221..faae31274e7 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -71,6 +71,28 @@ agent_framework/ for streaming and non-streaming requests. Provides `__aenter__()` returning `self`, `__aexit__()` delegating to `close()`, and a default no-op async `close()`. Connectors override cleanup for owned connections and credentials; injected resources remain caller-owned. `SupportsChatGetResponse` does not require lifecycle methods. +- **`FailoverChatClient`** (`_failover.py`, experimental under `CHAT_CLIENT_FAILOVER`) - Tries an ordered list of + chat clients for each model call. It owns the single function-invocation loop and calls each candidate below that + candidate's `FunctionInvocationLayer` (the same entry the candidate's own loop uses per model call), so candidate + chat middleware and telemetry run per attempt while tools run once, in the failover client's loop. A failure after + earlier model calls requested tools therefore never re-executes them. Every model call starts again with the first + client. Any `Exception` fails over unless `should_failover` returns `False`; cancellation is never caught, and + `ResponseInvalidatedException` and `MiddlewareException` always propagate. Streaming fails over only before an + update with contents is released; content-less updates are held until then and discarded with a failed attempt. + History must be client-managed: `store=True` or a service continuation option (the union of candidates' + `SERVICE_CONTINUATION_OPTION_NAMES`, including a nested failover client's computed union) raises + `ChatClientInvalidRequestException` in `get_response` before the function-invocation loop resolves approvals, so + no approved tool runs first; the per-model-call check remains for middleware-modified options. Candidates with + `STORES_BY_DEFAULT` receive `store=False` per attempt when `store` is unset. The caller's session is carried past + the failover client's own chat middleware layer through a request-local client kwarg, so candidate chat middleware + receives it. Reasoning is provider-specific: the client records a hashed client-type/model/service-URL source on + each `text_reasoning` content, in streamed updates and final responses + (`Content.additional_properties["agent_framework.failover"]`; content-level so it survives streamed-history + aggregation and persistence; first writer wins so nested clients keep the innermost producer) and omits that + reasoning, plus the group's compaction `has_reasoning` flag, from the per-attempt copy sent to a candidate that + does not accept that source. Calls and + results are always sent, the caller's transcript is not modified, and unrecorded reasoning is sent unchanged. + Candidates are caller-owned and not closed. ### Types (`_types.py`) diff --git a/python/packages/core/agent_framework/__init__.py b/python/packages/core/agent_framework/__init__.py index e695e8d1e69..ffbba04421c 100644 --- a/python/packages/core/agent_framework/__init__.py +++ b/python/packages/core/agent_framework/__init__.py @@ -113,6 +113,7 @@ "tool_called_check", "tool_calls_present", ), + "._failover": ("FailoverChatClient",), "._feature_stage": ("ExperimentalFeature", "ReleaseCandidateFeature"), "._harness._agent": ("DEFAULT_HARNESS_INSTRUCTIONS", "create_harness_agent"), "._harness._background_agents": ( @@ -514,6 +515,7 @@ "Executor", "ExpectedToolCall", "ExperimentalFeature", + "FailoverChatClient", "FanInEdgeGroup", "FanOutEdgeGroup", "FieldTypes", diff --git a/python/packages/core/agent_framework/__init__.pyi b/python/packages/core/agent_framework/__init__.pyi index a46b7dcdc94..4af8217ac70 100644 --- a/python/packages/core/agent_framework/__init__.pyi +++ b/python/packages/core/agent_framework/__init__.pyi @@ -75,6 +75,7 @@ from ._evaluation import ( tool_called_check, tool_calls_present, ) +from ._failover import FailoverChatClient from ._feature_stage import ExperimentalFeature, ReleaseCandidateFeature from ._harness._agent import DEFAULT_HARNESS_INSTRUCTIONS, create_harness_agent from ._harness._background_agents import ( @@ -469,6 +470,7 @@ __all__ = [ "Executor", "ExpectedToolCall", "ExperimentalFeature", + "FailoverChatClient", "FanInEdgeGroup", "FanOutEdgeGroup", "FieldTypes", diff --git a/python/packages/core/agent_framework/_failover.py b/python/packages/core/agent_framework/_failover.py new file mode 100644 index 00000000000..d1643cf78f9 --- /dev/null +++ b/python/packages/core/agent_framework/_failover.py @@ -0,0 +1,578 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Ordered failover across chat clients for each model call of the function-invocation loop.""" + +from __future__ import annotations + +import copy +import hashlib +import logging +import sys +from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, Generic, Literal, TypedDict, cast, overload + +from ._clients import BaseChatClient, SupportsChatGetResponse +from ._compaction import GROUP_ANNOTATION_KEY, GROUP_HAS_REASONING_KEY, GROUP_ID_KEY +from ._feature_stage import ExperimentalFeature, experimental +from ._middleware import ChatMiddlewareLayer, MiddlewareException +from ._sessions import is_local_history_conversation_id +from ._tools import FunctionInvocationLayer, ResponseModelBoundT +from ._types import ChatResponse, ChatResponseUpdate, Content, Message, ResponseStream +from .exceptions import ChatClientInvalidRequestException, ResponseInvalidatedException + +if sys.version_info >= (3, 13): + from typing import TypeVar # pragma: no cover +else: + from typing_extensions import TypeVar # pragma: no cover + +if TYPE_CHECKING: + from ._compaction import CompactionStrategy, TokenizerProtocol + from ._middleware import ChatAndFunctionMiddlewareTypes + from ._tools import FunctionInvocationConfiguration + from ._types import ChatOptions + +logger = logging.getLogger("agent_framework") + +OptionsCoT = TypeVar( + "OptionsCoT", + bound=TypedDict, # type: ignore[valid-type] + default="ChatOptions[None]", + covariant=True, +) + +FailoverPredicate = Callable[[Exception], bool] +"""Decide whether a failed chat client attempt may be retried on the next chat client.""" + +_FAILOVER_METADATA_KEY = "agent_framework.failover" +"""Reasoning ``Content.additional_properties`` namespace recording which candidate produced the reasoning. + +It is stored on the content rather than the message because streamed messages are rebuilt from updates, which keeps +content metadata but not message metadata. +""" + +_CANDIDATE_SESSION_KEY = "_agent_framework_failover_session" +"""Request-local client kwarg that carries the caller's session past this client's chat middleware layer.""" + + +def _client_source(client: SupportsChatGetResponse[Any]) -> str: + """Return a stable, opaque identity for the service a client calls. + + The identity combines the client type, model, and service URL, so two deployments of the same model in + different regions are different sources. It is hashed so endpoints are not copied into persisted history. + """ + service_url = getattr(client, "service_url", None) + try: + url = service_url() if callable(service_url) else None + except Exception: + url = None + identity = f"{type(client).__module__}.{type(client).__qualname__}|{getattr(client, 'model', None)}|{url}" + return hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16] + + +def _content_source(content: Content) -> str | None: + metadata = content.additional_properties.get(_FAILOVER_METADATA_KEY) + if isinstance(metadata, Mapping): + source = cast(Mapping[str, Any], metadata).get("source") + if isinstance(source, str): + return source + return None + + +def _is_foreign_reasoning(content: Content, accepted: frozenset[str]) -> bool: + if content.type != "text_reasoning": + return False + source = _content_source(content) + return source is not None and source not in accepted + + +def _group_annotation(message: Message) -> Mapping[str, Any] | None: + annotation = message.additional_properties.get(GROUP_ANNOTATION_KEY) + return cast(Mapping[str, Any], annotation) if isinstance(annotation, Mapping) else None + + +def _without_reasoning(message: Message, accepted: frozenset[str]) -> Message: + """Return a copy of ``message`` without foreign reasoning contents or a group reasoning flag.""" + projected = copy.copy(message) + projected.contents = [content for content in message.contents if not _is_foreign_reasoning(content, accepted)] + projected.additional_properties = dict(message.additional_properties) + annotation = _group_annotation(message) + if annotation is not None and annotation.get(GROUP_HAS_REASONING_KEY) is True: + projected.additional_properties[GROUP_ANNOTATION_KEY] = {**annotation, GROUP_HAS_REASONING_KEY: False} + return projected + + +@experimental(feature_id=ExperimentalFeature.CHAT_CLIENT_FAILOVER) +class FailoverChatClient( + FunctionInvocationLayer[OptionsCoT], + ChatMiddlewareLayer[OptionsCoT], + BaseChatClient[OptionsCoT], + Generic[OptionsCoT], +): + """Send each model call to an ordered list of chat clients until one succeeds. + + ``FailoverChatClient`` owns the single function-invocation loop for a request and fails over per model call, + not per run. When a chat client fails after earlier model calls already requested tools, the next chat client + receives the same conversation, including the tool results produced so far, so no tool runs again. Wrapping + complete chat clients in a plain ``try``/``except`` does not have this property: each client runs its own tool + loop, so a failure on a later model call restarts that loop on the next client and re-executes tools. + + Each attempt goes through the selected client's chat middleware and telemetry layers but not its + function-invocation layer. Configure tools, function middleware, and ``function_invocation_configuration`` on + the ``FailoverChatClient`` (or the agent that uses it). Chat middleware configured on the failover client, or + passed by an agent, runs once per model call around all attempts. Each client's own chat middleware runs for + each of its attempts and receives the caller's session. + + Failover rules: + + - Clients are tried in order for every model call. The first client is tried first again on the next model + call, so a recovered primary resumes serving requests. + - Any ``Exception`` raised by an attempt triggers failover unless ``should_failover`` returns ``False``. + Cancellation is never caught. ``ResponseInvalidatedException`` and middleware control-flow exceptions always + propagate unchanged, because handing an invalidated provider response to another client could run tool calls + the provider already cancelled. + - A streaming attempt fails over only before it releases an update that carries content. Updates without + content (for example, response-created metadata) are held back until content arrives, so an attempt that fails + after only sending metadata is discarded. Once content has been released, a later error propagates. + - When the last client fails, its exception propagates unchanged. + + Conversation history must be client-managed because history stored by one service cannot be read by another. + Requests that set ``store=True`` or carry a service-managed continuation (such as ``conversation_id`` or + ``previous_response_id``) are rejected with ``ChatClientInvalidRequestException`` before the function-invocation + loop starts, so no approved tool runs for a request that cannot be served. Clients that store history by default + (``STORES_BY_DEFAULT``) receive ``store=False`` for their attempt unless the request sets ``store`` explicitly. + Agents using this client keep history locally, as they do for any client that does not store history by default. + + Reasoning is provider-specific: a provider can only replay reasoning it produced itself (for example, OpenAI + requires its encrypted reasoning payload). This client records on each reasoning content, in streamed and + non-streamed responses, which client produced it. That record is kept in the conversation history, so it also + applies to later runs. Reasoning produced by a different client is omitted from the copy of the conversation sent + to a client, together with the compaction flag that marks its group as containing reasoning. Function calls and + results are always sent, and the caller's transcript is never modified. Reasoning without a record (for example, + history created before this client was used) is sent unchanged. + + The chat clients are caller-owned: ``close()`` and ``async with`` do not close them. + + Examples: + .. code-block:: python + + from agent_framework import Agent, FailoverChatClient + from agent_framework.anthropic import AnthropicClient + from agent_framework.openai import OpenAIChatClient + + client = FailoverChatClient([ + OpenAIChatClient(model="gpt-5.4-mini"), + AnthropicClient(model="claude-sonnet-5-5"), + ]) + agent = Agent(client=client, instructions="You are a support assistant.", tools=[issue_refund]) + response = await agent.run("Refund order 123.") + """ + + def __init__( + self, + clients: Sequence[SupportsChatGetResponse[Any]], + *, + should_failover: FailoverPredicate | None = None, + middleware: Sequence[ChatAndFunctionMiddlewareTypes] | None = None, + function_invocation_configuration: FunctionInvocationConfiguration | None = None, + compaction_strategy: CompactionStrategy | None = None, + tokenizer: TokenizerProtocol | None = None, + additional_properties: dict[str, Any] | None = None, + ) -> None: + """Initialize a FailoverChatClient. + + Args: + clients: The chat clients to try, in order. The sequence is copied; the clients remain caller-owned. + + Keyword Args: + should_failover: Optional predicate called with the exception from a failed attempt. Return ``True`` to + try the next client or ``False`` to propagate the exception. It is not called for the last client, + for cancellation, for ``ResponseInvalidatedException``, or for middleware control-flow exceptions. + When omitted, every other ``Exception`` triggers failover. + middleware: Chat and function middleware for this client. Chat middleware runs once per model call + around all attempts; function middleware runs around tool executions. + function_invocation_configuration: Configuration for the function-invocation loop owned by this client. + compaction_strategy: Optional compaction strategy applied before each model call, before any attempt. + tokenizer: Optional tokenizer used by token-aware compaction strategies. + additional_properties: Additional properties for the client. + + Raises: + ValueError: If ``clients`` is empty or contains the same client more than once. + TypeError: If an entry does not provide ``get_response``. + """ + candidates = tuple(clients) + if not candidates: + raise ValueError("FailoverChatClient requires at least one chat client.") + if len({id(candidate) for candidate in candidates}) != len(candidates): + raise ValueError("FailoverChatClient requires distinct chat clients; the same client appears twice.") + for candidate in candidates: + if not callable(getattr(candidate, "get_response", None)): + raise TypeError( + f"FailoverChatClient clients must provide get_response(); got {type(candidate).__name__}." + ) + self._clients = candidates + self._should_failover = should_failover + + continuation_option_names = {"conversation_id"} + for candidate in candidates: + # A nested failover client must contribute its own candidates' names, not the inherited default. + declared = ( + candidate._continuation_option_names + if isinstance(candidate, FailoverChatClient) + else getattr(candidate, "SERVICE_CONTINUATION_OPTION_NAMES", ()) + ) + if isinstance(declared, Sequence) and not isinstance(declared, (str, bytes, bytearray)): + continuation_option_names.update( + name for name in cast(Sequence[Any], declared) if isinstance(name, str) + ) + self._continuation_option_names = tuple(sorted(continuation_option_names)) + + # Reasoning recorded with one of a candidate's sources is replayed to that candidate. A nested failover + # client records its own candidates' sources, so it accepts all of them. + candidate_sources: list[str | None] = [] + accepted_sources: list[frozenset[str]] = [] + for candidate in candidates: + if isinstance(candidate, FailoverChatClient): + candidate_sources.append(None) + accepted_sources.append(candidate._all_sources) + else: + source = _client_source(candidate) + candidate_sources.append(source) + accepted_sources.append(frozenset({source})) + self._candidate_sources = tuple(candidate_sources) + self._accepted_sources = tuple(accepted_sources) + self._all_sources: frozenset[str] = frozenset[str]().union(*accepted_sources) + super().__init__( + middleware=middleware, + function_invocation_configuration=function_invocation_configuration, + compaction_strategy=compaction_strategy, + tokenizer=tokenizer, + additional_properties=additional_properties, + ) + + @property + def clients(self) -> tuple[SupportsChatGetResponse[Any], ...]: + """The chat clients tried for each model call, in order.""" + return self._clients + + @overload + def get_response( + self, + messages: Sequence[Message], + *, + stream: Literal[False] = ..., + options: ChatOptions[ResponseModelBoundT], + middleware: Sequence[ChatAndFunctionMiddlewareTypes] | None = None, + compaction_strategy: CompactionStrategy | None = None, + tokenizer: TokenizerProtocol | None = None, + function_invocation_kwargs: Mapping[str, Any] | None = None, + client_kwargs: Mapping[str, Any] | None = None, + ) -> Awaitable[ChatResponse[ResponseModelBoundT]]: ... + + @overload + def get_response( + self, + messages: Sequence[Message], + *, + stream: Literal[False] = ..., + options: OptionsCoT | ChatOptions[None] | None = None, + middleware: Sequence[ChatAndFunctionMiddlewareTypes] | None = None, + compaction_strategy: CompactionStrategy | None = None, + tokenizer: TokenizerProtocol | None = None, + function_invocation_kwargs: Mapping[str, Any] | None = None, + client_kwargs: Mapping[str, Any] | None = None, + ) -> Awaitable[ChatResponse[Any]]: ... + + @overload + def get_response( + self, + messages: Sequence[Message], + *, + stream: Literal[True], + options: OptionsCoT | ChatOptions[Any] | None = None, + middleware: Sequence[ChatAndFunctionMiddlewareTypes] | None = None, + compaction_strategy: CompactionStrategy | None = None, + tokenizer: TokenizerProtocol | None = None, + function_invocation_kwargs: Mapping[str, Any] | None = None, + client_kwargs: Mapping[str, Any] | None = None, + ) -> ResponseStream[ChatResponseUpdate, ChatResponse[Any]]: ... + + def get_response( + self, + messages: Sequence[Message], + *, + stream: bool = False, + options: OptionsCoT | ChatOptions[Any] | None = None, + middleware: Sequence[ChatAndFunctionMiddlewareTypes] | None = None, + compaction_strategy: CompactionStrategy | None = None, + tokenizer: TokenizerProtocol | None = None, + function_invocation_kwargs: Mapping[str, Any] | None = None, + client_kwargs: Mapping[str, Any] | None = None, + ) -> Awaitable[ChatResponse[Any]] | ResponseStream[ChatResponseUpdate, ChatResponse[Any]]: + """Get a response, failing over to the next chat client for each model call that fails. + + Args: + messages: The messages to send. + + Keyword Args: + stream: Whether to stream the response. + options: Chat options forwarded to each attempted client. + middleware: Additional chat and function middleware for this request. + compaction_strategy: Optional per-call compaction override. + tokenizer: Optional per-call tokenizer override. + function_invocation_kwargs: Keyword arguments forwarded only to tool invocations. + client_kwargs: Additional client keyword arguments forwarded to each attempted client. + + Returns: + When ``stream`` is ``False``, an awaitable response; otherwise a response stream. + + Raises: + ChatClientInvalidRequestException: If ``options`` set ``store=True`` or carry a service-managed + continuation. This is raised before any approved tool runs or any client is called. + """ + # Reject service-managed history before the inherited loop resolves approvals, which can run tools + # without a model call. The per-model-call check in _inner_get_response still covers middleware changes. + self._ensure_client_managed_history(cast("Mapping[str, Any] | None", options) or {}) + forwarded_client_kwargs = dict(client_kwargs) if client_kwargs is not None else {} + if (session := forwarded_client_kwargs.get("session")) is not None: + # This client's chat middleware layer consumes "session"; carry it to the candidates' chat layers. + forwarded_client_kwargs[_CANDIDATE_SESSION_KEY] = session + get_response = cast("Callable[..., Any]", super().get_response) + return cast( + "Awaitable[ChatResponse[Any]] | ResponseStream[ChatResponseUpdate, ChatResponse[Any]]", + get_response( + messages, + stream=stream, + options=options, + middleware=middleware, + compaction_strategy=compaction_strategy, + tokenizer=tokenizer, + function_invocation_kwargs=function_invocation_kwargs, + client_kwargs=forwarded_client_kwargs, + ), + ) + + def _inner_get_response( + self, + *, + messages: Sequence[Message], + stream: bool, + options: Mapping[str, Any], + **kwargs: Any, + ) -> Awaitable[ChatResponse] | ResponseStream[ChatResponseUpdate, ChatResponse]: + client_kwargs = dict(kwargs) + if (session := client_kwargs.pop(_CANDIDATE_SESSION_KEY, None)) is not None: + client_kwargs["session"] = session + if stream: + return self._stream_with_failover(messages=messages, options=options, client_kwargs=client_kwargs) + return self._get_response_with_failover(messages=messages, options=options, client_kwargs=client_kwargs) + + async def _get_response_with_failover( + self, + *, + messages: Sequence[Message], + options: Mapping[str, Any], + client_kwargs: Mapping[str, Any], + ) -> ChatResponse: + self._ensure_client_managed_history(options) + last_index = len(self._clients) - 1 + for index, client in enumerate(self._clients): + try: + response = await cast( + "Awaitable[ChatResponse]", + self._call_client( + index, messages=messages, stream=False, options=options, client_kwargs=client_kwargs + ), + ) + except Exception as exception: + if index == last_index or not self._is_failover_allowed(exception): + raise + self._log_failover(index, client, exception) + continue + for message in response.messages: + self._record_reasoning_source(index, message.contents) + return response + raise AssertionError("unreachable") # pragma: no cover + + def _stream_with_failover( + self, + *, + messages: Sequence[Message], + options: Mapping[str, Any], + client_kwargs: Mapping[str, Any], + ) -> ResponseStream[ChatResponseUpdate, ChatResponse]: + response_format = options.get("response_format") + final_responses: list[ChatResponse] = [] + + async def _updates() -> AsyncIterator[ChatResponseUpdate]: + self._ensure_client_managed_history(options) + last_index = len(self._clients) - 1 + for index, client in enumerate(self._clients): + is_last = index == last_index + try: + attempt = await cast( + "ResponseStream[ChatResponseUpdate, ChatResponse]", + self._call_client( + index, messages=messages, stream=True, options=options, client_kwargs=client_kwargs + ), + ) + except Exception as exception: + if is_last or not self._is_failover_allowed(exception): + raise + self._log_failover(index, client, exception) + continue + + held: list[ChatResponseUpdate] = [] + released = False + failed = False + async with attempt: + iterator = attempt.__aiter__() + while True: + try: + update = await iterator.__anext__() + except StopAsyncIteration: + break + except Exception as exception: + if released or is_last or not self._is_failover_allowed(exception): + raise + self._log_failover(index, client, exception) + failed = True + break + # Record on the streamed contents too: callers rebuild streamed history from updates. + self._record_reasoning_source(index, update.contents) + if not released and not update.contents: + held.append(update) + continue + if not released: + released = True + for held_update in held: + yield held_update + held.clear() + yield update + if failed: + continue + # The attempt finalizes when its iteration ends, so a finalization error has already been + # handled above; this returns the finalized response. + final_response = await attempt.get_final_response() + for held_update in held: + yield held_update + for message in final_response.messages: + self._record_reasoning_source(index, message.contents) + final_responses.append(final_response) + return + + def _finalize(updates: Sequence[ChatResponseUpdate]) -> ChatResponse: + if final_responses: + return final_responses[-1] + # Reached only when a direct consumer of this stream stops before an attempt completes. + return ChatResponse.from_updates(updates, output_format_type=response_format) + + return ResponseStream(_updates(), finalizer=_finalize) + + def _call_client( + self, + index: int, + *, + messages: Sequence[Message], + stream: bool, + options: Mapping[str, Any], + client_kwargs: Mapping[str, Any], + ) -> Awaitable[ChatResponse[Any]] | ResponseStream[ChatResponseUpdate, ChatResponse[Any]]: + """Run one model call on the client at ``index`` without its function-invocation loop.""" + client = self._clients[index] + attempt_options = dict(options) + if getattr(client, "STORES_BY_DEFAULT", False) is True and attempt_options.get("store") is None: + attempt_options["store"] = False + attempt_client_kwargs = dict(client_kwargs) + if isinstance(client, FailoverChatClient) and (session := attempt_client_kwargs.get("session")) is not None: + # A nested failover client is entered below its own get_response, so hand it the session the same way. + attempt_client_kwargs[_CANDIDATE_SESSION_KEY] = session + target: Any = client + if isinstance(client, FunctionInvocationLayer): + # This is the same call a client's own function-invocation loop makes for each model call: it enters the + # layer below FunctionInvocationLayer, keeping the client's chat middleware and telemetry. + target = super(FunctionInvocationLayer, client) + get_response = cast( + "Callable[..., Awaitable[ChatResponse[Any]] | ResponseStream[ChatResponseUpdate, ChatResponse[Any]]]", + target.get_response, + ) + return get_response( + messages=self._messages_for_client(index, messages), + stream=stream, + options=attempt_options, + client_kwargs=attempt_client_kwargs, + ) + + def _record_reasoning_source(self, index: int, contents: Sequence[Content]) -> None: + """Record which client produced each reasoning content, keeping a nested client's record.""" + source = self._candidate_sources[index] + if source is None: + return + for content in contents: + if content.type == "text_reasoning": + content.additional_properties.setdefault(_FAILOVER_METADATA_KEY, {"source": source}) + + def _messages_for_client(self, index: int, messages: Sequence[Message]) -> Sequence[Message]: + """Return the conversation to send to the client at ``index``. + + Reasoning recorded as produced by a different client is omitted, along with the compaction flag that marks + its group as containing reasoning. The caller's messages are not modified; unchanged conversations are + returned as-is. + """ + accepted = self._accepted_sources[index] + projected: list[Message] | None = None + foreign_groups: set[Any] = set() + for position, message in enumerate(messages): + if not any(_is_foreign_reasoning(content, accepted) for content in message.contents): + continue + if projected is None: + projected = list(messages) + annotation = _group_annotation(message) + if annotation is not None and annotation.get(GROUP_ID_KEY) is not None: + foreign_groups.add(annotation[GROUP_ID_KEY]) + projected[position] = _without_reasoning(message, accepted) + if projected is None: + return messages + if foreign_groups: + for position, message in enumerate(projected): + annotation = _group_annotation(message) + if ( + annotation is not None + and annotation.get(GROUP_ID_KEY) in foreign_groups + and annotation.get(GROUP_HAS_REASONING_KEY) is True + ): + projected[position] = _without_reasoning(message, accepted) + return [message for message in projected if message.contents] + + def _ensure_client_managed_history(self, options: Mapping[str, Any]) -> None: + if options.get("store") is True: + raise ChatClientInvalidRequestException( + "FailoverChatClient requires client-managed history so another chat client can continue the " + "conversation; store=True is not supported. Leave store unset or set store=False." + ) + for name in self._continuation_option_names: + value = options.get(name) + if value is None or value == "": + continue + if name == "conversation_id" and isinstance(value, str) and is_local_history_conversation_id(value): + continue + raise ChatClientInvalidRequestException( + f"FailoverChatClient cannot continue a service-managed conversation ('{name}' is set) because the " + "other chat clients cannot read history stored by that service. Use client-managed history instead." + ) + + def _is_failover_allowed(self, exception: Exception) -> bool: + if isinstance(exception, (ResponseInvalidatedException, MiddlewareException)): + return False + if self._should_failover is None: + return True + return bool(self._should_failover(exception)) + + def _log_failover(self, index: int, client: SupportsChatGetResponse[Any], exception: Exception) -> None: + logger.warning( + "Chat client %d of %d (%s) failed with %s; trying the next chat client.", + index + 1, + len(self._clients), + type(client).__name__, + type(exception).__name__, + exc_info=exception, + ) diff --git a/python/packages/core/agent_framework/_feature_stage.py b/python/packages/core/agent_framework/_feature_stage.py index b4988db3c19..c1be125d2c7 100644 --- a/python/packages/core/agent_framework/_feature_stage.py +++ b/python/packages/core/agent_framework/_feature_stage.py @@ -52,6 +52,7 @@ class ExperimentalFeature(str, Enum): AGENT_TRIGGERED_COMPACTION = "AGENT_TRIGGERED_COMPACTION" AGENT_HOOKS = "AGENT_HOOKS" + CHAT_CLIENT_FAILOVER = "CHAT_CLIENT_FAILOVER" COMPUTER_USE = "COMPUTER_USE" DECLARATIVE_AGENTS = "DECLARATIVE_AGENTS" EVALS = "EVALS" diff --git a/python/packages/core/tests/core/test_failover.py b/python/packages/core/tests/core/test_failover.py new file mode 100644 index 00000000000..2ef74bfc5cb --- /dev/null +++ b/python/packages/core/tests/core/test_failover.py @@ -0,0 +1,798 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +from collections.abc import Awaitable, Mapping, Sequence +from typing import Any, ClassVar, cast + +import pytest + +from agent_framework import ( + Agent, + BaseChatClient, + ChatContext, + ChatMiddleware, + ChatMiddlewareLayer, + ChatOptions, + ChatResponse, + ChatResponseUpdate, + Content, + FailoverChatClient, + FunctionInvocationContext, + FunctionInvocationLayer, + FunctionMiddleware, + Message, + ResponseStream, + tool, +) +from agent_framework._middleware import MiddlewareException +from agent_framework.exceptions import ( + ChatClientException, + ChatClientInvalidRequestException, + ResponseInvalidatedException, +) + +pytestmark = pytest.mark.filterwarnings("ignore:.*CHAT_CLIENT_FAILOVER.*:FutureWarning") + +# A step is one scripted model call: a response, an exception, or (streaming) a list of updates and exceptions. +Step = ChatResponse | BaseException | Sequence[ChatResponseUpdate | BaseException] + + +class _RawScriptedClient(BaseChatClient[Any]): + """Chat client without middleware or a tool loop (like ``Raw*`` provider clients) and scripted model calls.""" + + def __init__( + self, + steps: Sequence[Step] = (), + *, + raise_on_call: BaseException | None = None, + finalize_error: Exception | None = None, + model: str | None = None, + ) -> None: + super().__init__() + self.model = model + self._steps = list(steps) + self._raise_on_call = raise_on_call + self._finalize_error = finalize_error + self.calls: list[dict[str, Any]] = [] + self.closed = False + + async def close(self) -> None: + self.closed = True + + def _inner_get_response( + self, *, messages: Sequence[Message], stream: bool, options: Mapping[str, Any], **kwargs: Any + ) -> Awaitable[ChatResponse] | ResponseStream[ChatResponseUpdate, ChatResponse]: + self.calls.append({"messages": list(messages), "options": dict(options)}) + if self._raise_on_call is not None: + raise self._raise_on_call + step: Step = self._steps.pop(0) if self._steps else _text_response("done") + + if stream: + items = _updates_for(step) if isinstance(step, ChatResponse) else step + + async def _updates(): + if isinstance(items, BaseException): + raise items + for item in items: + if isinstance(item, BaseException): + raise item + yield item + + if self._finalize_error is None: + return self._build_response_stream(_updates()) + finalize_error = self._finalize_error + + def _failing_finalizer(_: Sequence[ChatResponseUpdate]) -> ChatResponse: + raise finalize_error + + return ResponseStream(_updates(), finalizer=_failing_finalizer) + + async def _response() -> ChatResponse: + if isinstance(step, BaseException): + raise step + assert isinstance(step, ChatResponse) + return step + + return _response() + + +class _ScriptedClient(FunctionInvocationLayer[Any], ChatMiddlewareLayer[Any], _RawScriptedClient): + """Chat client with the same layering as shipped clients and scripted model calls.""" + + def __init__( + self, + steps: Sequence[Step] = (), + *, + raise_on_call: BaseException | None = None, + finalize_error: Exception | None = None, + middleware: Sequence[Any] | None = None, + model: str | None = None, + ) -> None: + super().__init__( + middleware=middleware, + steps=steps, + raise_on_call=raise_on_call, + finalize_error=finalize_error, + model=model, + ) + + +class _StoringScriptedClient(_ScriptedClient): + STORES_BY_DEFAULT: ClassVar[bool] = True + + +class _ResponsesLikeScriptedClient(_StoringScriptedClient): + SERVICE_CONTINUATION_OPTION_NAMES: ClassVar[tuple[str, ...]] = ("conversation_id", "previous_response_id") + + +def _text_response(text: str) -> ChatResponse: + return ChatResponse(messages=Message(role="assistant", contents=[text])) + + +def _tool_call_response(call_id: str = "call-1") -> ChatResponse: + call = Content.from_function_call(call_id=call_id, name="send_email", arguments={"to": "cfo@contoso.com"}) + return ChatResponse(messages=Message(role="assistant", contents=[call])) + + +def _updates_for(response: ChatResponse) -> list[ChatResponseUpdate]: + return [ChatResponseUpdate(contents=list(message.contents), role="assistant") for message in response.messages] + + +def _rate_limited(name: str) -> ChatClientException: + return ChatClientException(f"{name}: 429 Too Many Requests", log_level=None) + + +def _function_results(messages: Sequence[Message]) -> list[Content]: + return [content for message in messages for content in message.contents if content.type == "function_result"] + + +@pytest.fixture +def sent_emails() -> list[str]: + return [] + + +@pytest.fixture +def send_email(sent_emails: list[str]) -> Any: + @tool(approval_mode="never_require") + def send_email(to: str) -> str: + """Send an email.""" + sent_emails.append(to) + return f"sent to {to}" + + return send_email + + +async def test_failover_after_tool_call_does_not_run_the_tool_again(send_email: Any, sent_emails: list[str]) -> None: + primary = _ScriptedClient([_tool_call_response(), _rate_limited("primary")]) + fallback = _ScriptedClient([_text_response("done via fallback")]) + client = FailoverChatClient([primary, fallback]) + + response = await client.get_response( + [Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]} + ) + + assert response.text == "done via fallback" + assert sent_emails == ["cfo@contoso.com"] + assert len(primary.calls) == 2 + assert len(fallback.calls) == 1 + # The fallback continues the same model call, so it sees the tool result produced before the failure. + assert [result.call_id for result in _function_results(fallback.calls[0]["messages"])] == ["call-1"] + + +async def test_streaming_failover_after_tool_call_does_not_run_the_tool_again( + send_email: Any, sent_emails: list[str] +) -> None: + primary = _ScriptedClient([_updates_for(_tool_call_response()), _rate_limited("primary")]) + fallback = _ScriptedClient([_updates_for(_text_response("done via fallback"))]) + client = FailoverChatClient([primary, fallback]) + + stream = client.get_response( + [Message(role="user", contents=["Email the CFO."])], stream=True, options={"tools": [send_email]} + ) + async for _ in stream: + pass + response = await stream.get_final_response() + + assert response.text == "done via fallback" + assert sent_emails == ["cfo@contoso.com"] + assert len(fallback.calls) == 1 + assert [result.call_id for result in _function_results(fallback.calls[0]["messages"])] == ["call-1"] + + +async def test_each_model_call_starts_with_the_first_client(send_email: Any, sent_emails: list[str]) -> None: + primary = _ScriptedClient([_rate_limited("primary"), _text_response("primary recovered")]) + fallback = _ScriptedClient([_tool_call_response()]) + client = FailoverChatClient([primary, fallback]) + + response = await client.get_response( + [Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]} + ) + + assert response.text == "primary recovered" + assert sent_emails == ["cfo@contoso.com"] + assert len(primary.calls) == 2 + assert len(fallback.calls) == 1 + + +async def test_successful_first_client_does_not_call_the_others() -> None: + primary = _ScriptedClient([_text_response("primary")]) + fallback = _ScriptedClient() + client = FailoverChatClient([primary, fallback]) + + response = await client.get_response([Message(role="user", contents=["hi"])]) + + assert response.text == "primary" + assert fallback.calls == [] + + +async def test_last_client_error_propagates_unchanged() -> None: + last_error = _rate_limited("fallback") + primary = _ScriptedClient([_rate_limited("primary")]) + fallback = _ScriptedClient([last_error]) + client = FailoverChatClient([primary, fallback]) + + with pytest.raises(ChatClientException) as raised: + await client.get_response([Message(role="user", contents=["hi"])]) + + assert raised.value is last_error + assert len(primary.calls) == 1 + assert len(fallback.calls) == 1 + + +async def test_should_failover_can_stop_failover() -> None: + first_error = ValueError("bad request") + seen: list[Exception] = [] + + def should_failover(exception: Exception) -> bool: + seen.append(exception) + return False + + primary = _ScriptedClient([first_error]) + fallback = _ScriptedClient() + client = FailoverChatClient([primary, fallback], should_failover=should_failover) + + with pytest.raises(ValueError) as raised: + await client.get_response([Message(role="user", contents=["hi"])]) + + assert raised.value is first_error + assert seen == [first_error] + assert fallback.calls == [] + + +@pytest.mark.parametrize( + "error", + [ + ResponseInvalidatedException("provider invalidated the response", log_level=None), + MiddlewareException("middleware stopped the call", log_level=None), + ], + ids=["response-invalidated", "middleware-control-flow"], +) +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_protected_errors_never_fail_over(error: Exception, stream: bool) -> None: + primary = _ScriptedClient([error]) + fallback = _ScriptedClient() + client = FailoverChatClient([primary, fallback], should_failover=lambda _: True) + + with pytest.raises(type(error)) as raised: + if stream: + async for _ in client.get_response([Message(role="user", contents=["hi"])], stream=True): + pass + else: + await client.get_response([Message(role="user", contents=["hi"])]) + + assert raised.value is error + assert fallback.calls == [] + + +async def test_cancellation_is_not_failed_over() -> None: + primary = _ScriptedClient([asyncio.CancelledError()]) + fallback = _ScriptedClient() + client = FailoverChatClient([primary, fallback]) + + with pytest.raises(asyncio.CancelledError): + await client.get_response([Message(role="user", contents=["hi"])]) + + assert fallback.calls == [] + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_error_raised_when_starting_the_call_fails_over(stream: bool) -> None: + primary = _ScriptedClient(raise_on_call=_rate_limited("primary")) + fallback = _ScriptedClient([_text_response("fallback")]) + client = FailoverChatClient([primary, fallback]) + + if stream: + response_stream = client.get_response([Message(role="user", contents=["hi"])], stream=True) + async for _ in response_stream: + pass + response = await response_stream.get_final_response() + else: + response = await client.get_response([Message(role="user", contents=["hi"])]) + + assert response.text == "fallback" + + +async def test_streaming_error_after_content_is_not_failed_over() -> None: + error = _rate_limited("primary") + primary = _ScriptedClient([ + [ChatResponseUpdate(contents=[Content.from_text("Your refund")], role="assistant"), error] + ]) + fallback = _ScriptedClient() + client = FailoverChatClient([primary, fallback]) + + received: list[str] = [] + with pytest.raises(ChatClientException) as raised: + async for update in client.get_response([Message(role="user", contents=["hi"])], stream=True): + received.append(update.text) + + assert raised.value is error + assert received == ["Your refund"] + assert fallback.calls == [] + + +async def test_streaming_metadata_from_a_failed_attempt_is_discarded() -> None: + primary = _ScriptedClient([ + [ChatResponseUpdate(response_id="primary-response", role="assistant"), _rate_limited("p")] + ]) + fallback = _ScriptedClient([ + [ + ChatResponseUpdate(response_id="fallback-response", role="assistant"), + ChatResponseUpdate(contents=[Content.from_text("done")], response_id="fallback-response", role="assistant"), + ] + ]) + client = FailoverChatClient([primary, fallback]) + + stream = client.get_response([Message(role="user", contents=["hi"])], stream=True) + updates = [update async for update in stream] + response = await stream.get_final_response() + + assert {update.response_id for update in updates} == {"fallback-response"} + assert response.text == "done" + assert response.response_id == "fallback-response" + + +async def test_streaming_attempt_without_content_updates_succeeds() -> None: + primary = _ScriptedClient([[ChatResponseUpdate(response_id="primary-response", role="assistant")]]) + client = FailoverChatClient([primary, _ScriptedClient()]) + + stream = client.get_response([Message(role="user", contents=["hi"])], stream=True) + updates = [update async for update in stream] + + assert [update.response_id for update in updates] == ["primary-response"] + assert (await stream.get_final_response()).response_id == "primary-response" + + +@pytest.mark.parametrize( + "options", + [ + {"store": True}, + {"conversation_id": "conv_123"}, + {"previous_response_id": "resp_123"}, + ], + ids=["store-true", "service-conversation", "candidate-declared-continuation"], +) +async def test_service_managed_history_is_rejected_before_any_call(options: dict[str, Any]) -> None: + primary = _ResponsesLikeScriptedClient() + fallback = _ScriptedClient() + client = FailoverChatClient([primary, fallback]) + + with pytest.raises(ChatClientInvalidRequestException, match="client-managed history|service-managed"): + await client.get_response([Message(role="user", contents=["hi"])], options=cast("ChatOptions[Any]", options)) + + assert primary.calls == [] + assert fallback.calls == [] + + +async def test_clients_that_store_by_default_receive_store_false() -> None: + primary = _StoringScriptedClient([_rate_limited("primary")]) + fallback = _ScriptedClient([_text_response("fallback")]) + client = FailoverChatClient([primary, fallback]) + + await client.get_response([Message(role="user", contents=["hi"])], options={"temperature": 0.2}) + + assert primary.calls[0]["options"]["store"] is False + assert "store" not in fallback.calls[0]["options"] + assert fallback.calls[0]["options"]["temperature"] == 0.2 + + +async def test_attempts_use_client_chat_middleware_but_not_client_tool_loop( + send_email: Any, sent_emails: list[str] +) -> None: + attempts: list[str] = [] + client_tool_calls: list[str] = [] + failover_tool_calls: list[str] = [] + + class _RecordAttempt(ChatMiddleware): + def __init__(self, name: str) -> None: + self.name = name + + async def process(self, context: ChatContext, call_next: Any) -> None: + attempts.append(self.name) + await call_next() + + class _RecordToolCall(FunctionMiddleware): + def __init__(self, calls: list[str]) -> None: + self.calls = calls + + async def process(self, context: FunctionInvocationContext, call_next: Any) -> None: + self.calls.append(context.function.name) + await call_next() + + primary = _ScriptedClient( + [_tool_call_response(), _rate_limited("primary")], + middleware=[_RecordAttempt("primary"), _RecordToolCall(client_tool_calls)], + ) + fallback = _ScriptedClient([_text_response("done")], middleware=[_RecordAttempt("fallback")]) + client = FailoverChatClient( + [primary, fallback], middleware=[_RecordAttempt("failover"), _RecordToolCall(failover_tool_calls)] + ) + + await client.get_response([Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]}) + + # Failover-level chat middleware runs once per model call, around every attempt of that call. + assert attempts == ["failover", "primary", "failover", "primary", "fallback"] + assert client_tool_calls == [] + assert failover_tool_calls == ["send_email"] + assert sent_emails == ["cfo@contoso.com"] + + +async def test_agent_keeps_history_client_side_across_failover(send_email: Any, sent_emails: list[str]) -> None: + primary = _StoringScriptedClient([_tool_call_response(), _rate_limited("primary"), _text_response("second")]) + fallback = _ScriptedClient([_text_response("first")]) + agent = Agent(client=FailoverChatClient([primary, fallback]), tools=[send_email]) + session = agent.create_session() + + first = await agent.run("Email the CFO.", session=session) + second = await agent.run("Thanks!", session=session) + + assert first.text == "first" + assert second.text == "second" + assert sent_emails == ["cfo@contoso.com"] + # The second run reaches the primary with the whole conversation, including the turn the fallback answered. + second_run_texts = [message.text for message in primary.calls[-1]["messages"]] + assert "Email the CFO." in second_run_texts + assert "first" in second_run_texts + assert "Thanks!" in second_run_texts + assert all(call["options"]["store"] is False for call in primary.calls) + + +async def test_nested_failover_client_is_a_valid_candidate(send_email: Any, sent_emails: list[str]) -> None: + regional = FailoverChatClient([ + _ScriptedClient([_tool_call_response(), _rate_limited("east")]), + _ScriptedClient([_rate_limited("west")]), + ]) + other_provider = _ScriptedClient([_text_response("other provider")]) + client = FailoverChatClient([regional, other_provider]) + + response = await client.get_response( + [Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]} + ) + + assert response.text == "other provider" + assert sent_emails == ["cfo@contoso.com"] + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_clients_without_a_tool_loop_are_supported(stream: bool, send_email: Any, sent_emails: list[str]) -> None: + primary = _RawScriptedClient([_tool_call_response(), _rate_limited("primary")]) + fallback = _RawScriptedClient([_text_response("done via fallback")]) + client = FailoverChatClient([primary, fallback]) + messages = [Message(role="user", contents=["Email the CFO."])] + + if stream: + response_stream = client.get_response(messages, stream=True, options={"tools": [send_email]}) + async for _ in response_stream: + pass + response = await response_stream.get_final_response() + else: + response = await client.get_response(messages, options={"tools": [send_email]}) + + assert response.text == "done via fallback" + assert sent_emails == ["cfo@contoso.com"] + + +async def test_streaming_finalization_error_before_content_fails_over() -> None: + primary = _ScriptedClient( + [[ChatResponseUpdate(response_id="primary-response", role="assistant")]], + finalize_error=_rate_limited("primary"), + ) + fallback = _ScriptedClient([_text_response("fallback")]) + client = FailoverChatClient([primary, fallback]) + + stream = client.get_response([Message(role="user", contents=["hi"])], stream=True) + updates = [update async for update in stream] + + assert {update.response_id for update in updates} == {None} + assert (await stream.get_final_response()).text == "fallback" + + +async def test_streaming_error_opening_the_last_client_propagates() -> None: + last_error = _rate_limited("fallback") + client = FailoverChatClient([ + _ScriptedClient(raise_on_call=_rate_limited("primary")), + _ScriptedClient(raise_on_call=last_error), + ]) + + with pytest.raises(ChatClientException) as raised: + async for _ in client.get_response([Message(role="user", contents=["hi"])], stream=True): + pass + + assert raised.value is last_error + + +async def test_streaming_consumer_can_stop_early() -> None: + primary = _ScriptedClient([ + [ + ChatResponseUpdate(contents=[Content.from_text("first")], role="assistant"), + ChatResponseUpdate(contents=[Content.from_text(" second")], role="assistant"), + ] + ]) + client = FailoverChatClient([primary, _ScriptedClient()]) + + stream = client.get_response([Message(role="user", contents=["hi"])], stream=True) + async for _ in stream: + break + await stream.close() + + assert (await stream.get_final_response()).text == "first" + + +async def test_local_history_conversation_id_is_not_a_service_continuation() -> None: + from agent_framework._sessions import LOCAL_HISTORY_CONVERSATION_ID + + client = FailoverChatClient([_ScriptedClient([_text_response("ok")])]) + + response = await client.get_response( + [Message(role="user", contents=["hi"])], options={"conversation_id": LOCAL_HISTORY_CONVERSATION_ID} + ) + + assert response.text == "ok" + + +async def test_close_leaves_clients_open() -> None: + primary = _ScriptedClient() + fallback = _ScriptedClient() + + async with FailoverChatClient([primary, fallback]) as client: + assert client.clients == (primary, fallback) + + assert not primary.closed + assert not fallback.closed + + +def test_constructor_validates_clients() -> None: + primary = _ScriptedClient() + + with pytest.raises(ValueError, match="at least one"): + FailoverChatClient([]) + with pytest.raises(ValueError, match="distinct"): + FailoverChatClient([primary, primary]) + not_a_client: Any = object() + with pytest.raises(TypeError, match="get_response"): + FailoverChatClient([primary, not_a_client]) + + +def test_failover_client_is_experimental() -> None: + from agent_framework import ExperimentalFeature + + assert getattr(FailoverChatClient, "__feature_id__", None) == ExperimentalFeature.CHAT_CLIENT_FAILOVER.value + + +# region Review regressions + + +def _reasoning_tool_call_response(call_id: str = "call-1") -> ChatResponse: + # Plain-text thinking without a provider replay payload, as local providers such as Ollama emit it. + reasoning = Content.from_text_reasoning(text="I should email the CFO.") + call = Content.from_function_call(call_id=call_id, name="send_email", arguments={"to": "cfo@contoso.com"}) + return ChatResponse(messages=Message(role="assistant", contents=[reasoning, call])) + + +def _content_types(messages: Sequence[Message]) -> list[str]: + return [content.type for message in messages for content in message.contents] + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_approval_resume_rejects_service_managed_history_before_running_tools(stream: bool) -> None: + executed: list[str] = [] + + @tool(approval_mode="always_require") + def delete_records(table: str) -> str: + """Delete records.""" + executed.append(table) + return "deleted" + + call = Content.from_function_call(call_id="call-1", name="delete_records", arguments={"table": "orders"}) + primary = _ScriptedClient([ChatResponse(messages=Message(role="assistant", contents=[call]))]) + agent = Agent(client=FailoverChatClient([primary, _ScriptedClient()]), tools=[delete_records]) + session = agent.create_session() + first = await agent.run("Delete the orders table.", session=session) + approval = first.user_input_requests[0].to_function_approval_response(approved=True) + + with pytest.raises(ChatClientInvalidRequestException, match="client-managed history"): + if stream: + await agent.run(approval, session=session, stream=True, options={"store": True}).get_final_response() + else: + await agent.run(approval, session=session, options={"store": True}) + + assert executed == [] + + +async def test_nested_failover_rejects_inner_continuation_options_before_any_call() -> None: + inner_primary = _ResponsesLikeScriptedClient() + inner_fallback = _ScriptedClient() + outer_fallback = _ScriptedClient() + client = FailoverChatClient([FailoverChatClient([inner_primary, inner_fallback]), outer_fallback]) + + with pytest.raises(ChatClientInvalidRequestException, match="previous_response_id"): + await client.get_response( + [Message(role="user", contents=["hi"])], + options=cast("ChatOptions[Any]", {"previous_response_id": "resp_123"}), + ) + + assert inner_primary.calls == [] + assert inner_fallback.calls == [] + assert outer_fallback.calls == [] + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_candidate_chat_middleware_receives_the_caller_session(stream: bool) -> None: + seen_sessions: list[Any] = [] + + class _RecordSession(ChatMiddleware): + async def process(self, context: ChatContext, call_next: Any) -> None: + seen_sessions.append(context.session) + await call_next() + + primary = _ScriptedClient([_rate_limited("primary")], middleware=[_RecordSession()]) + fallback = _ScriptedClient([_text_response("ok")], middleware=[_RecordSession()]) + agent = Agent(client=FailoverChatClient([primary, fallback])) + session = agent.create_session() + + if stream: + await agent.run("hi", session=session, stream=True).get_final_response() + else: + await agent.run("hi", session=session) + + assert seen_sessions == [session, session] + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_nested_failover_candidates_receive_the_caller_session(stream: bool) -> None: + seen_sessions: list[Any] = [] + + class _RecordSession(ChatMiddleware): + async def process(self, context: ChatContext, call_next: Any) -> None: + seen_sessions.append(context.session) + await call_next() + + regional = FailoverChatClient([ + _ScriptedClient([_rate_limited("east")], middleware=[_RecordSession()]), + _ScriptedClient([_text_response("ok")], middleware=[_RecordSession()]), + ]) + agent = Agent(client=FailoverChatClient([regional, _ScriptedClient()])) + session = agent.create_session() + + if stream: + await agent.run("hi", session=session, stream=True).get_final_response() + else: + await agent.run("hi", session=session) + + assert seen_sessions == [session, session] + + +async def test_reasoning_from_another_client_is_not_sent_to_the_fallback( + send_email: Any, sent_emails: list[str] +) -> None: + primary = _ScriptedClient([_reasoning_tool_call_response(), _rate_limited("primary")], model="local-model") + fallback = _ScriptedClient([_text_response("done")], model="hosted-model") + client = FailoverChatClient([primary, fallback]) + + response = await client.get_response( + [Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]} + ) + + assert response.text == "done" + assert sent_emails == ["cfo@contoso.com"] + # The client that produced the reasoning still receives it on its next model call. + assert "text_reasoning" in _content_types(primary.calls[1]["messages"]) + # A different client receives the tool call and result, but not reasoning it cannot replay. + fallback_types = _content_types(fallback.calls[0]["messages"]) + assert "text_reasoning" not in fallback_types + assert {"function_call", "function_result"} <= set(fallback_types) + # The caller-visible transcript is unchanged. + assert "text_reasoning" in _content_types(response.messages) + + +@pytest.mark.parametrize("first_run_stream", [False, True], ids=["non-streamed-history", "streamed-history"]) +async def test_reasoning_recorded_in_an_earlier_run_is_not_sent_to_another_client( + first_run_stream: bool, send_email: Any, sent_emails: list[str] +) -> None: + primary = _ScriptedClient( + [_reasoning_tool_call_response(), _text_response("first answer"), _rate_limited("primary")], + model="local-model", + ) + fallback = _ScriptedClient([_text_response("second answer")], model="hosted-model") + agent = Agent(client=FailoverChatClient([primary, fallback]), tools=[send_email]) + session = agent.create_session() + + if first_run_stream: + await agent.run("Email the CFO.", session=session, stream=True).get_final_response() + else: + await agent.run("Email the CFO.", session=session) + second = await agent.run("Thanks.", session=session) + + assert second.text == "second answer" + assert sent_emails == ["cfo@contoso.com"] + fallback_types = _content_types(fallback.calls[0]["messages"]) + assert "text_reasoning" not in fallback_types + assert {"function_call", "function_result"} <= set(fallback_types) + + +async def test_nested_failover_keeps_reasoning_for_the_client_that_produced_it( + send_email: Any, sent_emails: list[str] +) -> None: + producer = _ScriptedClient([_reasoning_tool_call_response(), _text_response("producer answered")], model="a") + regional = FailoverChatClient([producer, _ScriptedClient(model="b")]) + client = FailoverChatClient([regional, _ScriptedClient(model="c")]) + + response = await client.get_response( + [Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]} + ) + + assert response.text == "producer answered" + assert sent_emails == ["cfo@contoso.com"] + assert "text_reasoning" in _content_types(producer.calls[1]["messages"]) + + +@pytest.mark.parametrize("annotate", [False, True], ids=["plain-history", "compaction-annotated-history"]) +async def test_openai_fallback_accepts_history_with_another_providers_reasoning( + annotate: bool, send_email: Any, sent_emails: list[str] +) -> None: + pytest.importorskip("agent_framework_openai") + from unittest.mock import MagicMock, patch + + from openai.types.responses import Response + + from agent_framework import CharacterEstimatorTokenizer + from agent_framework.openai import OpenAIChatClient + + sdk_response = Response.model_validate({ + "id": "resp_fallback", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-test", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Done.", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + }) + # The client reads responses through the SDK's raw-response wrapper, as in the OpenAI package tests. + raw_response = MagicMock(headers={}) + raw_response.parse.return_value = sdk_response + + fallback = OpenAIChatClient(model="gpt-test", api_key="test-key") + primary = _ScriptedClient([_reasoning_tool_call_response(), _rate_limited("primary")], model="local-model") + client = FailoverChatClient([primary, fallback], tokenizer=CharacterEstimatorTokenizer() if annotate else None) + + with patch.object(fallback.client.responses, "create", return_value=raw_response) as create: + response = await client.get_response( + [Message(role="user", contents=["Email the CFO."])], options={"tools": [send_email]} + ) + + assert response.text == "Done." + assert sent_emails == ["cfo@contoso.com"] + # The real OpenAI request preparation accepted the conversation and sent it to the service once. + assert create.call_count == 1 + sent_item_types = [item.get("type") for item in create.call_args.kwargs["input"]] + assert "reasoning" not in sent_item_types + assert {"function_call", "function_call_output"} <= set(sent_item_types) diff --git a/python/samples/02-agents/chat_client/README.md b/python/samples/02-agents/chat_client/README.md index abb80337130..fa6eddc8251 100644 --- a/python/samples/02-agents/chat_client/README.md +++ b/python/samples/02-agents/chat_client/README.md @@ -7,6 +7,7 @@ This folder contains examples for direct chat client usage patterns. | File | Description | |------|-------------| | [`built_in_chat_clients.py`](built_in_chat_clients.py) | Consolidated sample for built-in chat clients. Passes a client name to `main()`, which creates the selected client with `get_client()`. | +| [`chat_client_failover.py`](chat_client_failover.py) | Demonstrates `FailoverChatClient`, which retries each model call on the next chat client when one fails, without re-running tools that already ran. Uses `FoundryChatClient` with a deliberately invalid primary deployment so the failover is visible. | | [`chat_response_cancellation.py`](chat_response_cancellation.py) | Demonstrates how to cancel chat responses during streaming, showing proper cancellation handling and cleanup. | | [`custom_chat_client.py`](custom_chat_client.py) | Demonstrates how to create custom chat clients by extending the `BaseChatClient` class. Shows a `EchoingChatClient` implementation and how to integrate it with `Agent` using the `as_agent()` method. | | [`require_per_service_call_history_persistence.py`](require_per_service_call_history_persistence.py) | Compares two otherwise identical `FoundryChatClient` agents with `store=False`; the only difference is whether `require_per_service_call_history_persistence` is enabled, and only the run without it stores the synthesized tool result when middleware terminates the loop early. | @@ -36,7 +37,7 @@ Example: uv run samples/02-agents/chat_client/built_in_chat_clients.py ``` -The `require_per_service_call_history_persistence.py` sample uses `FoundryChatClient`, so set the usual Foundry settings first and sign in with the Azure CLI: +The `require_per_service_call_history_persistence.py` and `chat_client_failover.py` samples use `FoundryChatClient`, so set the usual Foundry settings first and sign in with the Azure CLI: ```bash export FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" diff --git a/python/samples/02-agents/chat_client/chat_client_failover.py b/python/samples/02-agents/chat_client/chat_client_failover.py new file mode 100644 index 00000000000..0907d5e0681 --- /dev/null +++ b/python/samples/02-agents/chat_client/chat_client_failover.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import logging +import os +from typing import Annotated + +from agent_framework import Agent, FailoverChatClient, tool +from agent_framework.foundry import FoundryChatClient +from azure.identity.aio import AzureCliCredential +from dotenv import load_dotenv +from pydantic import Field + +# Load environment variables from .env file +load_dotenv() + +""" +Chat Client Failover Example + +This sample sends each model call to an ordered list of chat clients with `FailoverChatClient`. When a +client fails (for example with a 429, a 5xx, or a regional outage), the same model call is retried on the +next client. Failover happens per model call inside the tool loop, so tools that already ran are not run +again, and the next client sees the tool results produced so far. + +To show failover without waiting for a real outage, the primary client points at a deployment name that does +not exist, so every call to it fails. In production the clients would be, for example, a deployment in a +second region or a model from another provider. + +Conversation history stays on the client side: `FailoverChatClient` sends `store=False` to clients that store +history in the service by default, because history stored by one service cannot be read by another. + +Environment variables: + FOUNDRY_PROJECT_ENDPOINT - Microsoft Foundry project endpoint. + FOUNDRY_MODEL - Model deployment name used by the fallback client, for example `gpt-4o`. +""" + +refunds_issued: list[str] = [] + + +# NOTE: approval_mode="never_require" is for sample brevity. +@tool(approval_mode="never_require") +def issue_refund(order_id: Annotated[str, Field(description="The order to refund.")]) -> str: + """Issue a refund for an order.""" + refunds_issued.append(order_id) + return f"Refund issued for order {order_id}." + + +async def main() -> None: + # Show the warning that FailoverChatClient logs for each failover. + logging.basicConfig(level=logging.WARNING, format="%(levelname)s %(message)s") + + project_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + + # FailoverChatClient does not close the clients it is given, so the sample owns the credential and both + # clients and closes them on success, failure, or cancellation. + async with ( + AzureCliCredential() as credential, + # 1. The primary client fails on every call because the deployment does not exist. + FoundryChatClient( + project_endpoint=project_endpoint, + model="deployment-that-does-not-exist", + credential=credential, + ) as primary, + # 2. The fallback client uses a working deployment. + FoundryChatClient( + project_endpoint=project_endpoint, + model=os.environ["FOUNDRY_MODEL"], + credential=credential, + ) as fallback, + ): + # 3. Tools, function middleware, and agent middleware are configured once, on the failover client or agent. + agent = Agent( + client=FailoverChatClient([primary, fallback]), + name="SupportAgent", + instructions="You are a customer support agent. Use the tools you have to resolve requests.", + tools=[issue_refund], + ) + + response = await agent.run("Please refund order 123 and confirm when it is done.") + print(f"Agent: {response.text}") + print(f"Refunds issued: {refunds_issued}") + + +if __name__ == "__main__": + asyncio.run(main()) + +""" +Sample output (abridged): + +WARNING Chat client 1 of 2 (FoundryChatClient) failed with ...; trying the next chat client. +Agent: I've issued a refund for order 123. +Refunds issued: ['123'] +"""