diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 787cac17bd3..03be1b3e940 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -58,6 +58,7 @@ from ._tools import FunctionTool, normalize_tools from ._types import ( _EVALUATION_TOOLS_METADATA_KEY, # pyright: ignore[reportPrivateUsage] + LOOP_INJECTED_MESSAGE_PROPERTY, AgentResponse, Message, ) @@ -158,11 +159,26 @@ class ExpectedToolCall: arguments: dict[str, Any] | None = None +def _is_loop_injected(message: Message) -> bool: + """Whether ``message`` was injected by the agent loop instead of sent by the user. + + ``AgentLoopMiddleware`` injects ``user``-role messages (nudges, progress summaries) between + iterations of a single agent run. They are part of the agent's own transcript, so they must not + be treated as the start of a new conversation turn. + """ + properties = message.additional_properties + return bool(properties and properties.get(LOOP_INJECTED_MESSAGE_PROPERTY)) + + def _split_last_turn(conversation: list[Message]) -> tuple[list[Message], list[Message]]: - """Split at the last user message (default strategy).""" + """Split at the last user message (default strategy). + + Loop-injected messages are skipped: in a loop-aggregated conversation they are nudges between + iterations of one run, not real user turns. + """ last_user_idx = -1 for i, msg in enumerate(conversation): - if msg.role == "user": + if msg.role == "user" and not _is_loop_injected(msg): last_user_idx = i if last_user_idx >= 0: return conversation[: last_user_idx + 1], conversation[last_user_idx + 1 :] @@ -172,7 +188,7 @@ def _split_last_turn(conversation: list[Message]) -> tuple[list[Message], list[M def _split_full(conversation: list[Message]) -> tuple[list[Message], list[Message]]: """Split after the first user message (evaluates whole trajectory).""" for i, msg in enumerate(conversation): - if msg.role == "user": + if msg.role == "user" and not _is_loop_injected(msg): return conversation[: i + 1], conversation[i + 1 :] return [], list(conversation) @@ -265,7 +281,8 @@ def per_turn_items( ) -> list[EvalItem]: """Split a multi-turn conversation into one ``EvalItem`` per turn. - Each user message starts a new turn. The resulting ``EvalItem`` + Each user message starts a new turn. Loop-injected messages (nudges between iterations of + one agent run) are not user turns and do not start a new item. The resulting ``EvalItem`` has cumulative context: ``query_messages`` contains the full conversation up to and including that user message, and ``response_messages`` contains the agent's actions up to the next @@ -280,7 +297,7 @@ def per_turn_items( Returns: A list of ``EvalItem`` instances, one per user turn. """ - user_indices = [i for i, m in enumerate(conversation) if m.role == "user"] + user_indices = [i for i, m in enumerate(conversation) if m.role == "user" and not _is_loop_injected(m)] if not user_indices: return [] @@ -1100,12 +1117,24 @@ def _check(item: EvalItem) -> CheckResult: return _check +def _response_messages(item: EvalItem) -> list[Message]: + """Return the response side of ``item``'s conversation split. + + These are the messages that produced ``item.response`` — the agent's + actions up to the next user message. Tool checks inspect them so that a + tool call made in an earlier turn does not satisfy a later turn's check. + """ + _, response_msgs = item.split_messages() + return response_msgs + + @experimental(feature_id=ExperimentalFeature.EVALS) def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") -> EvalCheck: - """Check that specific tools were called during the conversation. + """Check that specific tools were called in the response. - Inspects the conversation history for ``tool_calls`` entries matching - the expected tool names. + Inspects the response side of the conversation split for ``tool_calls`` + entries matching the expected tool names. A tool call made in an earlier + turn does not count towards a later turn's check. Args: *tool_names: Names of tools that should have been called. @@ -1125,7 +1154,7 @@ def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") -> def _check(item: EvalItem) -> CheckResult: expected = set(tool_names) called: set[str] = set() - for msg in item.conversation: + for msg in _response_messages(item): for c in msg.contents or []: if c.type == "function_call" and c.name: called.add(c.name) @@ -1164,9 +1193,9 @@ def _check(item: EvalItem) -> CheckResult: def _extract_tool_calls(item: EvalItem) -> list[tuple[str, dict[str, Any] | None]]: - """Extract (name, arguments) pairs from the conversation's function calls.""" + """Extract (name, arguments) pairs from the response's function calls.""" calls: list[tuple[str, dict[str, Any] | None]] = [] - for msg in item.conversation: + for msg in _response_messages(item): for c in msg.contents or []: if c.type == "function_call" and c.name: args: dict[str, Any] | None = None @@ -1194,8 +1223,9 @@ def tool_calls_present(item: EvalItem) -> CheckResult: """Check that all expected tool calls were made (unordered, extras OK). Uses ``item.expected_tool_calls`` — checks that every expected tool name - appears at least once in the conversation. Does not check arguments or - ordering. Extra (unexpected) tool calls are not penalized. + appears at least once in the response side of the conversation split. + Does not check arguments or ordering. Extra (unexpected) tool calls are + not penalized. Example: @@ -1235,10 +1265,10 @@ def tool_calls_present(item: EvalItem) -> CheckResult: def tool_call_args_match(item: EvalItem) -> CheckResult: """Check that expected tool calls match on name and arguments. - For each expected tool call, finds matching calls in the conversation - by name. If ``ExpectedToolCall.arguments`` is provided, checks that - the actual arguments contain all expected key-value pairs (subset - match — extra actual arguments are OK). + For each expected tool call, finds matching calls in the response side of + the conversation split by name. If ``ExpectedToolCall.arguments`` is + provided, checks that the actual arguments contain all expected key-value + pairs (subset match — extra actual arguments are OK). Example: diff --git a/python/packages/core/agent_framework/_harness/_loop.py b/python/packages/core/agent_framework/_harness/_loop.py index 308f06dd0c5..7b42f335c16 100644 --- a/python/packages/core/agent_framework/_harness/_loop.py +++ b/python/packages/core/agent_framework/_harness/_loop.py @@ -41,6 +41,7 @@ from .._middleware import AgentContext, AgentMiddleware, MiddlewareTermination from .._sessions import SessionContext from .._types import ( + LOOP_INJECTED_MESSAGE_PROPERTY, AgentResponse, AgentResponseUpdate, AgentRunInputs, @@ -91,6 +92,22 @@ ) +def _mark_loop_injected(messages: Sequence[Message]) -> None: + """Tag loop-injected messages in place. + + The loop injects ``user``-role messages (nudges, progress summaries) between iterations of a + single agent run. They are part of the agent's own transcript, not real user turns, so they + carry :data:`LOOP_INJECTED_MESSAGE_PROPERTY` in ``additional_properties`` to let consumers tell + them apart. + """ + for message in messages: + properties = message.additional_properties + if properties is None: + message.additional_properties = {LOOP_INJECTED_MESSAGE_PROPERTY: True} + else: + properties[LOOP_INJECTED_MESSAGE_PROPERTY] = True + + def _render_criteria_block(criteria: Sequence[str] | None) -> str: """Render a list of criteria into a bullet block for the judge instructions (``""`` if none).""" if not criteria: @@ -801,6 +818,7 @@ def _message_to_update(message: Message) -> AgentResponseUpdate: role=message.role, author_name=message.author_name, message_id=message.message_id, + additional_properties={LOOP_INJECTED_MESSAGE_PROPERTY: True}, ) @staticmethod @@ -861,6 +879,11 @@ async def _resolve_next_message( entries = progress if (session is None or self.fresh_context) else progress[-1:] progress_msg = self._render_progress(entries) + # Mark the injected messages so consumers can tell them apart from real user turns. + _mark_loop_injected(next_msgs) + if progress_msg is not None: + _mark_loop_injected([progress_msg]) + if self.fresh_context: result = list(original_messages) if progress_msg is not None: diff --git a/python/packages/core/agent_framework/_types.py b/python/packages/core/agent_framework/_types.py index a41195f6df8..f1630df369a 100644 --- a/python/packages/core/agent_framework/_types.py +++ b/python/packages/core/agent_framework/_types.py @@ -2378,6 +2378,17 @@ def _combine_annotations( """ +LOOP_INJECTED_MESSAGE_PROPERTY: Final[str] = "agent_framework_loop_injected" +"""Key in ``Message.additional_properties`` marking a loop-injected message. + +``AgentLoopMiddleware`` injects ``user``-role messages (nudges, progress +summaries) between iterations of a single agent run. Those messages are not +real conversation turns, so consumers that reason about turn boundaries — for +example the evaluation checks that scope tool calls to the response side of a +conversation — must not treat them as the start of a new turn. +""" + + # region Message diff --git a/python/packages/core/tests/core/local_eval_helpers.py b/python/packages/core/tests/core/local_eval_helpers.py new file mode 100644 index 00000000000..484b5a0e72e --- /dev/null +++ b/python/packages/core/tests/core/local_eval_helpers.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Helpers for the local evaluation tests.""" + +from agent_framework import Content, Message +from agent_framework._types import LOOP_INJECTED_MESSAGE_PROPERTY + + +def two_turn_conversation() -> list[Message]: + """Return a two-turn conversation whose first turn contains a tool call.""" + return [ + Message("user", ["Weather in NYC?"]), + Message("assistant", [Content.from_function_call("c1", "get_weather", arguments={"location": "NYC"})]), + Message("assistant", ["Sunny."]), + Message("user", ["And tomorrow?"]), + Message("assistant", ["No idea."]), + ] + + +def loop_aggregated_conversation() -> list[Message]: + """Return a conversation shaped like a loop-aggregated agent run. + + The agent loop injects a ``user``-role nudge between iterations of a single run. Such a + message is not a real user turn, so the tool call made in the first iteration still belongs to + the response side of the split. + """ + nudge = Message( + "user", + ["Continue working on the task."], + additional_properties={LOOP_INJECTED_MESSAGE_PROPERTY: True}, + ) + return [ + Message("user", ["Weather in NYC?"]), + Message("assistant", [Content.from_function_call("c1", "get_weather", arguments={"location": "NYC"})]), + nudge, + Message("assistant", ["Sunny."]), + ] diff --git a/python/packages/core/tests/core/test_harness_loop.py b/python/packages/core/tests/core/test_harness_loop.py index 1ad4663027f..72ee6731c38 100644 --- a/python/packages/core/tests/core/test_harness_loop.py +++ b/python/packages/core/tests/core/test_harness_loop.py @@ -43,6 +43,7 @@ DEFAULT_NEXT_MESSAGE, AgentLoopMiddleware, ) +from agent_framework._types import LOOP_INJECTED_MESSAGE_PROPERTY from .conftest import MockBaseChatClient @@ -1332,6 +1333,35 @@ async def test_streaming_injects_nudge_messages_as_user_updates() -> None: assert any(DEFAULT_NEXT_MESSAGE in text for text in user_texts) +async def test_non_streaming_marks_loop_injected_messages() -> None: + client = RecordingChatClient(texts=["first answer", "second answer"]) + agent = Agent( + client=client, + middleware=[AgentLoopMiddleware(always_continue, max_iterations=2, inject_progress=False)], + ) + + response = await agent.run("start") + + assert isinstance(response, AgentResponse) + injected = [m for m in response.messages if m.role == "user"] + assert len(injected) == 1 + properties = injected[0].additional_properties or {} + assert properties[LOOP_INJECTED_MESSAGE_PROPERTY] is True + + +async def test_streaming_marks_loop_injected_updates() -> None: + client = RecordingChatClient(texts=["first", "second"]) + agent = Agent(client=client, middleware=[AgentLoopMiddleware(always_continue, max_iterations=2)]) + + stream = agent.run("go", stream=True) + updates = [update async for update in stream] + await stream.get_final_response() + + user_updates = [u for u in updates if u.role == "user"] + assert user_updates + assert all((u.additional_properties or {})[LOOP_INJECTED_MESSAGE_PROPERTY] is True for u in user_updates) + + async def test_streaming_stops_at_max_iterations() -> None: client = RecordingChatClient() agent = Agent(client=client, middleware=[AgentLoopMiddleware(always_continue, max_iterations=2)]) diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index 06f39d634a6..490fca85ede 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -12,6 +12,7 @@ from agent_framework._evaluation import ( CheckResult, + ConversationSplit, EvalItem, EvalItemResult, EvalNotPassedError, @@ -29,6 +30,8 @@ ) from agent_framework._types import Content, Message +from .local_eval_helpers import loop_aggregated_conversation, two_turn_conversation + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -1381,3 +1384,119 @@ def test_recursion_into_sub_results(self) -> None: with pytest.raises(EvalNotPassedError) as exc: parent.assert_no_failed_items() assert "sub-bad:fail" in str(exc.value) + + +# --------------------------------------------------------------------------- +# Tool checks follow the conversation split +# --------------------------------------------------------------------------- + + +class TestToolChecksUseSplitResponse: + """Tool checks must inspect the response side of the conversation split. + + A tool call made in an earlier turn must not satisfy a later turn's + tool check. See issue #10091. + """ + + def test_tool_called_check_ignores_earlier_turn(self): + items = EvalItem.per_turn_items(two_turn_conversation()) + second = items[1] + assert second.response == "No idea." + + result = tool_called_check("get_weather")(second) + assert result.passed is False # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert "get_weather" in result.reason # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + + def test_tool_called_check_still_sees_current_turn(self): + items = EvalItem.per_turn_items(two_turn_conversation()) + first = items[0] + + result = tool_called_check("get_weather")(first) + assert result.passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + + def test_tool_calls_present_ignores_earlier_turn(self): + items = EvalItem.per_turn_items(two_turn_conversation()) + second = items[1] + second.expected_tool_calls = [ExpectedToolCall("get_weather")] + + result = tool_calls_present(second) + assert result.passed is False # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert "get_weather" in result.reason # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + + def test_tool_call_args_match_ignores_earlier_turn(self): + items = EvalItem.per_turn_items(two_turn_conversation()) + second = items[1] + second.expected_tool_calls = [ExpectedToolCall("get_weather", {"location": "NYC"})] + + result = tool_call_args_match(second) + assert result.passed is False # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert "not called" in result.reason # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + + def test_full_split_still_counts_whole_trajectory(self): + items = EvalItem.per_turn_items(two_turn_conversation()) + second = items[1] + second.split_strategy = ConversationSplit.FULL + second.expected_tool_calls = [ExpectedToolCall("get_weather")] + + result = tool_calls_present(second) + assert result.passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + + def test_single_turn_conversation_is_unchanged(self): + conversation = [ + Message("user", ["Weather in NYC?"]), + Message("assistant", [Content.from_function_call("c1", "get_weather", arguments={"location": "NYC"})]), + Message("assistant", ["Sunny."]), + ] + item = EvalItem(conversation=conversation) + item.expected_tool_calls = [ExpectedToolCall("get_weather", {"location": "NYC"})] + + assert tool_called_check("get_weather")(item).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_calls_present(item).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + assert tool_call_args_match(item).passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + + +class TestToolChecksIgnoreLoopInjectedTurns: + """A loop-injected nudge is not a real user turn. + + ``AgentLoopMiddleware`` injects ``user``-role messages between iterations of a single run. In + the aggregated response those messages sit between the iterations, so a tool call made in an + earlier iteration must still be visible to the checks. + """ + + def test_tool_called_check_sees_call_from_earlier_loop_iteration(self): + item = EvalItem(conversation=loop_aggregated_conversation()) + + result = tool_called_check("get_weather")(item) + assert result.passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + + def test_tool_calls_present_sees_call_from_earlier_loop_iteration(self): + item = EvalItem(conversation=loop_aggregated_conversation()) + item.expected_tool_calls = [ExpectedToolCall("get_weather", {"location": "NYC"})] + + result = tool_calls_present(item) + assert result.passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + + def test_tool_call_args_match_sees_call_from_earlier_loop_iteration(self): + item = EvalItem(conversation=loop_aggregated_conversation()) + item.expected_tool_calls = [ExpectedToolCall("get_weather", {"location": "NYC"})] + + result = tool_call_args_match(item) + assert result.passed is True # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + + def test_loop_injected_nudge_is_not_a_turn(self): + items = EvalItem.per_turn_items(loop_aggregated_conversation()) + + assert len(items) == 1 + assert items[0].query == "Weather in NYC?" + + def test_real_user_turn_after_loop_injection_still_splits(self): + conversation = loop_aggregated_conversation() + conversation.append(Message("user", ["And tomorrow?"])) + conversation.append(Message("assistant", ["No idea."])) + + items = EvalItem.per_turn_items(conversation) + + assert len(items) == 2 + assert items[1].response == "No idea." + result = tool_called_check("get_weather")(items[1]) + assert result.passed is False # type: ignore[union-attr] # ty: ignore[unresolved-attribute]