Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 47 additions & 17 deletions python/packages/core/agent_framework/_evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
from ._tools import FunctionTool, normalize_tools
from ._types import (
_EVALUATION_TOOLS_METADATA_KEY, # pyright: ignore[reportPrivateUsage]
LOOP_INJECTED_MESSAGE_PROPERTY,
AgentResponse,
Message,
)
Expand Down Expand Up @@ -158,11 +159,26 @@ class ExpectedToolCall:
arguments: dict[str, Any] | None = None


def _is_loop_injected(message: Message) -> bool:
"""Whether ``message`` was injected by the agent loop instead of sent by the user.

``AgentLoopMiddleware`` injects ``user``-role messages (nudges, progress summaries) between
iterations of a single agent run. They are part of the agent's own transcript, so they must not
be treated as the start of a new conversation turn.
"""
properties = message.additional_properties
return bool(properties and properties.get(LOOP_INJECTED_MESSAGE_PROPERTY))


def _split_last_turn(conversation: list[Message]) -> tuple[list[Message], list[Message]]:
"""Split at the last user message (default strategy)."""
"""Split at the last user message (default strategy).

Loop-injected messages are skipped: in a loop-aggregated conversation they are nudges between
iterations of one run, not real user turns.
"""
last_user_idx = -1
for i, msg in enumerate(conversation):
if msg.role == "user":
if msg.role == "user" and not _is_loop_injected(msg):
last_user_idx = i
if last_user_idx >= 0:
return conversation[: last_user_idx + 1], conversation[last_user_idx + 1 :]
Expand All @@ -172,7 +188,7 @@ def _split_last_turn(conversation: list[Message]) -> tuple[list[Message], list[M
def _split_full(conversation: list[Message]) -> tuple[list[Message], list[Message]]:
"""Split after the first user message (evaluates whole trajectory)."""
for i, msg in enumerate(conversation):
if msg.role == "user":
if msg.role == "user" and not _is_loop_injected(msg):
return conversation[: i + 1], conversation[i + 1 :]
return [], list(conversation)

Expand Down Expand Up @@ -265,7 +281,8 @@ def per_turn_items(
) -> list[EvalItem]:
"""Split a multi-turn conversation into one ``EvalItem`` per turn.

Each user message starts a new turn. The resulting ``EvalItem``
Each user message starts a new turn. Loop-injected messages (nudges between iterations of
one agent run) are not user turns and do not start a new item. The resulting ``EvalItem``
has cumulative context: ``query_messages`` contains the full
conversation up to and including that user message, and
``response_messages`` contains the agent's actions up to the next
Expand All @@ -280,7 +297,7 @@ def per_turn_items(
Returns:
A list of ``EvalItem`` instances, one per user turn.
"""
user_indices = [i for i, m in enumerate(conversation) if m.role == "user"]
user_indices = [i for i, m in enumerate(conversation) if m.role == "user" and not _is_loop_injected(m)]
if not user_indices:
return []

Expand Down Expand Up @@ -1100,12 +1117,24 @@ def _check(item: EvalItem) -> CheckResult:
return _check


def _response_messages(item: EvalItem) -> list[Message]:
"""Return the response side of ``item``'s conversation split.

These are the messages that produced ``item.response`` — the agent's
actions up to the next user message. Tool checks inspect them so that a
tool call made in an earlier turn does not satisfy a later turn's check.
"""
_, response_msgs = item.split_messages()
return response_msgs


@experimental(feature_id=ExperimentalFeature.EVALS)
def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") -> EvalCheck:
"""Check that specific tools were called during the conversation.
"""Check that specific tools were called in the response.

Inspects the conversation history for ``tool_calls`` entries matching
the expected tool names.
Inspects the response side of the conversation split for ``tool_calls``
entries matching the expected tool names. A tool call made in an earlier
turn does not count towards a later turn's check.

Args:
*tool_names: Names of tools that should have been called.
Expand All @@ -1125,7 +1154,7 @@ def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") ->
def _check(item: EvalItem) -> CheckResult:
expected = set(tool_names)
called: set[str] = set()
for msg in item.conversation:
for msg in _response_messages(item):

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tang-ontheway, this response-side scan treats the last role="user" message as the turn boundary, but AgentLoopMiddleware injects user-role nudge/progress messages between iterations in its default aggregated response. A tool call from an earlier loop iteration then falls on the query side and tool_called_check / expected-tool-call checks report it missing. Please distinguish real conversation turns from loop-injected messages (or scope this only to explicit multi-turn items) and add a loop-aggregated regression.

for c in msg.contents or []:
if c.type == "function_call" and c.name:
called.add(c.name)
Expand Down Expand Up @@ -1164,9 +1193,9 @@ def _check(item: EvalItem) -> CheckResult:


def _extract_tool_calls(item: EvalItem) -> list[tuple[str, dict[str, Any] | None]]:
"""Extract (name, arguments) pairs from the conversation's function calls."""
"""Extract (name, arguments) pairs from the response's function calls."""
calls: list[tuple[str, dict[str, Any] | None]] = []
for msg in item.conversation:
for msg in _response_messages(item):
for c in msg.contents or []:
if c.type == "function_call" and c.name:
args: dict[str, Any] | None = None
Expand Down Expand Up @@ -1194,8 +1223,9 @@ def tool_calls_present(item: EvalItem) -> CheckResult:
"""Check that all expected tool calls were made (unordered, extras OK).

Uses ``item.expected_tool_calls`` — checks that every expected tool name
appears at least once in the conversation. Does not check arguments or
ordering. Extra (unexpected) tool calls are not penalized.
appears at least once in the response side of the conversation split.
Does not check arguments or ordering. Extra (unexpected) tool calls are
not penalized.

Example:

Expand Down Expand Up @@ -1235,10 +1265,10 @@ def tool_calls_present(item: EvalItem) -> CheckResult:
def tool_call_args_match(item: EvalItem) -> CheckResult:
"""Check that expected tool calls match on name and arguments.

For each expected tool call, finds matching calls in the conversation
by name. If ``ExpectedToolCall.arguments`` is provided, checks that
the actual arguments contain all expected key-value pairs (subset
match — extra actual arguments are OK).
For each expected tool call, finds matching calls in the response side of
the conversation split by name. If ``ExpectedToolCall.arguments`` is
provided, checks that the actual arguments contain all expected key-value
pairs (subset match — extra actual arguments are OK).

Example:

Expand Down
23 changes: 23 additions & 0 deletions python/packages/core/agent_framework/_harness/_loop.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
from .._middleware import AgentContext, AgentMiddleware, MiddlewareTermination
from .._sessions import SessionContext
from .._types import (
LOOP_INJECTED_MESSAGE_PROPERTY,
AgentResponse,
AgentResponseUpdate,
AgentRunInputs,
Expand Down Expand Up @@ -91,6 +92,22 @@
)


def _mark_loop_injected(messages: Sequence[Message]) -> None:
"""Tag loop-injected messages in place.

The loop injects ``user``-role messages (nudges, progress summaries) between iterations of a
single agent run. They are part of the agent's own transcript, not real user turns, so they
carry :data:`LOOP_INJECTED_MESSAGE_PROPERTY` in ``additional_properties`` to let consumers tell
them apart.
"""
for message in messages:
properties = message.additional_properties
if properties is None:
message.additional_properties = {LOOP_INJECTED_MESSAGE_PROPERTY: True}
else:
properties[LOOP_INJECTED_MESSAGE_PROPERTY] = True


def _render_criteria_block(criteria: Sequence[str] | None) -> str:
"""Render a list of criteria into a bullet block for the judge instructions (``""`` if none)."""
if not criteria:
Expand Down Expand Up @@ -801,6 +818,7 @@ def _message_to_update(message: Message) -> AgentResponseUpdate:
role=message.role,
author_name=message.author_name,
message_id=message.message_id,
additional_properties={LOOP_INJECTED_MESSAGE_PROPERTY: True},
)

@staticmethod
Expand Down Expand Up @@ -861,6 +879,11 @@ async def _resolve_next_message(
entries = progress if (session is None or self.fresh_context) else progress[-1:]
progress_msg = self._render_progress(entries)

# Mark the injected messages so consumers can tell them apart from real user turns.
_mark_loop_injected(next_msgs)
if progress_msg is not None:
_mark_loop_injected([progress_msg])

if self.fresh_context:
result = list(original_messages)
if progress_msg is not None:
Expand Down
11 changes: 11 additions & 0 deletions python/packages/core/agent_framework/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -2378,6 +2378,17 @@ def _combine_annotations(
"""


LOOP_INJECTED_MESSAGE_PROPERTY: Final[str] = "agent_framework_loop_injected"
"""Key in ``Message.additional_properties`` marking a loop-injected message.

``AgentLoopMiddleware`` injects ``user``-role messages (nudges, progress
summaries) between iterations of a single agent run. Those messages are not
real conversation turns, so consumers that reason about turn boundaries — for
example the evaluation checks that scope tool calls to the response side of a
conversation — must not treat them as the start of a new turn.
"""


# region Message


Expand Down
37 changes: 37 additions & 0 deletions python/packages/core/tests/core/local_eval_helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Copyright (c) Microsoft. All rights reserved.

"""Helpers for the local evaluation tests."""

from agent_framework import Content, Message
from agent_framework._types import LOOP_INJECTED_MESSAGE_PROPERTY


def two_turn_conversation() -> list[Message]:
"""Return a two-turn conversation whose first turn contains a tool call."""
return [
Message("user", ["Weather in NYC?"]),
Message("assistant", [Content.from_function_call("c1", "get_weather", arguments={"location": "NYC"})]),
Message("assistant", ["Sunny."]),
Message("user", ["And tomorrow?"]),
Message("assistant", ["No idea."]),
]


def loop_aggregated_conversation() -> list[Message]:
"""Return a conversation shaped like a loop-aggregated agent run.

The agent loop injects a ``user``-role nudge between iterations of a single run. Such a
message is not a real user turn, so the tool call made in the first iteration still belongs to
the response side of the split.
"""
nudge = Message(
"user",
["Continue working on the task."],
additional_properties={LOOP_INJECTED_MESSAGE_PROPERTY: True},
)
return [
Message("user", ["Weather in NYC?"]),
Message("assistant", [Content.from_function_call("c1", "get_weather", arguments={"location": "NYC"})]),
nudge,
Message("assistant", ["Sunny."]),
]
30 changes: 30 additions & 0 deletions python/packages/core/tests/core/test_harness_loop.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
DEFAULT_NEXT_MESSAGE,
AgentLoopMiddleware,
)
from agent_framework._types import LOOP_INJECTED_MESSAGE_PROPERTY

from .conftest import MockBaseChatClient

Expand Down Expand Up @@ -1332,6 +1333,35 @@ async def test_streaming_injects_nudge_messages_as_user_updates() -> None:
assert any(DEFAULT_NEXT_MESSAGE in text for text in user_texts)


async def test_non_streaming_marks_loop_injected_messages() -> None:
client = RecordingChatClient(texts=["first answer", "second answer"])
agent = Agent(
client=client,
middleware=[AgentLoopMiddleware(always_continue, max_iterations=2, inject_progress=False)],
)

response = await agent.run("start")

assert isinstance(response, AgentResponse)
injected = [m for m in response.messages if m.role == "user"]
assert len(injected) == 1
properties = injected[0].additional_properties or {}
assert properties[LOOP_INJECTED_MESSAGE_PROPERTY] is True


async def test_streaming_marks_loop_injected_updates() -> None:
client = RecordingChatClient(texts=["first", "second"])
agent = Agent(client=client, middleware=[AgentLoopMiddleware(always_continue, max_iterations=2)])

stream = agent.run("go", stream=True)
updates = [update async for update in stream]
await stream.get_final_response()

user_updates = [u for u in updates if u.role == "user"]
assert user_updates
assert all((u.additional_properties or {})[LOOP_INJECTED_MESSAGE_PROPERTY] is True for u in user_updates)


async def test_streaming_stops_at_max_iterations() -> None:
client = RecordingChatClient()
agent = Agent(client=client, middleware=[AgentLoopMiddleware(always_continue, max_iterations=2)])
Expand Down
Loading
Loading