Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
Sequence,
)
from contextlib import AbstractAsyncContextManager, AsyncExitStack, aclosing, suppress
from copy import copy
from copy import copy, deepcopy
from dataclasses import asdict, dataclass, is_dataclass
from typing import TYPE_CHECKING, Any, Generic, Literal, TypeGuard, TypeVar, cast
from urllib.parse import urlparse
Expand Down Expand Up @@ -2546,15 +2546,18 @@ def __init__(
) -> None:
self._stream = stream
self._allowed_oauth_consent_origins = allowed_oauth_consent_origins
self._usage_details: UsageDetails | None = None
# Usage per (author_name, response_id) stream. A workflow sends several agents' updates
# through one tracker, so a "replace" snapshot may only supersede usage from its own stream.
# Usage restored from a checkpoint is kept under the ``None`` key.
self._usage_by_stream: dict[tuple[str | None, str | None] | None, UsageDetails] = {}
persisted_usage = _hosted_internal_metadata(stream).get(_HOSTED_PROVIDER_USAGE_KEY)
if persisted_usage is not None:
if not isinstance(persisted_usage, Mapping) or any(
not isinstance(key, str) or (value is not None and type(value) is not int)
for key, value in cast(Mapping[object, object], persisted_usage).items()
):
raise RuntimeError("The persisted provider usage is invalid.")
self._usage_details = cast(UsageDetails, dict(cast(Mapping[str, int | None], persisted_usage)))
self._usage_by_stream[None] = cast(UsageDetails, dict(cast(Mapping[str, int | None], persisted_usage)))
self._active_type: str | None = None
self._active_id: str | None = None
# message_id of the update that opened the active text item, used to detect a new
Expand Down Expand Up @@ -2636,26 +2639,30 @@ def __init__(
@property
def usage_details(self) -> UsageDetails | None:
"""Return usage retained with the provider output checkpoint."""
return self._usage_details
usage_details: UsageDetails | None = None
for stream_usage in self._usage_by_stream.values():
usage_details = add_usage_details(usage_details, stream_usage)
return usage_details

@property
def usage(self) -> ResponseUsage | None:
"""Return accumulated usage in the Responses API schema."""
if self._usage_details is None:
usage_details = self.usage_details
if usage_details is None:
return None

input_tokens = int(self._usage_details.get("input_token_count") or 0)
output_tokens = int(self._usage_details.get("output_token_count") or 0)
total_tokens = self._usage_details.get("total_token_count")
input_tokens = int(usage_details.get("input_token_count") or 0)
output_tokens = int(usage_details.get("output_token_count") or 0)
total_tokens = usage_details.get("total_token_count")
return ResponseUsage(
input_tokens=input_tokens,
input_tokens_details=ResponseUsageInputTokensDetails(
cached_tokens=int(self._usage_details.get("cache_read_input_token_count") or 0),
cache_write_tokens=int(self._usage_details.get("cache_creation_input_token_count") or 0),
cached_tokens=int(usage_details.get("cache_read_input_token_count") or 0),
cache_write_tokens=int(usage_details.get("cache_creation_input_token_count") or 0),
),
output_tokens=output_tokens,
output_tokens_details=ResponseUsageOutputTokensDetails(
reasoning_tokens=int(self._usage_details.get("reasoning_output_token_count") or 0)
reasoning_tokens=int(usage_details.get("reasoning_output_token_count") or 0)
),
total_tokens=int(total_tokens) if total_tokens is not None else input_tokens + output_tokens,
)
Expand Down Expand Up @@ -2725,6 +2732,7 @@ async def handle_update(
message_id=update.message_id,
approval_storage=approval_storage,
start_new_content_part=complete_message,
usage_stream=(update.author_name, update.response_id),
):
yield event

Expand All @@ -2735,6 +2743,7 @@ async def handle(
*,
approval_storage: FunctionApprovalStore | None = None,
start_new_content_part: bool = False,
usage_stream: tuple[str | None, str | None] = (None, None),
) -> AsyncGenerator[ResponseStreamEvent]:
"""Process a content item, yielding its events.

Expand All @@ -2748,6 +2757,8 @@ async def handle(
(anything not recognized as a streaming delta type) to save/load approval requests.
start_new_content_part: Whether text and refusal content start a new independent
content part instead of continuing the current streaming occurrence.
usage_stream: The ``(author_name, response_id)`` of the update ``content`` came from.
A ``replace`` usage snapshot only supersedes earlier usage from the same stream.
"""
if _is_refusal_text_content(content) and content.text is not None:
for event in self._ensure_message_content("refusal", message_id, start_new_part=start_new_content_part):
Expand Down Expand Up @@ -3052,7 +3063,17 @@ async def handle(
yield builder.emit_done(oauth_item)

elif content.type == "usage":
self._usage_details = add_usage_details(self._usage_details, content.usage_details)
# A "replace" update is a cumulative snapshot of its own stream, so it supersedes that
# stream's earlier usage instead of adding to it. Other streams keep their usage.
if content.merge_mode == "replace":
if content.usage_details is None:
self._usage_by_stream.pop(usage_stream, None)
else:
self._usage_by_stream[usage_stream] = deepcopy(content.usage_details)
else:
self._usage_by_stream[usage_stream] = add_usage_details(
self._usage_by_stream.get(usage_stream), content.usage_details
)

else:
for event in self._close():
Expand Down
148 changes: 147 additions & 1 deletion python/packages/foundry_hosting/tests/test_responses.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
ServiceSessionId,
SessionStore,
SupportsAgentRun,
UsageDetails,
WorkflowAgent,
WorkflowBuilder,
WorkflowContext,
Expand Down Expand Up @@ -979,6 +980,56 @@ async def test_output_item_tracker_keeps_mixed_text_and_refusal_in_one_message()
]


async def test_output_item_tracker_replace_usage_is_copied() -> None:
stream = ResponseEventStream(response_id="resp_usage")
stream.emit_created()
stream.emit_in_progress()
tracker = _OutputItemTracker(stream)
usage = Content.from_usage(
{"input_token_count": 250, "output_token_count": 25, "total_token_count": 275},
merge_mode="replace",
)

async for _ in tracker.handle(usage):
pass
assert usage.usage_details is not None
usage.usage_details["input_token_count"] = 999

assert tracker.usage_details == {"input_token_count": 250, "output_token_count": 25, "total_token_count": 275}


async def test_output_item_tracker_scopes_replace_usage_to_its_stream() -> None:
stream = ResponseEventStream(response_id="resp_usage")
stream.emit_created()
stream.emit_in_progress()
tracker = _OutputItemTracker(stream)

def usage_update(author_name: str, response_id: str, input_tokens: int) -> AgentResponseUpdate:
return AgentResponseUpdate(
contents=[
Content.from_usage(
{"input_token_count": input_tokens, "output_token_count": input_tokens // 10},
merge_mode="replace",
)
],
role="assistant",
author_name=author_name,
response_id=response_id,
)

for update in (
usage_update("agent-a", "resp_a", 400),
usage_update("agent-a", "resp_a", 1000),
usage_update("agent-b", "resp_b", 50),
# The same agent run again is a new response stream, so its usage adds up.
usage_update("agent-a", "resp_a2", 20),
):
async for _ in tracker.handle_update(update):
pass

assert tracker.usage_details == {"input_token_count": 1070, "output_token_count": 107}


async def test_item_to_message_marks_refusal_text() -> None:
message = await _item_to_message(
cast(
Expand Down Expand Up @@ -5482,6 +5533,59 @@ async def test_usage_is_aggregated_in_completed_response(self, caplog: pytest.Lo
}
assert "Content type 'usage' is not supported yet" not in caplog.text

@pytest.mark.parametrize(
("merge_modes", "expected"),
[
pytest.param(
("replace", "replace"),
{"input_tokens": 250, "output_tokens": 25, "total_tokens": 275},
id="replace-snapshots",
),
pytest.param(
("append", "append"),
{"input_tokens": 350, "output_tokens": 35, "total_tokens": 385},
id="append-increments",
),
pytest.param(
("append", "replace"),
{"input_tokens": 250, "output_tokens": 25, "total_tokens": 275},
id="replace-after-append",
),
],
)
async def test_usage_honors_merge_mode(
self,
merge_modes: tuple[Literal["append", "replace"], Literal["append", "replace"]],
expected: dict[str, int],
) -> None:
agent = _make_agent(
stream_updates=[
AgentResponseUpdate(contents=[Content.from_text("Hello")], role="assistant"),
*(
AgentResponseUpdate(
contents=[
Content.from_usage(
{
"input_token_count": input_tokens,
"output_token_count": input_tokens // 10,
"total_token_count": input_tokens + input_tokens // 10,
},
merge_mode=merge_mode,
)
],
role="assistant",
)
for input_tokens, merge_mode in zip((100, 250), merge_modes, strict=True)
),
]
)

resp = await _post(_make_server(agent), stream=True)

assert resp.status_code == 200
completed = _parse_sse_events(resp.text)[-1]["data"]["response"]
assert {key: completed["usage"][key] for key in expected} == expected

async def test_function_call_streaming(self) -> None:
agent = _make_agent(
stream_updates=[
Expand Down Expand Up @@ -9635,12 +9739,20 @@ async def start(messages: list[Message], ctx: WorkflowContext[AgentExecutorReque
class _MultiUpdateWorkflowAgentMock(SupportsAgentRun):
"""Inner agent that streams one update per text in a single ``run`` call, and tracks ``run_count``."""

def __init__(self, name: str, texts: Sequence[str], *, gate: asyncio.Event | None = None) -> None:
def __init__(
self,
name: str,
texts: Sequence[str],
*,
gate: asyncio.Event | None = None,
usage_snapshots: Sequence[UsageDetails] = (),
) -> None:
self.id = str(uuid.uuid4())
self.name = name
self.description: str | None = None
self._texts = list(texts)
self._gate = gate
self._usage_snapshots = list(usage_snapshots)
self.run_count = 0
self.started = asyncio.Event()
self.cancelled = asyncio.Event()
Expand Down Expand Up @@ -9687,6 +9799,7 @@ def run(
texts = self._texts
name = self.name
gate = self._gate
usage_snapshots = self._usage_snapshots

async def _aiter() -> AsyncIterator[AgentResponseUpdate]:
self.started.set()
Expand All @@ -9702,6 +9815,12 @@ async def _aiter() -> AsyncIterator[AgentResponseUpdate]:
role="assistant",
author_name=name,
)
for usage in usage_snapshots:
yield AgentResponseUpdate(
contents=[Content.from_usage(usage, merge_mode="replace")],
role="assistant",
author_name=name,
)

return ResponseStream(_aiter(), finalizer=AgentResponse.from_updates)

Expand Down Expand Up @@ -9773,6 +9892,33 @@ class TestWorkflowAgentHosting:
relative to the regular agent path.
"""

async def test_replace_usage_from_each_executor_is_summed(self) -> None:
agent_a = _MultiUpdateWorkflowAgentMock(
"agent-a",
["from a"],
usage_snapshots=[
UsageDetails(input_token_count=400, output_token_count=40),
UsageDetails(input_token_count=1000, output_token_count=100),
],
)
agent_b = _MultiUpdateWorkflowAgentMock(
"agent-b",
["from b"],
usage_snapshots=[UsageDetails(input_token_count=50, output_token_count=5)],
)
workflow = WorkflowBuilder(start_executor=agent_a).add_edge(agent_a, agent_b).build()

resp = await _post(_make_server(WorkflowAgent(workflow=workflow, name="Usage Workflow Agent")), stream=True)

assert resp.status_code == 200
assert [agent_a.run_count, agent_b.run_count] == [1, 1]
completed = _parse_sse_events(resp.text)[-1]["data"]["response"]
assert {key: completed["usage"][key] for key in ("input_tokens", "output_tokens", "total_tokens")} == {
"input_tokens": 1050,
"output_tokens": 105,
"total_tokens": 1155,
}

async def test_async_factory_creates_workflow_agent_for_each_request(self) -> None:
created: list[tuple[WorkflowAgent, _MultiUpdateWorkflowAgentMock]] = []

Expand Down
Loading