From 4ebf4844518fe6b7cefdf84d98380514e291f3fa Mon Sep 17 00:00:00 2001 From: Mahdi-Hasan <43575088+Mahdi-Hasan@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:18:22 +0600 Subject: [PATCH 1/4] Python: Honor replace usage snapshots in Foundry hosting _OutputItemTracker added every usage update with add_usage_details, even when the update had merge_mode="replace". Two cumulative snapshots of 100/10 and 250/25 were reported as 350/35. Replace the tracked usage for replace updates and keep adding append updates, the same rule as Content._add_usage_content and AgentResponse.from_updates. Fixes #10189 Co-Authored-By: Claude Opus 5.5 --- .../_responses.py | 8 ++- .../foundry_hosting/tests/test_responses.py | 49 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py index 02b6993d50e..7f027ecfb48 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py @@ -22,7 +22,7 @@ Sequence, ) from contextlib import AbstractAsyncContextManager, AsyncExitStack, aclosing, suppress -from copy import copy +from copy import copy, deepcopy from dataclasses import asdict, dataclass, is_dataclass from typing import TYPE_CHECKING, Any, Generic, Literal, TypeGuard, TypeVar, cast from urllib.parse import urlparse @@ -3052,7 +3052,11 @@ async def handle( yield builder.emit_done(oauth_item) elif content.type == "usage": - self._usage_details = add_usage_details(self._usage_details, content.usage_details) + # A "replace" update is a cumulative snapshot, so it supersedes earlier usage instead of adding to it. + if content.merge_mode == "replace": + self._usage_details = deepcopy(content.usage_details) + else: + self._usage_details = add_usage_details(self._usage_details, content.usage_details) else: for event in self._close(): diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index b605b27a625..5491e2d28b2 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -5482,6 +5482,55 @@ async def test_usage_is_aggregated_in_completed_response(self, caplog: pytest.Lo } assert "Content type 'usage' is not supported yet" not in caplog.text + @pytest.mark.parametrize( + ("merge_modes", "expected"), + [ + pytest.param( + ("replace", "replace"), + {"input_tokens": 250, "output_tokens": 25, "total_tokens": 275}, + id="replace-snapshots", + ), + pytest.param( + ("append", "append"), + {"input_tokens": 350, "output_tokens": 35, "total_tokens": 385}, + id="append-increments", + ), + pytest.param( + ("append", "replace"), + {"input_tokens": 250, "output_tokens": 25, "total_tokens": 275}, + id="replace-after-append", + ), + ], + ) + async def test_usage_honors_merge_mode(self, merge_modes: tuple[str, str], expected: dict[str, int]) -> None: + agent = _make_agent( + stream_updates=[ + AgentResponseUpdate(contents=[Content.from_text("Hello")], role="assistant"), + *( + AgentResponseUpdate( + contents=[ + Content.from_usage( + { + "input_token_count": input_tokens, + "output_token_count": input_tokens // 10, + "total_token_count": input_tokens + input_tokens // 10, + }, + merge_mode=merge_mode, + ) + ], + role="assistant", + ) + for input_tokens, merge_mode in zip((100, 250), merge_modes, strict=True) + ), + ] + ) + + resp = await _post(_make_server(agent), stream=True) + + assert resp.status_code == 200 + completed = _parse_sse_events(resp.text)[-1]["data"]["response"] + assert {key: completed["usage"][key] for key in expected} == expected + async def test_function_call_streaming(self) -> None: agent = _make_agent( stream_updates=[ From 8bf00365da481c2c85faf1fe74bd82ab98675842 Mon Sep 17 00:00:00 2001 From: Mahdi-Hasan <43575088+Mahdi-Hasan@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:12:12 +0600 Subject: [PATCH 2/4] Python: Test that replace usage is copied by the output tracker The merge-mode test only checked totals, so it would still pass if the tracker kept the update's usage_details by reference. Mutate the update after handle() and check the tracker's usage is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../foundry_hosting/tests/test_responses.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index 5491e2d28b2..f44a6ff8603 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -979,6 +979,24 @@ async def test_output_item_tracker_keeps_mixed_text_and_refusal_in_one_message() ] +async def test_output_item_tracker_replace_usage_is_copied() -> None: + stream = ResponseEventStream(response_id="resp_usage") + stream.emit_created() + stream.emit_in_progress() + tracker = _OutputItemTracker(stream) + usage = Content.from_usage( + {"input_token_count": 250, "output_token_count": 25, "total_token_count": 275}, + merge_mode="replace", + ) + + async for _ in tracker.handle(usage): + pass + assert usage.usage_details is not None + usage.usage_details["input_token_count"] = 999 + + assert tracker.usage_details == {"input_token_count": 250, "output_token_count": 25, "total_token_count": 275} + + async def test_item_to_message_marks_refusal_text() -> None: message = await _item_to_message( cast( From d68981f802fa697e65d24b17a67b9c51629225de Mon Sep 17 00:00:00 2001 From: Mahdi-Hasan <43575088+Mahdi-Hasan@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:33:39 +0600 Subject: [PATCH 3/4] Python: Scope replace usage snapshots to their own stream A hosted workflow sends every executor's updates through one _OutputItemTracker. A replace snapshot from a later agent erased usage already tracked for earlier agents: 1000/100 then 50/5 gave 50/5 instead of 1050/105. Track usage per (author_name, response_id) stream. A replace update overwrites only its own stream and the reported usage is the sum over all streams. Add a tracker test and a two-executor workflow regression. Co-Authored-By: Claude Opus 5.5 --- .../_responses.py | 43 +++++++---- .../foundry_hosting/tests/test_responses.py | 77 ++++++++++++++++++- 2 files changed, 106 insertions(+), 14 deletions(-) diff --git a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py index 7f027ecfb48..5542ec1ad03 100644 --- a/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py +++ b/python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py @@ -2546,7 +2546,10 @@ def __init__( ) -> None: self._stream = stream self._allowed_oauth_consent_origins = allowed_oauth_consent_origins - self._usage_details: UsageDetails | None = None + # Usage per (author_name, response_id) stream. A workflow sends several agents' updates + # through one tracker, so a "replace" snapshot may only supersede usage from its own stream. + # Usage restored from a checkpoint is kept under the ``None`` key. + self._usage_by_stream: dict[tuple[str | None, str | None] | None, UsageDetails] = {} persisted_usage = _hosted_internal_metadata(stream).get(_HOSTED_PROVIDER_USAGE_KEY) if persisted_usage is not None: if not isinstance(persisted_usage, Mapping) or any( @@ -2554,7 +2557,7 @@ def __init__( for key, value in cast(Mapping[object, object], persisted_usage).items() ): raise RuntimeError("The persisted provider usage is invalid.") - self._usage_details = cast(UsageDetails, dict(cast(Mapping[str, int | None], persisted_usage))) + self._usage_by_stream[None] = cast(UsageDetails, dict(cast(Mapping[str, int | None], persisted_usage))) self._active_type: str | None = None self._active_id: str | None = None # message_id of the update that opened the active text item, used to detect a new @@ -2636,26 +2639,30 @@ def __init__( @property def usage_details(self) -> UsageDetails | None: """Return usage retained with the provider output checkpoint.""" - return self._usage_details + usage_details: UsageDetails | None = None + for stream_usage in self._usage_by_stream.values(): + usage_details = add_usage_details(usage_details, stream_usage) + return usage_details @property def usage(self) -> ResponseUsage | None: """Return accumulated usage in the Responses API schema.""" - if self._usage_details is None: + usage_details = self.usage_details + if usage_details is None: return None - input_tokens = int(self._usage_details.get("input_token_count") or 0) - output_tokens = int(self._usage_details.get("output_token_count") or 0) - total_tokens = self._usage_details.get("total_token_count") + input_tokens = int(usage_details.get("input_token_count") or 0) + output_tokens = int(usage_details.get("output_token_count") or 0) + total_tokens = usage_details.get("total_token_count") return ResponseUsage( input_tokens=input_tokens, input_tokens_details=ResponseUsageInputTokensDetails( - cached_tokens=int(self._usage_details.get("cache_read_input_token_count") or 0), - cache_write_tokens=int(self._usage_details.get("cache_creation_input_token_count") or 0), + cached_tokens=int(usage_details.get("cache_read_input_token_count") or 0), + cache_write_tokens=int(usage_details.get("cache_creation_input_token_count") or 0), ), output_tokens=output_tokens, output_tokens_details=ResponseUsageOutputTokensDetails( - reasoning_tokens=int(self._usage_details.get("reasoning_output_token_count") or 0) + reasoning_tokens=int(usage_details.get("reasoning_output_token_count") or 0) ), total_tokens=int(total_tokens) if total_tokens is not None else input_tokens + output_tokens, ) @@ -2725,6 +2732,7 @@ async def handle_update( message_id=update.message_id, approval_storage=approval_storage, start_new_content_part=complete_message, + usage_stream=(update.author_name, update.response_id), ): yield event @@ -2735,6 +2743,7 @@ async def handle( *, approval_storage: FunctionApprovalStore | None = None, start_new_content_part: bool = False, + usage_stream: tuple[str | None, str | None] = (None, None), ) -> AsyncGenerator[ResponseStreamEvent]: """Process a content item, yielding its events. @@ -2748,6 +2757,8 @@ async def handle( (anything not recognized as a streaming delta type) to save/load approval requests. start_new_content_part: Whether text and refusal content start a new independent content part instead of continuing the current streaming occurrence. + usage_stream: The ``(author_name, response_id)`` of the update ``content`` came from. + A ``replace`` usage snapshot only supersedes earlier usage from the same stream. """ if _is_refusal_text_content(content) and content.text is not None: for event in self._ensure_message_content("refusal", message_id, start_new_part=start_new_content_part): @@ -3052,11 +3063,17 @@ async def handle( yield builder.emit_done(oauth_item) elif content.type == "usage": - # A "replace" update is a cumulative snapshot, so it supersedes earlier usage instead of adding to it. + # A "replace" update is a cumulative snapshot of its own stream, so it supersedes that + # stream's earlier usage instead of adding to it. Other streams keep their usage. if content.merge_mode == "replace": - self._usage_details = deepcopy(content.usage_details) + if content.usage_details is None: + self._usage_by_stream.pop(usage_stream, None) + else: + self._usage_by_stream[usage_stream] = deepcopy(content.usage_details) else: - self._usage_details = add_usage_details(self._usage_details, content.usage_details) + self._usage_by_stream[usage_stream] = add_usage_details( + self._usage_by_stream.get(usage_stream), content.usage_details + ) else: for event in self._close(): diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index f44a6ff8603..83143fa0b6a 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -54,6 +54,7 @@ ServiceSessionId, SessionStore, SupportsAgentRun, + UsageDetails, WorkflowAgent, WorkflowBuilder, WorkflowContext, @@ -997,6 +998,38 @@ async def test_output_item_tracker_replace_usage_is_copied() -> None: assert tracker.usage_details == {"input_token_count": 250, "output_token_count": 25, "total_token_count": 275} +async def test_output_item_tracker_scopes_replace_usage_to_its_stream() -> None: + stream = ResponseEventStream(response_id="resp_usage") + stream.emit_created() + stream.emit_in_progress() + tracker = _OutputItemTracker(stream) + + def usage_update(author_name: str, response_id: str, input_tokens: int) -> AgentResponseUpdate: + return AgentResponseUpdate( + contents=[ + Content.from_usage( + {"input_token_count": input_tokens, "output_token_count": input_tokens // 10}, + merge_mode="replace", + ) + ], + role="assistant", + author_name=author_name, + response_id=response_id, + ) + + for update in ( + usage_update("agent-a", "resp_a", 400), + usage_update("agent-a", "resp_a", 1000), + usage_update("agent-b", "resp_b", 50), + # The same agent run again is a new response stream, so its usage adds up. + usage_update("agent-a", "resp_a2", 20), + ): + async for _ in tracker.handle_update(update): + pass + + assert tracker.usage_details == {"input_token_count": 1070, "output_token_count": 107} + + async def test_item_to_message_marks_refusal_text() -> None: message = await _item_to_message( cast( @@ -9702,12 +9735,20 @@ async def start(messages: list[Message], ctx: WorkflowContext[AgentExecutorReque class _MultiUpdateWorkflowAgentMock(SupportsAgentRun): """Inner agent that streams one update per text in a single ``run`` call, and tracks ``run_count``.""" - def __init__(self, name: str, texts: Sequence[str], *, gate: asyncio.Event | None = None) -> None: + def __init__( + self, + name: str, + texts: Sequence[str], + *, + gate: asyncio.Event | None = None, + usage_snapshots: Sequence[UsageDetails] = (), + ) -> None: self.id = str(uuid.uuid4()) self.name = name self.description: str | None = None self._texts = list(texts) self._gate = gate + self._usage_snapshots = list(usage_snapshots) self.run_count = 0 self.started = asyncio.Event() self.cancelled = asyncio.Event() @@ -9754,6 +9795,7 @@ def run( texts = self._texts name = self.name gate = self._gate + usage_snapshots = self._usage_snapshots async def _aiter() -> AsyncIterator[AgentResponseUpdate]: self.started.set() @@ -9769,6 +9811,12 @@ async def _aiter() -> AsyncIterator[AgentResponseUpdate]: role="assistant", author_name=name, ) + for usage in usage_snapshots: + yield AgentResponseUpdate( + contents=[Content.from_usage(usage, merge_mode="replace")], + role="assistant", + author_name=name, + ) return ResponseStream(_aiter(), finalizer=AgentResponse.from_updates) @@ -9840,6 +9888,33 @@ class TestWorkflowAgentHosting: relative to the regular agent path. """ + async def test_replace_usage_from_each_executor_is_summed(self) -> None: + agent_a = _MultiUpdateWorkflowAgentMock( + "agent-a", + ["from a"], + usage_snapshots=[ + UsageDetails(input_token_count=400, output_token_count=40), + UsageDetails(input_token_count=1000, output_token_count=100), + ], + ) + agent_b = _MultiUpdateWorkflowAgentMock( + "agent-b", + ["from b"], + usage_snapshots=[UsageDetails(input_token_count=50, output_token_count=5)], + ) + workflow = WorkflowBuilder(start_executor=agent_a).add_edge(agent_a, agent_b).build() + + resp = await _post(_make_server(WorkflowAgent(workflow=workflow, name="Usage Workflow Agent")), stream=True) + + assert resp.status_code == 200 + assert [agent_a.run_count, agent_b.run_count] == [1, 1] + completed = _parse_sse_events(resp.text)[-1]["data"]["response"] + assert {key: completed["usage"][key] for key in ("input_tokens", "output_tokens", "total_tokens")} == { + "input_tokens": 1050, + "output_tokens": 105, + "total_tokens": 1155, + } + async def test_async_factory_creates_workflow_agent_for_each_request(self) -> None: created: list[tuple[WorkflowAgent, _MultiUpdateWorkflowAgentMock]] = [] From d0a005ad85301e2c2c620124fc83b760165c3add Mon Sep 17 00:00:00 2001 From: Mahdi-Hasan <43575088+Mahdi-Hasan@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:59:23 +0600 Subject: [PATCH 4/4] Python: Type merge_modes as Literal in usage merge test The parametrized test annotated merge_modes as tuple[str, str], which fails pyright, mypy, ty, zuban and pyrefly because Content.from_usage expects Literal['append', 'replace']. Co-Authored-By: Claude Haiku 5.5 --- python/packages/foundry_hosting/tests/test_responses.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/packages/foundry_hosting/tests/test_responses.py b/python/packages/foundry_hosting/tests/test_responses.py index 83143fa0b6a..cd705f5a50a 100644 --- a/python/packages/foundry_hosting/tests/test_responses.py +++ b/python/packages/foundry_hosting/tests/test_responses.py @@ -5553,7 +5553,11 @@ async def test_usage_is_aggregated_in_completed_response(self, caplog: pytest.Lo ), ], ) - async def test_usage_honors_merge_mode(self, merge_modes: tuple[str, str], expected: dict[str, int]) -> None: + async def test_usage_honors_merge_mode( + self, + merge_modes: tuple[Literal["append", "replace"], Literal["append", "replace"]], + expected: dict[str, int], + ) -> None: agent = _make_agent( stream_updates=[ AgentResponseUpdate(contents=[Content.from_text("Hello")], role="assistant"),