From 594cac80c3e9dbbf146dac8194ae36c823e2cb12 Mon Sep 17 00:00:00 2001 From: Kip Date: Fri, 30 Oct 2026 22:11:45 +0800 Subject: [PATCH 1/3] Python: preserve hosted tool results in ToolResultCompactionStrategy Handle shell, code interpreter, image generation, and computer tool results when collapsing old tool-call groups. Preserve status and timeout information, avoid embedding image payloads, and fall back to an explicit "no output returned" instead of "no results". Add regression tests for all four result types, status-only and timeout cases, mixed hosted results, multiple outputs/errors, and payload exclusion. Co-Authored-By: Claude Opus 5 --- .../core/agent_framework/_compaction.py | 69 ++++++++ .../core/tests/core/test_compaction.py | 167 ++++++++++++++++++ 2 files changed, 236 insertions(+) diff --git a/python/packages/core/agent_framework/_compaction.py b/python/packages/core/agent_framework/_compaction.py index 28ec1a7f017..6b372171eb2 100644 --- a/python/packages/core/agent_framework/_compaction.py +++ b/python/packages/core/agent_framework/_compaction.py @@ -1995,6 +1995,15 @@ async def __call__(self, messages: list[Message]) -> bool: tool_name = call_id_to_name.get(content.call_id or "", "") label = f"{tool_name}: {result_text}" if tool_name else result_text tool_results.append(label.strip()) + elif content.type in ( + "code_interpreter_tool_result", + "computer_tool_result", + "image_generation_tool_result", + "shell_tool_result", + ): + result_text = _hosted_tool_result_summary(content) + if result_text: + tool_results.append(f"{content.type}: {result_text}") summary_text = self._summary_text(tool_results) summary_id = f"tool_summary_{group_id}" @@ -2180,6 +2189,66 @@ def _format_summary_content(content: Content) -> str: return "" +def _hosted_tool_result_summary(content: Content) -> str: + """Render hosted tool outputs without embedding image payloads in summaries.""" + if content.type == "shell_tool_result": + parts: list[str] = [] + for output in content.outputs or []: + if output.stdout: + parts.append(f"stdout: {output.stdout}") + if output.stderr: + parts.append(f"stderr: {output.stderr}") + if output.timed_out: + parts.append("command timed out") + elif output.exit_code is not None: + parts.append(f"exit_code: {output.exit_code}") + return "\n".join(parts) or "no output returned" + + if content.type == "computer_tool_result": + details = [f"status={content.status}"] if content.status else [] + if content.screenshot is not None: + details.append("screenshot") + return ", ".join(details) or "result received" + + outputs = content.outputs + if isinstance(outputs, Content): + output_items = [outputs] + elif isinstance(outputs, Sequence) and not isinstance(outputs, (str, bytes, bytearray)): + output_items = [item for item in outputs if isinstance(item, Content)] + else: + output_items = [] + + if output_items: + parts = [] + for item in output_items: + if item.type == "text" and item.text: + parts.append(item.text) + elif item.type == "error": + error_text = item.message or item.error_details or "tool output error" + if item.error_code: + error_text = f"error({item.error_code}): {error_text}" + parts.append(error_text) + else: + # Hosted image and file outputs can carry data URIs or binary + # payloads. Keep only descriptive metadata in the compact trace. + metadata = [f"media_type={item.media_type}"] if item.media_type else [] + if item.file_id: + metadata.append(f"file_id={item.file_id}") + parts.append(f"{item.type} content" + (f" ({', '.join(metadata)})" if metadata else "")) + status = getattr(content, "status", None) + if status: + parts.insert(0, f"status={status}") + return "\n".join(parts) or "no output returned" + + status = getattr(content, "status", None) + parts = [f"status={status}"] if status else [] + if content.type == "image_generation_tool_result" and outputs: + parts.append("image output available") + elif not parts: + parts.append("no output returned") + return "\n".join(parts) + + def _format_summary_message(index: int, message: Message) -> str: parts: list[str] = [] pending_text: list[str] = [] diff --git a/python/packages/core/tests/core/test_compaction.py b/python/packages/core/tests/core/test_compaction.py index 8e4f6f54eab..f04e0451939 100644 --- a/python/packages/core/tests/core/test_compaction.py +++ b/python/packages/core/tests/core/test_compaction.py @@ -2471,6 +2471,173 @@ async def test_tool_result_compaction_preserves_hosted_search_result() -> None: assert "no results" not in summaries[0] +@pytest.mark.parametrize("result_kind", ["shell", "code_interpreter", "image_generation", "computer"]) +async def test_tool_result_compaction_preserves_hosted_tool_results(result_kind: str) -> None: + if result_kind == "shell": + call = Content.from_shell_tool_call(call_id="shell-1", commands=["echo hello"]) + result = Content.from_shell_tool_result( + call_id="shell-1", + outputs=[Content.from_shell_command_output(stdout="hello from shell", stderr="warning", exit_code=1)], + ) + expected = ["hello from shell", "warning", "exit_code: 1"] + excluded = [] + elif result_kind == "code_interpreter": + call = Content.from_code_interpreter_tool_call( + call_id="code-1", inputs=[Content.from_text(text="print('hello')")] + ) + result = Content.from_code_interpreter_tool_result( + call_id="code-1", outputs=[Content.from_text(text="hello from interpreter")] + ) + expected = ["hello from interpreter"] + excluded = [] + elif result_kind == "image_generation": + call = Content.from_image_generation_tool_call(image_id="image-1") + image_data = Content.from_data(data=b"private-image-payload", media_type="image/png") + result = Content.from_image_generation_tool_result(image_id="image-1", outputs=[image_data]) + expected = ["image_generation_tool_result", "media_type=image/png"] + excluded = [image_data.uri or "", "base64,"] + else: + call = Content.from_computer_tool_call(id="computer-1", call_id="computer-1", actions=[{"type": "click"}]) + screenshot = Content.from_data(data=b"private-screen-payload", media_type="image/png") + result = Content.from_computer_tool_result(call_id="computer-1", screenshot=screenshot) + expected = ["computer_tool_result: screenshot"] + excluded = [screenshot.uri or "", "base64,"] + + call_message = Message(role="assistant", contents=[call]) + result_message = Message(role="tool", contents=[result]) + messages = [ + Message(role="user", contents=["run the hosted tool"]), + call_message, + result_message, + Message(role="assistant", contents=["done"]), + ] + annotate_message_groups(messages) + + changed = await ToolResultCompactionStrategy(keep_last_tool_call_groups=0)(messages) + + summaries = [ + message.text or "" + for message in included_messages(messages) + if (message.text or "").startswith("[Tool results:") + ] + assert changed is True + assert call_message.additional_properties[EXCLUDED_KEY] is True + assert result_message.additional_properties[EXCLUDED_KEY] is True + assert len(summaries) == 1 + assert "no results" not in summaries[0] + for fragment in expected: + assert fragment in summaries[0] + for fragment in excluded: + if fragment: + assert fragment not in summaries[0] + + +@pytest.mark.parametrize( + ("result", "expected"), + [ + ( + Content.from_code_interpreter_tool_result(call_id="code-1", outputs=None, status="failed"), + "status=failed", + ), + ( + Content.from_shell_tool_result( + call_id="shell-1", outputs=[Content.from_shell_command_output(timed_out=True)] + ), + "command timed out", + ), + (Content.from_image_generation_tool_result(image_id="image-1", outputs=None), "no output returned"), + ], +) +async def test_tool_result_compaction_preserves_hosted_tool_result_status_or_empty_state( + result: Content, expected: str +) -> None: + if result.type == "code_interpreter_tool_result": + call = Content.from_code_interpreter_tool_call(call_id="code-1", inputs=[Content.from_text(text="run")]) + elif result.type == "shell_tool_result": + call = Content.from_shell_tool_call(call_id="shell-1", commands=["sleep 10"]) + else: + call = Content.from_image_generation_tool_call(image_id="image-1") + + messages = [ + Message(role="assistant", contents=[call]), + Message(role="tool", contents=[result]), + Message(role="assistant", contents=["done"]), + ] + annotate_message_groups(messages) + + changed = await ToolResultCompactionStrategy(keep_last_tool_call_groups=0)(messages) + + summaries = [ + message.text or "" + for message in included_messages(messages) + if (message.text or "").startswith("[Tool results:") + ] + assert changed is True + assert len(summaries) == 1 + assert expected in summaries[0] + assert "no results" not in summaries[0] + + +async def test_tool_result_compaction_preserves_multiple_hosted_outputs_and_errors() -> None: + call = Content.from_code_interpreter_tool_call(call_id="code-1", inputs=[Content.from_text(text="run")]) + result = Content.from_code_interpreter_tool_result( + call_id="code-1", + outputs=[ + Content.from_text(text="first output"), + Content.from_error(message="execution failed", error_code="runtime_error"), + Content.from_text(text="last output"), + ], + ) + messages = [ + Message(role="assistant", contents=[call]), + Message(role="tool", contents=[result]), + Message(role="assistant", contents=["done"]), + ] + annotate_message_groups(messages) + + await ToolResultCompactionStrategy(keep_last_tool_call_groups=0)(messages) + + summaries = [ + message.text or "" + for message in included_messages(messages) + if (message.text or "").startswith("[Tool results:") + ] + assert len(summaries) == 1 + assert "first output" in summaries[0] + assert "error(runtime_error): execution failed" in summaries[0] + assert "last output" in summaries[0] + + +async def test_tool_result_compaction_preserves_mixed_hosted_results_in_one_group() -> None: + shell_call = Content.from_shell_tool_call(call_id="shell-1", commands=["echo shell"]) + code_call = Content.from_code_interpreter_tool_call(call_id="code-1", inputs=[Content.from_text(text="run code")]) + shell_result = Content.from_shell_tool_result( + call_id="shell-1", outputs=[Content.from_shell_command_output(stdout="shell output", exit_code=0)] + ) + code_result = Content.from_code_interpreter_tool_result( + call_id="code-1", outputs=[Content.from_text(text="interpreter output")] + ) + call_message = Message(role="assistant", contents=[shell_call, code_call]) + result_message = Message(role="tool", contents=[shell_result, code_result]) + messages = [call_message, result_message, Message(role="assistant", contents=["done"])] + annotate_message_groups(messages) + + changed = await ToolResultCompactionStrategy(keep_last_tool_call_groups=0)(messages) + + summaries = [ + message.text or "" + for message in included_messages(messages) + if (message.text or "").startswith("[Tool results:") + ] + assert changed is True + assert call_message.additional_properties[EXCLUDED_KEY] is True + assert result_message.additional_properties[EXCLUDED_KEY] is True + assert len(summaries) == 1 + assert "shell output" in summaries[0] + assert "interpreter output" in summaries[0] + assert "no results" not in summaries[0] + + async def test_tool_result_compaction_collapses_old_groups_into_summary() -> None: """Old tool-call groups are collapsed into summary messages, newest kept.""" messages = [ From f33eda9ba39163a49b8989f9b1d740535965457c Mon Sep 17 00:00:00 2001 From: Kip Date: Mon, 2 Nov 2026 16:22:18 +0800 Subject: [PATCH 2/3] Fix Pyright typing for hosted tool outputs --- python/packages/core/agent_framework/_compaction.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/packages/core/agent_framework/_compaction.py b/python/packages/core/agent_framework/_compaction.py index 6b372171eb2..001f10a8257 100644 --- a/python/packages/core/agent_framework/_compaction.py +++ b/python/packages/core/agent_framework/_compaction.py @@ -2211,12 +2211,13 @@ def _hosted_tool_result_summary(content: Content) -> str: return ", ".join(details) or "result received" outputs = content.outputs + output_items: list[Content] = [] if isinstance(outputs, Content): - output_items = [outputs] + output_items.append(outputs) elif isinstance(outputs, Sequence) and not isinstance(outputs, (str, bytes, bytearray)): - output_items = [item for item in outputs if isinstance(item, Content)] - else: - output_items = [] + for output in cast("Sequence[Any]", outputs): + if isinstance(output, Content): + output_items.append(output) if output_items: parts = [] From d0f222f1c0a438e4aa4d761ae69ac1caca68196a Mon Sep 17 00:00:00 2001 From: Kip Date: Mon, 2 Nov 2026 16:37:05 +0800 Subject: [PATCH 3/3] Python: annotate hosted tool result test collection type mypy and zuban report "Need type annotation for excluded" at test_compaction.py:2483 because the first assignment is an empty list. Co-Authored-By: Claude Opus 5 --- python/packages/core/tests/core/test_compaction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/packages/core/tests/core/test_compaction.py b/python/packages/core/tests/core/test_compaction.py index f04e0451939..cb4711d6fe0 100644 --- a/python/packages/core/tests/core/test_compaction.py +++ b/python/packages/core/tests/core/test_compaction.py @@ -2480,7 +2480,7 @@ async def test_tool_result_compaction_preserves_hosted_tool_results(result_kind: outputs=[Content.from_shell_command_output(stdout="hello from shell", stderr="warning", exit_code=1)], ) expected = ["hello from shell", "warning", "exit_code: 1"] - excluded = [] + excluded: list[str] = [] elif result_kind == "code_interpreter": call = Content.from_code_interpreter_tool_call( call_id="code-1", inputs=[Content.from_text(text="print('hello')")]