diff --git a/python/packages/core/agent_framework/_compaction.py b/python/packages/core/agent_framework/_compaction.py index 28ec1a7f017..001f10a8257 100644 --- a/python/packages/core/agent_framework/_compaction.py +++ b/python/packages/core/agent_framework/_compaction.py @@ -1995,6 +1995,15 @@ async def __call__(self, messages: list[Message]) -> bool: tool_name = call_id_to_name.get(content.call_id or "", "") label = f"{tool_name}: {result_text}" if tool_name else result_text tool_results.append(label.strip()) + elif content.type in ( + "code_interpreter_tool_result", + "computer_tool_result", + "image_generation_tool_result", + "shell_tool_result", + ): + result_text = _hosted_tool_result_summary(content) + if result_text: + tool_results.append(f"{content.type}: {result_text}") summary_text = self._summary_text(tool_results) summary_id = f"tool_summary_{group_id}" @@ -2180,6 +2189,67 @@ def _format_summary_content(content: Content) -> str: return "" +def _hosted_tool_result_summary(content: Content) -> str: + """Render hosted tool outputs without embedding image payloads in summaries.""" + if content.type == "shell_tool_result": + parts: list[str] = [] + for output in content.outputs or []: + if output.stdout: + parts.append(f"stdout: {output.stdout}") + if output.stderr: + parts.append(f"stderr: {output.stderr}") + if output.timed_out: + parts.append("command timed out") + elif output.exit_code is not None: + parts.append(f"exit_code: {output.exit_code}") + return "\n".join(parts) or "no output returned" + + if content.type == "computer_tool_result": + details = [f"status={content.status}"] if content.status else [] + if content.screenshot is not None: + details.append("screenshot") + return ", ".join(details) or "result received" + + outputs = content.outputs + output_items: list[Content] = [] + if isinstance(outputs, Content): + output_items.append(outputs) + elif isinstance(outputs, Sequence) and not isinstance(outputs, (str, bytes, bytearray)): + for output in cast("Sequence[Any]", outputs): + if isinstance(output, Content): + output_items.append(output) + + if output_items: + parts = [] + for item in output_items: + if item.type == "text" and item.text: + parts.append(item.text) + elif item.type == "error": + error_text = item.message or item.error_details or "tool output error" + if item.error_code: + error_text = f"error({item.error_code}): {error_text}" + parts.append(error_text) + else: + # Hosted image and file outputs can carry data URIs or binary + # payloads. Keep only descriptive metadata in the compact trace. + metadata = [f"media_type={item.media_type}"] if item.media_type else [] + if item.file_id: + metadata.append(f"file_id={item.file_id}") + parts.append(f"{item.type} content" + (f" ({', '.join(metadata)})" if metadata else "")) + status = getattr(content, "status", None) + if status: + parts.insert(0, f"status={status}") + return "\n".join(parts) or "no output returned" + + status = getattr(content, "status", None) + parts = [f"status={status}"] if status else [] + if content.type == "image_generation_tool_result" and outputs: + parts.append("image output available") + elif not parts: + parts.append("no output returned") + return "\n".join(parts) + + def _format_summary_message(index: int, message: Message) -> str: parts: list[str] = [] pending_text: list[str] = [] diff --git a/python/packages/core/tests/core/test_compaction.py b/python/packages/core/tests/core/test_compaction.py index 8e4f6f54eab..cb4711d6fe0 100644 --- a/python/packages/core/tests/core/test_compaction.py +++ b/python/packages/core/tests/core/test_compaction.py @@ -2471,6 +2471,173 @@ async def test_tool_result_compaction_preserves_hosted_search_result() -> None: assert "no results" not in summaries[0] +@pytest.mark.parametrize("result_kind", ["shell", "code_interpreter", "image_generation", "computer"]) +async def test_tool_result_compaction_preserves_hosted_tool_results(result_kind: str) -> None: + if result_kind == "shell": + call = Content.from_shell_tool_call(call_id="shell-1", commands=["echo hello"]) + result = Content.from_shell_tool_result( + call_id="shell-1", + outputs=[Content.from_shell_command_output(stdout="hello from shell", stderr="warning", exit_code=1)], + ) + expected = ["hello from shell", "warning", "exit_code: 1"] + excluded: list[str] = [] + elif result_kind == "code_interpreter": + call = Content.from_code_interpreter_tool_call( + call_id="code-1", inputs=[Content.from_text(text="print('hello')")] + ) + result = Content.from_code_interpreter_tool_result( + call_id="code-1", outputs=[Content.from_text(text="hello from interpreter")] + ) + expected = ["hello from interpreter"] + excluded = [] + elif result_kind == "image_generation": + call = Content.from_image_generation_tool_call(image_id="image-1") + image_data = Content.from_data(data=b"private-image-payload", media_type="image/png") + result = Content.from_image_generation_tool_result(image_id="image-1", outputs=[image_data]) + expected = ["image_generation_tool_result", "media_type=image/png"] + excluded = [image_data.uri or "", "base64,"] + else: + call = Content.from_computer_tool_call(id="computer-1", call_id="computer-1", actions=[{"type": "click"}]) + screenshot = Content.from_data(data=b"private-screen-payload", media_type="image/png") + result = Content.from_computer_tool_result(call_id="computer-1", screenshot=screenshot) + expected = ["computer_tool_result: screenshot"] + excluded = [screenshot.uri or "", "base64,"] + + call_message = Message(role="assistant", contents=[call]) + result_message = Message(role="tool", contents=[result]) + messages = [ + Message(role="user", contents=["run the hosted tool"]), + call_message, + result_message, + Message(role="assistant", contents=["done"]), + ] + annotate_message_groups(messages) + + changed = await ToolResultCompactionStrategy(keep_last_tool_call_groups=0)(messages) + + summaries = [ + message.text or "" + for message in included_messages(messages) + if (message.text or "").startswith("[Tool results:") + ] + assert changed is True + assert call_message.additional_properties[EXCLUDED_KEY] is True + assert result_message.additional_properties[EXCLUDED_KEY] is True + assert len(summaries) == 1 + assert "no results" not in summaries[0] + for fragment in expected: + assert fragment in summaries[0] + for fragment in excluded: + if fragment: + assert fragment not in summaries[0] + + +@pytest.mark.parametrize( + ("result", "expected"), + [ + ( + Content.from_code_interpreter_tool_result(call_id="code-1", outputs=None, status="failed"), + "status=failed", + ), + ( + Content.from_shell_tool_result( + call_id="shell-1", outputs=[Content.from_shell_command_output(timed_out=True)] + ), + "command timed out", + ), + (Content.from_image_generation_tool_result(image_id="image-1", outputs=None), "no output returned"), + ], +) +async def test_tool_result_compaction_preserves_hosted_tool_result_status_or_empty_state( + result: Content, expected: str +) -> None: + if result.type == "code_interpreter_tool_result": + call = Content.from_code_interpreter_tool_call(call_id="code-1", inputs=[Content.from_text(text="run")]) + elif result.type == "shell_tool_result": + call = Content.from_shell_tool_call(call_id="shell-1", commands=["sleep 10"]) + else: + call = Content.from_image_generation_tool_call(image_id="image-1") + + messages = [ + Message(role="assistant", contents=[call]), + Message(role="tool", contents=[result]), + Message(role="assistant", contents=["done"]), + ] + annotate_message_groups(messages) + + changed = await ToolResultCompactionStrategy(keep_last_tool_call_groups=0)(messages) + + summaries = [ + message.text or "" + for message in included_messages(messages) + if (message.text or "").startswith("[Tool results:") + ] + assert changed is True + assert len(summaries) == 1 + assert expected in summaries[0] + assert "no results" not in summaries[0] + + +async def test_tool_result_compaction_preserves_multiple_hosted_outputs_and_errors() -> None: + call = Content.from_code_interpreter_tool_call(call_id="code-1", inputs=[Content.from_text(text="run")]) + result = Content.from_code_interpreter_tool_result( + call_id="code-1", + outputs=[ + Content.from_text(text="first output"), + Content.from_error(message="execution failed", error_code="runtime_error"), + Content.from_text(text="last output"), + ], + ) + messages = [ + Message(role="assistant", contents=[call]), + Message(role="tool", contents=[result]), + Message(role="assistant", contents=["done"]), + ] + annotate_message_groups(messages) + + await ToolResultCompactionStrategy(keep_last_tool_call_groups=0)(messages) + + summaries = [ + message.text or "" + for message in included_messages(messages) + if (message.text or "").startswith("[Tool results:") + ] + assert len(summaries) == 1 + assert "first output" in summaries[0] + assert "error(runtime_error): execution failed" in summaries[0] + assert "last output" in summaries[0] + + +async def test_tool_result_compaction_preserves_mixed_hosted_results_in_one_group() -> None: + shell_call = Content.from_shell_tool_call(call_id="shell-1", commands=["echo shell"]) + code_call = Content.from_code_interpreter_tool_call(call_id="code-1", inputs=[Content.from_text(text="run code")]) + shell_result = Content.from_shell_tool_result( + call_id="shell-1", outputs=[Content.from_shell_command_output(stdout="shell output", exit_code=0)] + ) + code_result = Content.from_code_interpreter_tool_result( + call_id="code-1", outputs=[Content.from_text(text="interpreter output")] + ) + call_message = Message(role="assistant", contents=[shell_call, code_call]) + result_message = Message(role="tool", contents=[shell_result, code_result]) + messages = [call_message, result_message, Message(role="assistant", contents=["done"])] + annotate_message_groups(messages) + + changed = await ToolResultCompactionStrategy(keep_last_tool_call_groups=0)(messages) + + summaries = [ + message.text or "" + for message in included_messages(messages) + if (message.text or "").startswith("[Tool results:") + ] + assert changed is True + assert call_message.additional_properties[EXCLUDED_KEY] is True + assert result_message.additional_properties[EXCLUDED_KEY] is True + assert len(summaries) == 1 + assert "shell output" in summaries[0] + assert "interpreter output" in summaries[0] + assert "no results" not in summaries[0] + + async def test_tool_result_compaction_collapses_old_groups_into_summary() -> None: """Old tool-call groups are collapsed into summary messages, newest kept.""" messages = [