From 8a9ab8066a7e4bada9960ec46303f90679008fc9 Mon Sep 17 00:00:00 2001 From: A Vertex SDK engineer Date: Wed, 22 Jul 2026 11:28:12 -0700 Subject: [PATCH] fix: Merge sandbox-only turns in SDK-side interaction-to-AgentData conversion. FUTURE_COPYBARA_INTEGRATE_REVIEW=https://github.com/googleapis/python-aiplatform/pull/7003 from googleapis:release-please--branches--main 5905f2d33aa57c08026c20aa59e1a6c48b2afa05 PiperOrigin-RevId: 952235858 --- agentplatform/_genai/_evals_builtin_tools.py | 56 +++++ agentplatform/_genai/_evals_common.py | 7 + tests/unit/agentplatform/genai/test_evals.py | 231 +++++++++++++++++++ 3 files changed, 294 insertions(+) diff --git a/agentplatform/_genai/_evals_builtin_tools.py b/agentplatform/_genai/_evals_builtin_tools.py index eb23d5dae7..f11ca362d9 100644 --- a/agentplatform/_genai/_evals_builtin_tools.py +++ b/agentplatform/_genai/_evals_builtin_tools.py @@ -25,6 +25,10 @@ publishing internal tool contract details. **If the server catalog changes, this SDK-side copy must be updated to match.** + +This module also provides sandbox-detection helpers +(``SANDBOX_TOOL_NAMES``, ``is_sandbox_only_turn``) used by the display +path (``_evals_common._interaction_dict_to_agent_data``). """ from typing import Any, Optional @@ -87,6 +91,58 @@ ] +# Names of sandbox orchestration tools, derived from ``SANDBOX_DECLARATIONS`` +# so there is a single source of truth. +SANDBOX_TOOL_NAMES: frozenset[str] = frozenset( + decl.name for decl in SANDBOX_DECLARATIONS if decl.name +) + + +def is_sandbox_only_turn( + events: list[Any], +) -> bool: + """Returns True if a turn contains only sandbox initialization events. + + Sandbox provisioning events (``provision_sandbox``, ``load_sandbox``) + are infrastructure setup steps that happen before the user's first + real prompt. + + A turn is sandbox-only when every event is either a + ``function_call`` or ``function_response`` referencing a sandbox + tool name. Events with plain text content (model output, user + input) disqualify the turn. + + Args: + events: The list of AgentEvents in the turn. + + Returns: + True if the turn is sandbox-only and should be merged into the + next real turn for display. + """ + if not events: + return True + + for event in events: + content = getattr(event, "content", None) + if not content: + continue + parts = getattr(content, "parts", None) + if not parts: + continue + for part in parts: + if getattr(part, "function_call", None): + if part.function_call.name not in SANDBOX_TOOL_NAMES: + return False + elif getattr(part, "function_response", None): + if part.function_response.name not in SANDBOX_TOOL_NAMES: + return False + elif getattr(part, "text", None): + # Text content means this is a real conversational event, + # not sandbox infrastructure. + return False + return True + + def agent_tools_to_config_tools( agent_tools: Optional[list[Any]], has_environment: bool = False, diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index d91d352dc1..b78322fc97 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -760,6 +760,13 @@ def _interaction_dict_to_agent_data( grouped.append([]) grouped[-1].append(event) + # Merge leading sandbox-only turns into the first real turn. + # Sandbox provisioning events (provision_sandbox, load_sandbox) are + # infrastructure setup that precedes the user's first real prompt. + while len(grouped) > 1 and _evals_builtin_tools.is_sandbox_only_turn(grouped[0]): + grouped[1] = grouped[0] + grouped[1] + grouped.pop(0) + if not grouped: return types.evals.AgentData( # pytype: disable=missing-parameter turns=[ diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index b93b6db39b..20e01a318e 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -11384,6 +11384,237 @@ def test_unknown_steps_skipped(self): assert len(events) == 2 +class TestIsSandboxOnlyTurn: + """Tests for _is_sandbox_only_turn.""" + + def _make_event(self, *, author, part): + return agentplatform_genai_types.evals.AgentEvent( + author=author, + content=genai_types.Content(role="model", parts=[part]), + ) + + def test_provision_sandbox_is_sandbox_only(self): + events = [ + self._make_event( + author="agent", + part=genai_types.Part( + function_call=genai_types.FunctionCall( + name="provision_sandbox", args={} + ) + ), + ), + self._make_event( + author="user", + part=genai_types.Part( + function_response=genai_types.FunctionResponse( + name="provision_sandbox", response={"status": "ready"} + ) + ), + ), + ] + assert _evals_builtin_tools.is_sandbox_only_turn(events) is True + + def test_load_sandbox_is_sandbox_only(self): + events = [ + self._make_event( + author="agent", + part=genai_types.Part( + function_call=genai_types.FunctionCall( + name="load_sandbox", args={} + ) + ), + ), + ] + assert _evals_builtin_tools.is_sandbox_only_turn(events) is True + + def test_non_sandbox_tool_is_not_sandbox_only(self): + events = [ + self._make_event( + author="agent", + part=genai_types.Part( + function_call=genai_types.FunctionCall( + name="run_command", args={} + ) + ), + ), + ] + assert _evals_builtin_tools.is_sandbox_only_turn(events) is False + + def test_text_event_is_not_sandbox_only(self): + events = [ + self._make_event( + author="user", + part=genai_types.Part(text="Hello"), + ), + ] + assert _evals_builtin_tools.is_sandbox_only_turn(events) is False + + def test_empty_events_is_sandbox_only(self): + assert _evals_builtin_tools.is_sandbox_only_turn([]) is True + + +class TestSandboxTurnMergingInAgentData: + """Tests that sandbox-only turns are merged in _interaction_dict_to_agent_data.""" + + def test_sandbox_prefix_merged_into_single_turn(self): + """Sandbox + 1 real interaction produces 1 turn, not 2.""" + interaction_dict = { + "status": "completed", + "steps": [ + # Sandbox provisioning (no user_input, creates Turn 0) + { + "type": "function_call", + "name": "provision_sandbox", + "arguments": {"display_name": "sb"}, + "id": "call_sb", + }, + { + "type": "function_result", + "name": "provision_sandbox", + "call_id": "call_sb", + "result": {"status": "ready"}, + }, + # Real user interaction (UserInputStep creates Turn 1) + { + "type": "user_input", + "content": [{"type": "text", "text": "What is 2+2?"}], + }, + { + "type": "model_output", + "content": [{"type": "text", "text": "4"}], + }, + ], + } + result = _evals_common._interaction_dict_to_agent_data(interaction_dict) + # Should produce 1 turn, not 2. + assert len(result.turns) == 1 + assert result.turns[0].turn_index == 0 + # The turn should contain all events: sandbox + real. + events = result.turns[0].events + assert len(events) == 4 + # First two events are sandbox tool call/response. + assert events[0].content.parts[0].function_call.name == "provision_sandbox" + assert events[1].content.parts[0].function_response.name == "provision_sandbox" + # Last two are the real conversation. + assert events[2].content.parts[0].text == "What is 2+2?" + assert events[3].content.parts[0].text == "4" + + def test_sandbox_prefix_with_multi_turn(self): + """Sandbox + 2 real turns produces 2 turns, not 3.""" + interaction_dict = { + "status": "completed", + "steps": [ + { + "type": "function_call", + "name": "provision_sandbox", + "arguments": {}, + "id": "call_sb", + }, + { + "type": "function_result", + "name": "provision_sandbox", + "call_id": "call_sb", + "result": {"status": "ready"}, + }, + { + "type": "user_input", + "content": [{"type": "text", "text": "Turn 1"}], + }, + { + "type": "model_output", + "content": [{"type": "text", "text": "Reply 1"}], + }, + { + "type": "user_input", + "content": [{"type": "text", "text": "Turn 2"}], + }, + { + "type": "model_output", + "content": [{"type": "text", "text": "Reply 2"}], + }, + ], + } + result = _evals_common._interaction_dict_to_agent_data(interaction_dict) + assert len(result.turns) == 2 + assert result.turns[0].events[0].content.parts[0].function_call.name == "provision_sandbox" + assert result.turns[0].events[-1].content.parts[0].text == "Reply 1" + assert result.turns[1].events[0].content.parts[0].text == "Turn 2" + + def test_no_sandbox_prefix_unchanged(self): + """Interaction without sandbox steps is unaffected.""" + interaction_dict = { + "status": "completed", + "steps": [ + { + "type": "user_input", + "content": [{"type": "text", "text": "Hello"}], + }, + { + "type": "model_output", + "content": [{"type": "text", "text": "Hi"}], + }, + ], + } + result = _evals_common._interaction_dict_to_agent_data(interaction_dict) + assert len(result.turns) == 1 + assert len(result.turns[0].events) == 2 + + def test_all_sandbox_only_not_discarded(self): + """If every step is sandbox-only, it remains as 1 turn (not discarded).""" + interaction_dict = { + "status": "completed", + "steps": [ + { + "type": "function_call", + "name": "provision_sandbox", + "arguments": {}, + "id": "call_sb", + }, + { + "type": "function_result", + "name": "provision_sandbox", + "call_id": "call_sb", + "result": {"status": "ready"}, + }, + ], + } + result = _evals_common._interaction_dict_to_agent_data(interaction_dict) + # Only 1 group, and it's sandbox-only — but there's no second group + # to merge into, so it stays as-is. + assert len(result.turns) == 1 + + def test_non_sandbox_tool_before_user_input_not_merged(self): + """A non-sandbox tool call before user_input is NOT merged.""" + interaction_dict = { + "status": "completed", + "steps": [ + { + "type": "function_call", + "name": "run_command", + "arguments": {"cmd": "ls"}, + "id": "call_1", + }, + { + "type": "function_result", + "name": "run_command", + "call_id": "call_1", + "result": {"output": "file.txt"}, + }, + { + "type": "user_input", + "content": [{"type": "text", "text": "Hello"}], + }, + { + "type": "model_output", + "content": [{"type": "text", "text": "Hi"}], + }, + ], + } + result = _evals_common._interaction_dict_to_agent_data(interaction_dict) + # run_command is NOT a sandbox tool, so Turn 0 stays separate. + assert len(result.turns) == 2 + + class TestMergeTextPartsInAgentData: """Tests for _merge_text_parts_in_agent_data."""