From c00bdfcbe25f409e86c0353144ac7a2f24d5edbb Mon Sep 17 00:00:00 2001 From: t Date: Thu, 15 Oct 2026 15:53:56 -0400 Subject: [PATCH 1/3] feat(verifiers): read coding-agent CLI trajectories in the judge action log The direct judge's action log (compaction_type=screenshot, trajectory_text=action_log) only parsed OTel spans. Trajectories from coding-agent CLIs (stream-json rows) and OpenCode (part events) produced an empty transcript, so a direct judge on those runs graded without seeing a single tool call. - transcript_from_raw detects the format: OTel spans (unchanged), stream-json rows (tool_use paired with tool_result by id; is_error recorded as {"error": ...}), OpenCode tool/text parts (re-emitted parts merged; random mcp__ prefix dropped). JSON-in-text tool results are decoded in the two new parsers. - action_log takes max_args_chars / max_result_chars; TrajectoryFilter gains action_log_max_args_chars / action_log_max_result_chars (defaults unchanged at 110 / 160) and rubrics_verifier passes them through. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/judge_utils/frame_selection.py | 159 +++++++++++++++--- .../judge_utils/trajectory_filter.py | 14 ++ .../task_steps/verifiers/rubrics_verifier.py | 4 +- .../test_frame_selection_cli_formats.py | 104 ++++++++++++ .../test_rubrics_verifier_per_criterion.py | 44 +++++ 5 files changed, 304 insertions(+), 21 deletions(-) create mode 100644 tst/unit/task_step/verifiers/judge_utils/test_frame_selection_cli_formats.py diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py index f7715cd3..be3279a1 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py @@ -11,10 +11,12 @@ import json import re from collections.abc import Iterable, Mapping -from dataclasses import dataclass +from dataclasses import dataclass, replace from agent_env.task_step.task_steps.verifiers.judge_utils.trajectory_filter import ( + DEFAULT_ACTION_LOG_MAX_ARGS_CHARS, DEFAULT_ACTION_LOG_MAX_LINES, + DEFAULT_ACTION_LOG_MAX_RESULT_CHARS, DEFAULT_FRAME_BUDGET, FINAL_FRAMES_RESERVED, ImageFrame, @@ -23,11 +25,15 @@ strip_base64_images, ) -# Cuts (with an ellipsis) applied to the rendered JSON of tool arguments — the same text the action log, -# the regex knobs and criterion matching see — tool results and agent messages. -MAX_ARGS_CHARS = 110 -MAX_RESULT_CHARS = 160 +# Cuts (with an ellipsis) applied to the rendered JSON of tool arguments — the text the regex knobs and +# criterion matching see, and the action log's default — tool results and agent messages. +MAX_ARGS_CHARS = DEFAULT_ACTION_LOG_MAX_ARGS_CHARS +MAX_RESULT_CHARS = DEFAULT_ACTION_LOG_MAX_RESULT_CHARS MAX_AGENT_MESSAGE_CHARS = 200 +# Row types of a coding-agent CLI's stream-json event list. +_STREAM_JSON_TYPES = frozenset({"system", "assistant", "user", "result"}) +# The per-server prefix some CLIs put on MCP tool names: random hex that carries no meaning. +_HEX_TOOL_PREFIX = re.compile(r"^mcp_[0-9a-f]+_") # Replaces a base64 image payload in rendered text before any cut, so the cut lands on text. _IMAGE_PLACEHOLDER = "" # A second candidate scoring at least this fraction of a criterion's best match is kept as a near-tie. @@ -102,15 +108,126 @@ def _text_blocks(completion: object) -> list[str]: def transcript_from_raw(raw_text: str) -> Transcript: - """Every ``execute_tool`` span as a ``TrajectoryAction`` and every text block of a ``chat`` span as an - ``AgentMessage``, in span order. The ``chain`` root is skipped: its completion is the final response the - judge already gets. Malformed input gives an empty transcript.""" + """The tool calls and agent text of a raw trajectory, in run order. Reads OTel span lists, the + stream-json rows of coding-agent CLIs (``type`` system / assistant / user / result) and OpenCode event + lists (``part.type`` tool / text). Malformed or unrecognised input gives an empty transcript.""" try: - spans = json.loads(raw_text) + rows = json.loads(raw_text) except (json.JSONDecodeError, TypeError): return Transcript([], []) - if not isinstance(spans, list): + if not isinstance(rows, list): return Transcript([], []) + dicts = [r for r in rows if isinstance(r, dict)] + if any(isinstance(r.get("attributes"), dict) for r in dicts): + return _transcript_from_otel(dicts) + if any(r.get("type") in _STREAM_JSON_TYPES for r in dicts): + return _transcript_from_stream_json(dicts) + if any(isinstance(r.get("part"), dict) for r in dicts): + return _transcript_from_opencode(dicts) + return Transcript([], []) + + +def _decoded(value: object, depth: int = 3) -> object: + """``value`` with JSON text decoded, also inside dict and list values, ``depth`` levels deep: CLI tool + results are often JSON wrapped in a JSON string, which rendered as text is mostly escape characters.""" + if depth <= 0: + return value + if isinstance(value, str): + text = value.strip() + if text[:1] in ("{", "[") and text[-1:] in ("}", "]"): + try: + return _decoded(json.loads(text), depth - 1) + except json.JSONDecodeError: + return value + return value + if isinstance(value, dict): + return {k: _decoded(v, depth) for k, v in value.items()} + if isinstance(value, list): + return [_decoded(v, depth) for v in value] + return value + + +def _tool_result_text(content: object) -> str: + """A stream-json ``tool_result`` body as text: text blocks joined, images as the placeholder.""" + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" if content is None else json.dumps(content, ensure_ascii=False) + parts = [] + for block in content: + if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str): + parts.append(block["text"]) + elif isinstance(block, dict) and block.get("type") == "image": + parts.append(_IMAGE_PLACEHOLDER) + return "\n".join(parts) + + +def _transcript_from_stream_json(rows: list[dict]) -> Transcript: + """``tool_use`` blocks of assistant rows as actions, each completed by the ``tool_result`` with its id + (JSON text decoded; an ``is_error`` result is recorded as ``{"error": ...}``); assistant ``text`` blocks + as messages. The closing ``result`` row is skipped: it is the final response the judge already gets.""" + actions: list[TrajectoryAction] = [] + messages: list[AgentMessage] = [] + position: dict[str, int] = {} + for row in rows: + message = row.get("message") + content = message.get("content") if isinstance(message, dict) else None + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + kind = block.get("type") + if row.get("type") == "assistant" and kind == "text" and str(block.get("text") or "").strip(): + messages.append(AgentMessage(len(actions), block["text"])) + elif row.get("type") == "assistant" and kind == "tool_use": + args = block.get("input") if isinstance(block.get("input"), dict) else {} + position[str(block.get("id"))] = len(actions) + actions.append(TrajectoryAction(index=len(actions) + 1, tool=str(block.get("name") or ""), + args=args, screenshot=None)) + elif row.get("type") == "user" and kind == "tool_result" and str(block.get("tool_use_id")) in position: + i = position[str(block["tool_use_id"])] + body = _decoded(_tool_result_text(block.get("content"))) + result = {"error": body} if block.get("is_error") else (body or None) + actions[i] = replace(actions[i], result=result) + return Transcript(actions, messages) + + +def _transcript_from_opencode(events: list[dict]) -> Transcript: + """``tool`` parts as actions (input, then output or error) and ``text`` parts as messages. A part that is + re-emitted as its state advances keeps its first position and its latest state.""" + actions: list[TrajectoryAction] = [] + messages: list[AgentMessage] = [] + position: dict[str, int] = {} + for event in events: + part = event.get("part") + if not isinstance(part, dict): + continue + if part.get("type") == "text" and str(part.get("text") or "").strip(): + messages.append(AgentMessage(len(actions), part["text"])) + elif part.get("type") == "tool": + state = part.get("state") if isinstance(part.get("state"), dict) else {} + args = state.get("input") if isinstance(state.get("input"), dict) else {} + if state.get("status") == "error": + result = {"error": _decoded(state.get("error"))} + else: + result = _decoded(state.get("output")) if state.get("output") not in ("", None) else None + tool = _HEX_TOOL_PREFIX.sub("", str(part.get("tool") or "")) + key = str(part.get("id") or len(actions)) + if key in position: + i = position[key] + actions[i] = replace(actions[i], tool=tool, args=args, result=result) + continue + position[key] = len(actions) + actions.append(TrajectoryAction(index=len(actions) + 1, tool=tool, args=args, screenshot=None, + result=result)) + return Transcript(actions, messages) + + +def _transcript_from_otel(spans: list[dict]) -> Transcript: + """Every ``execute_tool`` span as a ``TrajectoryAction`` and every text block of a ``chat`` span as an + ``AgentMessage``, in span order. The ``chain`` root is skipped: its completion is the final response the + judge already gets.""" actions: list[TrajectoryAction] = [] messages: list[AgentMessage] = [] for span in spans: @@ -150,14 +267,14 @@ def _render_json(value: object, max_chars: int) -> str: def rendered_action(a: TrajectoryAction) -> str: """``" "``, the arguments cut to ``MAX_ARGS_CHARS`` — the text the regex knobs (``evidence_exclude_pattern``, ``always_show_actions``) and criterion matching see, and what the action - log shows, so a value past the cut cannot be matched.""" + log shows at its default cut, so a value past the cut cannot be matched.""" return f"{a.tool} {_render_json(a.args, MAX_ARGS_CHARS)}" -def _action_line(a: TrajectoryAction) -> str: - line = f"{a.index}. {rendered_action(a)}" +def _action_line(a: TrajectoryAction, args_chars: int, result_chars: int) -> str: + line = f"{a.index}. {a.tool} {_render_json(a.args, args_chars)}" if a.result is not None: - line += f" -> {_render_json(a.result, MAX_RESULT_CHARS)}" + line += f" -> {_render_json(a.result, result_chars)}" return line @@ -169,17 +286,19 @@ def _message_line(m: AgentMessage) -> str: def action_log(actions: Iterable[TrajectoryAction], *, messages: Iterable[AgentMessage] = (), - max_lines: int = DEFAULT_ACTION_LOG_MAX_LINES) -> str: - """The run as text: ``". -> "`` per tool call and, in run order between them, - ``'agent said: "..."'`` per agent message — the tool lines are facts, the agent's lines its claims. - Over ``max_lines`` the head and the tail are kept (half each, the head taking the odd line) around one - ``"... (N lines not shown) ..."`` marker, so the final actions are always in the log.""" + max_lines: int = DEFAULT_ACTION_LOG_MAX_LINES, max_args_chars: int = MAX_ARGS_CHARS, + max_result_chars: int = MAX_RESULT_CHARS) -> str: + """The run as text: ``". -> "`` per tool call, arguments and result cut to + ``max_args_chars`` / ``max_result_chars``, and, in run order between them, ``'agent said: "..."'`` per + agent message — the tool lines are facts, the agent's lines its claims. Over ``max_lines`` the head and + the tail are kept (half each, the head taking the odd line) around one ``"... (N lines not shown) ..."`` + marker, so the final actions are always in the log.""" pending = sorted(messages, key=lambda m: m.after_action) lines: list[str] = [] for a in actions: while pending and pending[0].after_action < a.index: lines.append(_message_line(pending.pop(0))) - lines.append(_action_line(a)) + lines.append(_action_line(a, max_args_chars, max_result_chars)) lines.extend(_message_line(m) for m in pending) if len(lines) <= max_lines: return "\n".join(lines) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/trajectory_filter.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/trajectory_filter.py index ce3865a4..176e704c 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/trajectory_filter.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/trajectory_filter.py @@ -35,6 +35,10 @@ # ACTION_LOG line cap (``action_log_max_lines``, see ``frame_selection.action_log``). DEFAULT_ACTION_LOG_MAX_LINES = 120 MIN_ACTION_LOG_MAX_LINES = 10 +# ACTION_LOG per-line cuts of the rendered tool arguments and result. +DEFAULT_ACTION_LOG_MAX_ARGS_CHARS = 110 +DEFAULT_ACTION_LOG_MAX_RESULT_CHARS = 160 +MIN_ACTION_LOG_FIELD_CHARS = 20 class CompactionType(str, Enum): @@ -113,6 +117,8 @@ class TrajectoryFilter: # (e.g. {"submitted": "^submit_form\\b"}); see ``frame_selection`` for the cap. always_show_actions: Optional[dict[str, str]] = None action_log_max_lines: int = DEFAULT_ACTION_LOG_MAX_LINES # >= MIN_ACTION_LOG_MAX_LINES + action_log_max_args_chars: int = DEFAULT_ACTION_LOG_MAX_ARGS_CHARS # >= MIN_ACTION_LOG_FIELD_CHARS + action_log_max_result_chars: int = DEFAULT_ACTION_LOG_MAX_RESULT_CHARS # >= MIN_ACTION_LOG_FIELD_CHARS def __post_init__(self) -> None: # Accept plain strings (e.g. constructed from JSON) and coerce to the enums. @@ -157,6 +163,10 @@ def __post_init__(self) -> None: raise ValueError( f"action_log_max_lines must be an integer >= {MIN_ACTION_LOG_MAX_LINES}, got {lines!r}" ) + for name in ("action_log_max_args_chars", "action_log_max_result_chars"): + chars = getattr(self, name) + if isinstance(chars, bool) or not isinstance(chars, int) or chars < MIN_ACTION_LOG_FIELD_CHARS: + raise ValueError(f"{name} must be an integer >= {MIN_ACTION_LOG_FIELD_CHARS}, got {chars!r}") def to_dict(self) -> dict: return { @@ -172,6 +182,8 @@ def to_dict(self) -> dict: "evidence_exclude_pattern": self.evidence_exclude_pattern, "always_show_actions": dict(self.always_show_actions) if self.always_show_actions is not None else None, "action_log_max_lines": self.action_log_max_lines, + "action_log_max_args_chars": self.action_log_max_args_chars, + "action_log_max_result_chars": self.action_log_max_result_chars, } @classmethod @@ -189,6 +201,8 @@ def from_dict(cls, data: dict) -> TrajectoryFilter: evidence_exclude_pattern=data.get("evidence_exclude_pattern"), always_show_actions=data.get("always_show_actions"), action_log_max_lines=data.get("action_log_max_lines", DEFAULT_ACTION_LOG_MAX_LINES), + action_log_max_args_chars=data.get("action_log_max_args_chars", DEFAULT_ACTION_LOG_MAX_ARGS_CHARS), + action_log_max_result_chars=data.get("action_log_max_result_chars", DEFAULT_ACTION_LOG_MAX_RESULT_CHARS), ) def compact_otel_trajectory( diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 9a04ac0e..94758795 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -864,7 +864,9 @@ def _compact_screenshot( image_blocks = [_image_url_block(f) for f in final] attached = [f.data for f in final] if action_log_text: - inline_text = action_log(transcript.actions, messages=transcript.messages, max_lines=tf.action_log_max_lines) + inline_text = action_log(transcript.actions, messages=transcript.messages, max_lines=tf.action_log_max_lines, + max_args_chars=tf.action_log_max_args_chars, + max_result_chars=tf.action_log_max_result_chars) else: inline_text = strip_trajectory_images(raw_text, attached=attached) # A trajectory with no screenshots selects nothing: the judge gets the text alone, with no frame rules. diff --git a/tst/unit/task_step/verifiers/judge_utils/test_frame_selection_cli_formats.py b/tst/unit/task_step/verifiers/judge_utils/test_frame_selection_cli_formats.py new file mode 100644 index 00000000..a7ac773b --- /dev/null +++ b/tst/unit/task_step/verifiers/judge_utils/test_frame_selection_cli_formats.py @@ -0,0 +1,104 @@ +"""transcript_from_raw on coding-agent CLI trajectories (stream-json rows, OpenCode events) and the action log's cuts.""" +from __future__ import annotations + +import json + +from agent_env.task_step.task_steps.verifiers.judge_utils import frame_selection as FS + + +def _assistant(*blocks: dict) -> dict: + return {"type": "assistant", "message": {"content": list(blocks)}} + + +def _user(*blocks: dict) -> dict: + return {"type": "user", "message": {"content": list(blocks)}} + + +def _use(tid: str, name: str, args: dict) -> dict: + return {"type": "tool_use", "id": tid, "name": name, "input": args} + + +def _res(tid: str, content: object, is_error: bool = False) -> dict: + return {"type": "tool_result", "tool_use_id": tid, "content": content, "is_error": is_error} + + +def _stream_run() -> str: + return json.dumps([ + {"type": "system", "subtype": "init", "tools": ["Bash"]}, + _assistant({"type": "text", "text": "Looking up the issue first."}, + _use("t1", "mcp__gw__linear_get_issue", {"id": "ENG-1"})), + _user(_res("t1", [{"type": "text", "text": '{"state": "Todo"}'}, {"type": "image", "source": {}}])), + _assistant(_use("t2", "mcp__gw__linear_update_issue", {"id": "ENG-1", "state": "Done"})), + _user(_res("t2", "permission denied", is_error=True)), + _assistant(_use("t3", "mcp__gw__gmail_send", {"to": "a@b.c"})), + _user(_res("t9", "an orphan result with no call")), + {"type": "result", "result": "Updated ENG-1."}, + ]) + + +def test_stream_json_pairs_each_call_with_its_result_by_id_in_run_order(): + t = FS.transcript_from_raw(_stream_run()) + assert [(a.index, a.tool, a.args) for a in t.actions] == [ + (1, "mcp__gw__linear_get_issue", {"id": "ENG-1"}), + (2, "mcp__gw__linear_update_issue", {"id": "ENG-1", "state": "Done"}), + (3, "mcp__gw__gmail_send", {"to": "a@b.c"})] + assert t.actions[0].result == '{"state": "Todo"}\n' + assert t.actions[1].result == {"error": "permission denied"} + assert t.actions[2].result is None and all(a.screenshot is None for a in t.actions) + assert t.messages == [FS.AgentMessage(0, "Looking up the issue first.")] + + +def test_stream_json_action_log_shows_calls_results_and_the_agent_text(): + t = FS.transcript_from_raw(_stream_run()) + lines = FS.action_log(t.actions, messages=t.messages).splitlines() + assert lines[0] == 'agent said: "Looking up the issue first."' + assert lines[1].startswith('1. mcp__gw__linear_get_issue {"id": "ENG-1"} -> ') + assert lines[2] == ('2. mcp__gw__linear_update_issue {"id": "ENG-1", "state": "Done"}' + ' -> {"error": "permission denied"}') + assert lines[3] == '3. mcp__gw__gmail_send {"to": "a@b.c"}' and "Updated ENG-1" not in "\n".join(lines) + + +def _part(pid: str, tool: str, state: dict) -> dict: + return {"type": "tool_use", "part": {"id": pid, "type": "tool", "tool": tool, "state": state}} + + +def test_opencode_tool_parts_become_actions_with_output_or_error_and_drop_the_hex_prefix(): + raw = json.dumps([ + {"part": {"type": "text", "text": "Checking the sheet."}}, + _part("p1", "mcp_3fa9c0_sheets_read", {"status": "running", "input": {"range": "A1:B2"}}), + _part("p1", "mcp_3fa9c0_sheets_read", {"status": "completed", "input": {"range": "A1:B2"}, "output": "2 rows"}), + _part("p2", "mcp_3fa9c0_slack_post", {"status": "error", "input": {"channel": "#ops"}, "error": "not_in_channel"}), + {"part": {"type": "step-finish", "reason": "stop"}}, + ]) + t = FS.transcript_from_raw(raw) + assert [(a.index, a.tool, a.args, a.result) for a in t.actions] == [ + (1, "sheets_read", {"range": "A1:B2"}, "2 rows"), + (2, "slack_post", {"channel": "#ops"}, {"error": "not_in_channel"})] + assert t.messages == [FS.AgentMessage(0, "Checking the sheet.")] + + +def test_unrecognised_lists_still_give_an_empty_transcript(): + for raw in (json.dumps([{"foo": 1}, 3, "x"]), json.dumps([]), json.dumps([{"type": "other"}])): + assert FS.transcript_from_raw(raw) == FS.Transcript([], []) + + +def test_the_action_log_cuts_args_and_results_at_the_given_widths_and_keeps_the_defaults_otherwise(): + acts = [FS.TrajectoryAction(index=1, tool="docs_write", args={"body": "a" * 500}, screenshot=None, + result="r" * 500)] + args_part, result_part = FS.action_log(acts).split(" -> ") + assert len(args_part) == len("1. docs_write ") + FS.MAX_ARGS_CHARS + 1 and len(result_part) == FS.MAX_RESULT_CHARS + 1 + args_part, result_part = FS.action_log(acts, max_args_chars=300, max_result_chars=400).split(" -> ") + assert len(args_part) == len("1. docs_write ") + 300 + 1 and len(result_part) == 400 + 1 + assert FS.rendered_action(acts[0]) == "docs_write " + FS._render_json(acts[0].args, FS.MAX_ARGS_CHARS) + + +def test_json_wrapped_in_json_text_is_decoded_so_the_log_is_not_mostly_escapes(): + inner = json.dumps({"id": "usr_1", "name": "alice", "teams": [{"id": "team_a"}]}) + raw = json.dumps([_assistant(_use("t1", "linear_get_user", {"query": "me"})), + _user(_res("t1", json.dumps({"result": inner}))), + _assistant(_use("t2", "notes", {})), + _user(_res("t2", "{not json at all}"))]) + t = FS.transcript_from_raw(raw) + assert t.actions[0].result == {"result": {"id": "usr_1", "name": "alice", "teams": [{"id": "team_a"}]}} + assert t.actions[1].result == "{not json at all}" + assert "\\" not in FS.action_log(t.actions, max_result_chars=500) diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_per_criterion.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_per_criterion.py index 65fd9228..3186ca39 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_per_criterion.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_per_criterion.py @@ -28,8 +28,11 @@ trajectory_mistakes_key, ) from agent_env.task_step.task_steps.verifiers.judge_utils.trajectory_filter import ( + DEFAULT_ACTION_LOG_MAX_ARGS_CHARS, DEFAULT_ACTION_LOG_MAX_LINES, + DEFAULT_ACTION_LOG_MAX_RESULT_CHARS, MAX_FRAME_BUDGET, + MIN_ACTION_LOG_FIELD_CHARS, MIN_ACTION_LOG_MAX_LINES, MIN_FRAME_BUDGET, CompactionType, @@ -154,6 +157,20 @@ def test_action_log_line_cap_round_trips_and_is_validated(): _filter(action_log_max_lines=MIN_ACTION_LOG_MAX_LINES) +def test_action_log_field_cuts_round_trip_and_are_validated(): + f = _filter(action_log_max_args_chars=300, action_log_max_result_chars=900) + back = TrajectoryFilter.from_dict(f.to_dict()) + assert (back.action_log_max_args_chars, back.action_log_max_result_chars) == (300, 900) + default = TrajectoryFilter.from_dict({"compaction_type": "screenshot"}) + assert (default.action_log_max_args_chars, default.action_log_max_result_chars) == ( + DEFAULT_ACTION_LOG_MAX_ARGS_CHARS, DEFAULT_ACTION_LOG_MAX_RESULT_CHARS) + for name in ("action_log_max_args_chars", "action_log_max_result_chars"): + for bad in (MIN_ACTION_LOG_FIELD_CHARS - 1, 0, "300", 300.0, True): + with pytest.raises(ValueError, match=name): + _filter(**{name: bad}) + _filter(**{name: MIN_ACTION_LOG_FIELD_CHARS}) + + # --------------------------------------------------------------------------- execute wiring def _verifier(**kw) -> RubricsVerifierTaskStep: @@ -613,3 +630,30 @@ def test_evidence_correction_prompt_asks_for_the_evidence_fields_and_the_binary_ format_evidence_judge_correction_prompt(**{**kw, "discrepancy": parse}) spec = get_judge_output_format_spec("rubric_evidence") assert spec.format_correction_prompt is format_evidence_judge_correction_prompt + + +def _cli_run(result_len: int) -> str: + """A coding-agent CLI trajectory (stream-json rows), no screenshots: one tool call with a long result.""" + return json.dumps([ + {"type": "system", "subtype": "init"}, + {"type": "assistant", "message": {"content": [ + {"type": "text", "text": "Closing the ticket."}, + {"type": "tool_use", "id": "t1", "name": "linear_update_issue", "input": {"id": "ENG-1", "state": "Done"}}]}}, + {"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "x" * result_len}]}}, + {"type": "result", "result": "Closed ENG-1."}, + ]) + + +@pytest.mark.asyncio +async def test_a_cli_trajectory_reaches_the_direct_judge_as_an_action_log_cut_at_the_filter_widths(monkeypatch): + tf = TrajectoryFilter(compaction_type=CompactionType.SCREENSHOT, trajectory_text=TrajectoryText.ACTION_LOG, + action_log_max_result_chars=500) + v = _verifier(output_format=JudgeOutputFormat.RUBRIC_BINARY, grading_policy_prompt=None, trajectory_filter=tf) + captured = _capture_judge(monkeypatch, v, _passing_list) + monkeypatch.setattr(v, "_read_trajectory_text", lambda uri: _cli_run(2000)) + await v.execute(_ctx()) + text = captured["eval_prompt"] + assert 'agent said: "Closing the ticket."' in text + assert '1. linear_update_issue {"id": "ENG-1", "state": "Done"} -> "' + "x" * 499 + "…" in text + assert "x" * 600 not in text and captured["image_blocks"] == [] From 13956265dceace368fcfdce3d6b55ced5fc50dac Mon Sep 17 00:00:00 2001 From: t Date: Thu, 15 Oct 2026 18:26:55 -0400 Subject: [PATCH 2/3] fix(verifiers): fail the action log on a trajectory format it cannot read transcript_from_raw returned an empty transcript for JSON in no format it reads, so a direct judge with trajectory_text=action_log graded an empty log without any sign that it could not see the run. Transcript now records its source format (otel / stream-json / opencode, empty, malformed), and the verifier raises when the action log is asked to read unrecognised JSON. Text that is not JSON at all stays a failed run, as before. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/judge_utils/frame_selection.py | 20 +++++++++++------- .../task_steps/verifiers/rubrics_verifier.py | 7 +++++++ .../test_frame_selection_cli_formats.py | 10 +++++++++ .../test_rubrics_verifier_per_criterion.py | 21 +++++++++++++++++++ 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py index be3279a1..52e6b629 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py @@ -11,7 +11,7 @@ import json import re from collections.abc import Iterable, Mapping -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from agent_env.task_step.task_steps.verifiers.judge_utils.trajectory_filter import ( DEFAULT_ACTION_LOG_MAX_ARGS_CHARS, @@ -70,10 +70,13 @@ class AgentMessage: @dataclass(frozen=True) class Transcript: - """What the RAW trajectory records the agent doing (``actions``) and saying (``messages``).""" + """What the RAW trajectory records the agent doing (``actions``) and saying (``messages``). ``source`` + names the format it was read from (``otel`` / ``stream-json`` / ``opencode``; ``empty`` for an empty + list, ``malformed`` for text that is not JSON); None means JSON in no format this module reads.""" actions: list[TrajectoryAction] messages: list[AgentMessage] + source: str | None = field(default=None, compare=False) @dataclass(frozen=True) @@ -110,20 +113,23 @@ def _text_blocks(completion: object) -> list[str]: def transcript_from_raw(raw_text: str) -> Transcript: """The tool calls and agent text of a raw trajectory, in run order. Reads OTel span lists, the stream-json rows of coding-agent CLIs (``type`` system / assistant / user / result) and OpenCode event - lists (``part.type`` tool / text). Malformed or unrecognised input gives an empty transcript.""" + lists (``part.type`` tool / text). Malformed or unrecognised input gives an empty transcript; ``source`` + tells the two apart, and both from a run that recorded nothing.""" try: rows = json.loads(raw_text) except (json.JSONDecodeError, TypeError): - return Transcript([], []) + return Transcript([], [], source="malformed") if not isinstance(rows, list): return Transcript([], []) + if not rows: + return Transcript([], [], source="empty") dicts = [r for r in rows if isinstance(r, dict)] if any(isinstance(r.get("attributes"), dict) for r in dicts): - return _transcript_from_otel(dicts) + return replace(_transcript_from_otel(dicts), source="otel") if any(r.get("type") in _STREAM_JSON_TYPES for r in dicts): - return _transcript_from_stream_json(dicts) + return replace(_transcript_from_stream_json(dicts), source="stream-json") if any(isinstance(r.get("part"), dict) for r in dicts): - return _transcript_from_opencode(dicts) + return replace(_transcript_from_opencode(dicts), source="opencode") return Transcript([], []) diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 94758795..78b2fce1 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -864,6 +864,13 @@ def _compact_screenshot( image_blocks = [_image_url_block(f) for f in final] attached = [f.data for f in final] if action_log_text: + if transcript.source is None: + # JSON the action log cannot read: grading its empty log would grade a run the judge cannot + # see. (Text that is not JSON at all stays a failed run, as before.) + raise RuntimeError( + f"trajectory_text=action_log, but the trajectory at {raw_uri} is in no format the action " + "log reads (OTel spans, coding-agent CLI stream-json, OpenCode events)" + ) inline_text = action_log(transcript.actions, messages=transcript.messages, max_lines=tf.action_log_max_lines, max_args_chars=tf.action_log_max_args_chars, max_result_chars=tf.action_log_max_result_chars) diff --git a/tst/unit/task_step/verifiers/judge_utils/test_frame_selection_cli_formats.py b/tst/unit/task_step/verifiers/judge_utils/test_frame_selection_cli_formats.py index a7ac773b..49918b43 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_frame_selection_cli_formats.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_frame_selection_cli_formats.py @@ -82,6 +82,16 @@ def test_unrecognised_lists_still_give_an_empty_transcript(): assert FS.transcript_from_raw(raw) == FS.Transcript([], []) +def test_the_transcript_names_its_source_format_and_none_when_unrecognised(): + assert FS.transcript_from_raw(json.dumps([{"name": "chat", "attributes": {}}])).source == "otel" + assert FS.transcript_from_raw(json.dumps([{"type": "system", "subtype": "init"}])).source == "stream-json" + assert FS.transcript_from_raw(json.dumps([{"part": {"type": "step-finish"}}])).source == "opencode" + assert FS.transcript_from_raw(json.dumps([])).source == "empty" + assert FS.transcript_from_raw("not json").source == "malformed" + for raw in (json.dumps({"a": 1}), json.dumps([{"foo": 1}, 3]), json.dumps([{"type": "other"}])): + assert FS.transcript_from_raw(raw).source is None + + def test_the_action_log_cuts_args_and_results_at_the_given_widths_and_keeps_the_defaults_otherwise(): acts = [FS.TrajectoryAction(index=1, tool="docs_write", args={"body": "a" * 500}, screenshot=None, result="r" * 500)] diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_per_criterion.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_per_criterion.py index 3186ca39..4f0e48f4 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_per_criterion.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_per_criterion.py @@ -657,3 +657,24 @@ async def test_a_cli_trajectory_reaches_the_direct_judge_as_an_action_log_cut_at assert 'agent said: "Closing the ticket."' in text assert '1. linear_update_issue {"id": "ENG-1", "state": "Done"} -> "' + "x" * 499 + "…" in text assert "x" * 600 not in text and captured["image_blocks"] == [] + + +@pytest.mark.asyncio +async def test_an_unrecognised_trajectory_format_fails_the_action_log_instead_of_grading_an_empty_log(monkeypatch): + tf = TrajectoryFilter(compaction_type=CompactionType.SCREENSHOT, trajectory_text=TrajectoryText.ACTION_LOG) + v = _verifier(output_format=JudgeOutputFormat.RUBRIC_BINARY, grading_policy_prompt=None, trajectory_filter=tf) + captured = _capture_judge(monkeypatch, v, _passing_list) + monkeypatch.setattr(v, "_read_trajectory_text", lambda uri: json.dumps([{"event": "tool", "name": "x"}])) + with pytest.raises(RuntimeError, match="no format the action log reads"): + await v.execute(_ctx()) + assert captured == {} + + +@pytest.mark.asyncio +async def test_an_empty_trajectory_still_reaches_the_judge_with_an_empty_action_log(monkeypatch): + tf = TrajectoryFilter(compaction_type=CompactionType.SCREENSHOT, trajectory_text=TrajectoryText.ACTION_LOG) + v = _verifier(output_format=JudgeOutputFormat.RUBRIC_BINARY, grading_policy_prompt=None, trajectory_filter=tf) + captured = _capture_judge(monkeypatch, v, _passing_list) + monkeypatch.setattr(v, "_read_trajectory_text", lambda uri: "[]") + await v.execute(_ctx()) + assert "eval_prompt" in captured From 37c7fb1d4435fdb42a7d2539f6927e610dae3e8f Mon Sep 17 00:00:00 2001 From: t Date: Thu, 15 Oct 2026 18:30:47 -0400 Subject: [PATCH 3/3] fix(verifiers): keep OpenCode error reasons in output and empty CLI results - An OpenCode tool in status "error" with its reason in `output` (no `error` field) rendered as {"error": null}. Fall back to `output`. - A stream-json result of "[]" or "{}" decoded to an empty collection and was then dropped as no result, so the judge could not tell a search with no matches from a call with no recorded result. Only an empty text result now counts as none. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/judge_utils/frame_selection.py | 7 +++++-- .../test_frame_selection_cli_formats.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py index 52e6b629..bfb7e227 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/frame_selection.py @@ -194,7 +194,8 @@ def _transcript_from_stream_json(rows: list[dict]) -> Transcript: elif row.get("type") == "user" and kind == "tool_result" and str(block.get("tool_use_id")) in position: i = position[str(block["tool_use_id"])] body = _decoded(_tool_result_text(block.get("content"))) - result = {"error": body} if block.get("is_error") else (body or None) + # An empty list or object is a real result (a search with no matches); only no text is none. + result = {"error": body} if block.get("is_error") else (None if body == "" else body) actions[i] = replace(actions[i], result=result) return Transcript(actions, messages) @@ -215,7 +216,9 @@ def _transcript_from_opencode(events: list[dict]) -> Transcript: state = part.get("state") if isinstance(part.get("state"), dict) else {} args = state.get("input") if isinstance(state.get("input"), dict) else {} if state.get("status") == "error": - result = {"error": _decoded(state.get("error"))} + # OpenCode puts the reason in ``error`` or, for some tools, in ``output``. + reason = state.get("error") if state.get("error") not in ("", None) else state.get("output") + result = {"error": _decoded(reason)} else: result = _decoded(state.get("output")) if state.get("output") not in ("", None) else None tool = _HEX_TOOL_PREFIX.sub("", str(part.get("tool") or "")) diff --git a/tst/unit/task_step/verifiers/judge_utils/test_frame_selection_cli_formats.py b/tst/unit/task_step/verifiers/judge_utils/test_frame_selection_cli_formats.py index 49918b43..0cef57ea 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_frame_selection_cli_formats.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_frame_selection_cli_formats.py @@ -77,6 +77,20 @@ def test_opencode_tool_parts_become_actions_with_output_or_error_and_drop_the_he assert t.messages == [FS.AgentMessage(0, "Checking the sheet.")] +def test_an_opencode_error_with_its_reason_in_output_keeps_the_reason(): + raw = json.dumps([_part("p1", "crm_search", {"status": "error", "input": {"name": "x"}, + "output": "Error: company not found"})]) + assert FS.transcript_from_raw(raw).actions[0].result == {"error": "Error: company not found"} + + +def test_an_empty_stream_json_result_is_kept_apart_from_no_result(): + raw = json.dumps([_assistant(_use("t1", "search", {"q": "x"}), _use("t2", "lookup", {"id": 1})), + _user(_res("t1", "[]"), _res("t2", ""))]) + first, second = FS.transcript_from_raw(raw).actions + assert first.result == [] and second.result is None + assert "search" in FS.action_log([first]) and "-> []" in FS.action_log([first]) + + def test_unrecognised_lists_still_give_an_empty_transcript(): for raw in (json.dumps([{"foo": 1}, 3, "x"]), json.dumps([]), json.dumps([{"type": "other"}])): assert FS.transcript_from_raw(raw) == FS.Transcript([], [])