feat(gooddata-eval): capture agent reasoning steps in ChatResult

packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py

@@ -170,6 +170,7 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult:
         "alertProposals": acc.alert_proposals,
         "toolCallEvents": acc.tool_call_events,
         "reasoningStepCount": len(acc.reasoning_steps),
+        "reasoningSteps": [step["summary"] for step in acc.reasoning_steps],
     }
     if acc.visualizations:
         payload["createdVisualizations"] = {

packages/gooddata-eval/src/gooddata_eval/core/models.py

@@ -98,6 +98,7 @@ class ChatResult(BaseModel):
     alert_proposals: list[dict] = Field(default_factory=list, alias="alertProposals")
     tool_call_events: list[ToolCallEvent] = Field(default_factory=list, alias="toolCallEvents")
     reasoning_step_count: int = Field(default=0, alias="reasoningStepCount")
+    reasoning_steps: list[str] = Field(default_factory=list, alias="reasoningSteps")
     conversation_id: str | None = Field(default=None, alias="conversationId")
     response_id: str | None = Field(default=None, alias="responseId")
 

packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py

@@ -36,6 +36,7 @@ def _build_run_dict(report: EvalReport) -> dict:
                 "detail": item.best_detail,
                 "conversation_id": item.conversation_id,
                 "response_id": item.response_id,
+                "reasoning": item.reasoning_steps,
             }
             for item in report.items
         },

packages/gooddata-eval/src/gooddata_eval/core/runner.py

@@ -33,6 +33,7 @@ class ItemReport:
     best_detail: dict = field(default_factory=dict)
     conversation_id: str | None = None
     response_id: str | None = None
+    reasoning_steps: list[str] = field(default_factory=list)
 
     @property
     def avg_latency_s(self) -> float:
@@ -116,6 +117,7 @@ def _run_one_item(
             chat_result = backend.ask(item)
             report.conversation_id = getattr(chat_result, "conversation_id", None) or report.conversation_id
             report.response_id = getattr(chat_result, "response_id", None) or report.response_id
+            report.reasoning_steps = getattr(chat_result, "reasoning_steps", None) or report.reasoning_steps
             evaluation = evaluator.evaluate(item, chat_result)
             latency = time.perf_counter() - t0
             report.runs += 1

packages/gooddata-eval/tests/test_reporting.py

@@ -23,6 +23,7 @@ def _report() -> EvalReport:
                 pass_at_k=True,
                 runs=2,
                 latency_s=2.5,
+                reasoning_steps=["step one", "step two"],
             ),
             ItemReport(
                 id="i2",
@@ -48,6 +49,8 @@ def test_build_json_report_keyed_by_item_id():
     assert data["items"]["i1"]["pass_at_k"] is True
     assert data["items"]["i1"]["latency_s"] == 2.5
     assert data["items"]["i1"]["avg_latency_s"] == 1.25
+    assert data["items"]["i1"]["reasoning"] == ["step one", "step two"]
+    assert data["items"]["i2"]["reasoning"] == []
 
 
 def test_write_json_report_creates_file(tmp_path):

packages/gooddata-eval/tests/test_runner.py

@@ -258,6 +258,41 @@ def ask(self, item: DatasetItem) -> ChatResult:
     assert "conversation_id" not in report.items[0].error
 
 
+def test_run_items_carries_reasoning_steps_from_chat_result():
+    """reasoning_steps from the ChatResult surfaces on the item report, same as conversation_id."""
+
+    class _ReasoningBackend:
+        def ask(self, item: DatasetItem) -> ChatResult:
+            return ChatResult.model_validate(
+                {"textResponse": "which metric?", "reasoningSteps": ["step one", "step two"]}
+            )
+
+    report = run_items([_item()], _ReasoningBackend(), runs=1)
+    assert report.items[0].reasoning_steps == ["step one", "step two"]
+
+
+def test_run_items_reasoning_steps_empty_when_chat_result_has_none():
+    backend = _FakeBackend([_empty_chat()])
+    report = run_items([_item()], backend, runs=1)
+    assert report.items[0].reasoning_steps == []
+
+
+def test_run_items_reasoning_steps_keeps_earlier_run_when_later_run_is_empty():
+    """A later run with no reasoning events must not clobber an earlier run's steps (runner.py:120's `or`)."""
+
+    class _MixedReasoningBackend:
+        def __init__(self):
+            self.calls = 0
+
+        def ask(self, item: DatasetItem) -> ChatResult:
+            self.calls += 1
+            steps = ["step one", "step two"] if self.calls == 1 else []
+            return ChatResult.model_validate({"textResponse": "answer", "reasoningSteps": steps})
+
+    report = run_items([_item()], _MixedReasoningBackend(), runs=2)
+    assert report.items[0].reasoning_steps == ["step one", "step two"]
+
+
 def test_run_items_callback_exception_is_logged_not_swallowed(capsys):
     """A raising callback prints a traceback to stderr but the run continues."""
     backend = _FakeBackend([_chat_with(_viz_obj())] * 2)

packages/gooddata-eval/tests/test_sse_client.py

@@ -60,9 +60,17 @@ def test_parse_sse_lines_counts_reasoning_steps():
     ]
     result = parse_sse_lines(lines)
     assert result.reasoning_step_count == 2
+    assert result.reasoning_steps == ["step one", "step two"]
     assert result.text_response == "Done"
 
 
+def test_parse_sse_lines_reasoning_steps_empty_when_no_reasoning_events():
+    lines = ['data: {"item": {"role": "assistant", "content": {"type": "text", "text": "Done"}}}']
+    result = parse_sse_lines(lines)
+    assert result.reasoning_step_count == 0
+    assert result.reasoning_steps == []
+
+
 def test_parse_sse_lines_prefers_multipart_viz_over_adhoc_fallback():
     """Real multipart visualization takes priority over adhoc tool call stash."""