Skip to content

Commit 110be83

Browse files
hkad98hkad98
committed
fix(gooddata-eval): address review of the reasoning-effort option
Review findings on the initial commit: - `reasoning_effort` sat mid-signature in the seven public `evaluate_agentic_*` functions. This is a released package, so an external positional caller would silently rebind a later argument to it on upgrade. Moved to the end of each signature; `run_agentic_items` and `_dispatch_agentic` are keyword-only and keep their position. - Only the agentic Langfuse writer recorded the effort. The single-turn `LangfuseSink` path built `gd-eval-{ts}-{model}` with minute granularity, so two runs differing only by effort collided on one indistinguishable run — the failure the option exists to prevent. The run name now carries the effort, and the sink reports it via trace `tags` (metadata is not a breakdown dimension, per the note already in that file) and dataset-run metadata. - Typed the value `ReasoningEffort = Literal["LOW","MEDIUM","HIGH"]` instead of bare `str`, so an invalid effort fails locally rather than as an opaque 422 after a full run. The CLI derives its `choices` from the alias so the two cannot drift. - Added tests for the run-name/metadata behaviour and the CLI wiring, both of which were previously unasserted. - Documented the flag in the README, including the feature-flag dependency and that summary items are unaffected. - Folded the duplicate test client-builder into the existing `_client_with_handler` via `**kwargs`, and dropped an unused local in `test_cli.py` that was failing lint on master.
1 parent e368629 commit 110be83

17 files changed

Lines changed: 168 additions & 32 deletions

‎packages/gooddata-eval/README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ Both provider name and provider id are accepted as the prefix.
9292
|---|---|---|
9393
| `--runs K` | `2` | Independent runs per item (pass@K). An item passes if any run passes. |
9494
| `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests. Progress output interleaves when K > 1. |
95+
| `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` items go through the summary endpoint, which has no such option. |
9596

9697
#### Output
9798

@@ -104,7 +105,7 @@ Both provider name and provider id are accepted as the prefix.
104105

105106
| Flag | Description |
106107
|---|---|
107-
| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. |
108+
| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`, suffixed `-effort-{level}` when `--reasoning-effort` is set so runs differing only by effort stay separate). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. |
108109

109110
### JSON report shape
110111

‎packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
from gooddata_eval.core.agentic.metric_skill import evaluate_agentic_metric_skill
1515
from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
1616
from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
17+
from gooddata_eval.core.config import ReasoningEffort
1718
from gooddata_eval.core.models import CreatedVisualization, DatasetItem
1819
from gooddata_eval.core.runner import EvalReport, ItemReport
1920

@@ -24,7 +25,7 @@ class _LfKw(TypedDict, total=False):
2425
dataset_name: str
2526
run_timestamp: str
2627
model_version_override: str | None
27-
reasoning_effort: str | None
28+
reasoning_effort: ReasoningEffort | None
2829

2930

3031
AGENTIC_TEST_KINDS = frozenset(
@@ -81,7 +82,7 @@ def _dispatch_agentic(
8182
langfuse: Any,
8283
run_ts: str,
8384
model_version_override: str | None,
84-
reasoning_effort: str | None = None,
85+
reasoning_effort: ReasoningEffort | None = None,
8586
) -> None:
8687
"""Call the appropriate evaluate_agentic_* function for the item's test_kind."""
8788
kind = item.test_kind
@@ -179,7 +180,7 @@ def run_agentic_items(
179180
*,
180181
k: int = 2,
181182
model_version: str | None = None,
182-
reasoning_effort: str | None = None,
183+
reasoning_effort: ReasoningEffort | None = None,
183184
use_langfuse: bool = False,
184185
run_ts: str,
185186
on_item_start: Any = None,

‎packages/gooddata-eval/src/gooddata_eval/cli/main.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import threading
77
from datetime import datetime, timezone
88
from pathlib import Path
9+
from typing import get_args
910

1011
import httpx
1112
from gooddata_api_client.exceptions import ApiException
@@ -14,7 +15,7 @@
1415

1516
from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items
1617
from gooddata_eval.core.chat.sse_client import ChatClient
17-
from gooddata_eval.core.config import RunConfig
18+
from gooddata_eval.core.config import ReasoningEffort, RunConfig
1819
from gooddata_eval.core.connection import ConnectionError_, resolve_connection
1920
from gooddata_eval.core.dataset.local import load_local_dataset
2021
from gooddata_eval.core.langfuse.sink import LangfuseSink
@@ -107,7 +108,7 @@ def _build_parser() -> argparse.ArgumentParser:
107108
run.add_argument(
108109
"--reasoning-effort",
109110
dest="reasoning_effort",
110-
choices=["LOW", "MEDIUM", "HIGH"],
111+
choices=list(get_args(ReasoningEffort)),
111112
help="Reasoning effort requested per message. Requires the enableGenAiReasoningEffort "
112113
"feature flag on the target organization; without it the server ignores the value.",
113114
)
@@ -299,6 +300,10 @@ def _run(config: RunConfig) -> int:
299300
progress_console.print(f"Provider={provider_display}, model={resolved.model_id}{switched}")
300301

301302
run_name = f"gd-eval-{run_ts}-{resolved.model_id}"
303+
if config.reasoning_effort:
304+
# Without this two runs differing only by effort share a name and are
305+
# indistinguishable in the report, which is the comparison this exists for.
306+
run_name = f"{run_name}-effort-{config.reasoning_effort.lower()}"
302307
if progress_console and config.log_to_langfuse:
303308
progress_console.print(f"Logging to Langfuse run '{run_name}'...")
304309

@@ -314,6 +319,7 @@ def _run(config: RunConfig) -> int:
314319
run_name=run_name,
315320
model_id=resolved.model_id,
316321
provider_type=resolved.provider_type,
322+
reasoning_effort=config.reasoning_effort,
317323
)
318324

319325
def on_langfuse_item_done(

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515

1616
import httpx
1717

18+
from gooddata_eval.core.config import ReasoningEffort
19+
1820
_log = logging.getLogger(__name__)
1921

2022
# ---------------------------------------------------------------------------
@@ -384,7 +386,7 @@ def build_run_context(
384386
run_timestamp: str | None,
385387
model_version_override: str | None,
386388
run_metadata_extra: dict[str, Any] | None = None,
387-
reasoning_effort: str | None = None,
389+
reasoning_effort: ReasoningEffort | None = None,
388390
) -> tuple[str, dict[str, Any]]:
389391
"""Return (run_name_base, run_metadata) with model version resolved from workspace API.
390392

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
from gooddata_eval.core.agentic._catalog import CatalogMetricAlert
1515
from gooddata_eval.core.chat.sse_client import ChatClient
16+
from gooddata_eval.core.config import ReasoningEffort
1617
from gooddata_eval.core.models import ToolCallEvent
1718

1819
try:
@@ -342,7 +343,7 @@ def run_agentic_alert_skill(
342343
k: int = _DEFAULT_K,
343344
max_iterations: int = _DEFAULT_MAX_ITERATIONS,
344345
initial_conversation_id: str | None = None,
345-
reasoning_effort: str | None = None,
346+
reasoning_effort: ReasoningEffort | None = None,
346347
) -> AgenticAlertSummary:
347348
"""Run the alert-skill agentic evaluation K times and return a summary."""
348349
expected = _normalize_expected_output(expected_output)
@@ -457,13 +458,13 @@ def evaluate_agentic_alert_skill(
457458
k: int = _DEFAULT_K,
458459
max_iterations: int = _DEFAULT_MAX_ITERATIONS,
459460
initial_conversation_id: str | None = None,
460-
reasoning_effort: str | None = None,
461461
langfuse: object | None = None,
462462
dataset_item_id: str = "",
463463
dataset_name: str = "alert_skill",
464464
run_timestamp: str | None = None,
465465
model_version_override: str | None = None,
466466
run_metadata_extra: dict | None = None,
467+
reasoning_effort: ReasoningEffort | None = None,
467468
) -> None:
468469
"""Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
469470
from datetime import datetime as _dt # noqa: PLC0415

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
from gooddata_eval.core.agentic.alert_skill import render_alert_proposal
1515
from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids
1616
from gooddata_eval.core.chat.sse_client import ChatClient
17+
from gooddata_eval.core.config import ReasoningEffort
1718
from gooddata_eval.core.models import ChatResult, ToolCallEvent
1819
from gooddata_eval.core.scoring import (
1920
check_filters,
@@ -278,7 +279,7 @@ def run_agentic_conversation(
278279
fixture: ConversationFixture,
279280
max_clarification_turns: int = 20,
280281
initial_conversation_id: str | None = None,
281-
reasoning_effort: str | None = None,
282+
reasoning_effort: ReasoningEffort | None = None,
282283
) -> ConversationResult:
283284
"""Run a multi-turn, multi-skill conversation evaluation (no K-runs).
284285
@@ -398,13 +399,13 @@ def evaluate_agentic_conversation(
398399
fixture: ConversationFixture,
399400
max_clarification_turns: int = 20,
400401
initial_conversation_id: str | None = None,
401-
reasoning_effort: str | None = None,
402402
langfuse: object | None = None,
403403
dataset_item_id: str = "",
404404
dataset_name: str = "conversation",
405405
run_timestamp: str | None = None,
406406
model_version_override: str | None = None,
407407
run_metadata_extra: dict | None = None,
408+
reasoning_effort: ReasoningEffort | None = None,
408409
) -> None:
409410
"""Run conversation evaluation, log to Langfuse, and raise on failure."""
410411
from datetime import datetime as _dt # noqa: PLC0415

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from dataclasses import dataclass
77

88
from gooddata_eval.core.chat.sse_client import ChatClient
9+
from gooddata_eval.core.config import ReasoningEffort
910
from gooddata_eval.core.evaluators._llm_judge import LLMJudge
1011

1112
_DEFAULT_K = 1
@@ -71,7 +72,7 @@ def run_agentic_general_question(
7172
expected_output: str,
7273
k: int = _DEFAULT_K,
7374
initial_conversation_id: str | None = None,
74-
reasoning_effort: str | None = None,
75+
reasoning_effort: ReasoningEffort | None = None,
7576
) -> AgenticGeneralQuestionSummary:
7677
"""Run the general-question agentic evaluation K times and return a summary."""
7778
run_results: list[GeneralQuestionResult] = []
@@ -148,13 +149,13 @@ def evaluate_agentic_general_question(
148149
expected_output: str,
149150
k: int = _DEFAULT_K,
150151
initial_conversation_id: str | None = None,
151-
reasoning_effort: str | None = None,
152152
langfuse: object | None = None,
153153
dataset_item_id: str = "",
154154
dataset_name: str = "general_question",
155155
run_timestamp: str | None = None,
156156
model_version_override: str | None = None,
157157
run_metadata_extra: dict | None = None,
158+
reasoning_effort: ReasoningEffort | None = None,
158159
) -> None:
159160
"""Run general-question evaluation, log to Langfuse, and raise on failure."""
160161
from datetime import datetime as _dt # noqa: PLC0415

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from dataclasses import dataclass
77

88
from gooddata_eval.core.chat.sse_client import ChatClient
9+
from gooddata_eval.core.config import ReasoningEffort
910
from gooddata_eval.core.evaluators._llm_judge import LLMJudge
1011

1112
_DEFAULT_K = 1
@@ -68,7 +69,7 @@ def run_agentic_guardrail(
6869
expected_output: str,
6970
k: int = _DEFAULT_K,
7071
initial_conversation_id: str | None = None,
71-
reasoning_effort: str | None = None,
72+
reasoning_effort: ReasoningEffort | None = None,
7273
) -> AgenticGuardrailSummary:
7374
"""Run the guardrail agentic evaluation K times and return a summary."""
7475
run_results: list[GuardrailResult] = []
@@ -145,13 +146,13 @@ def evaluate_agentic_guardrail(
145146
expected_output: str,
146147
k: int = _DEFAULT_K,
147148
initial_conversation_id: str | None = None,
148-
reasoning_effort: str | None = None,
149149
langfuse: object | None = None,
150150
dataset_item_id: str = "",
151151
dataset_name: str = "guardrail",
152152
run_timestamp: str | None = None,
153153
model_version_override: str | None = None,
154154
run_metadata_extra: dict | None = None,
155+
reasoning_effort: ReasoningEffort | None = None,
155156
) -> None:
156157
"""Run guardrail evaluation, log to Langfuse, and raise on failure."""
157158
from datetime import datetime as _dt # noqa: PLC0415

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from gooddata_sdk import GoodDataSdk
1212

1313
from gooddata_eval.core.chat.sse_client import ChatClient
14+
from gooddata_eval.core.config import ReasoningEffort
1415
from gooddata_eval.core.models import ToolCallEvent
1516

1617
try:
@@ -232,7 +233,7 @@ def run_agentic_metric_skill(
232233
k: int = _DEFAULT_K,
233234
max_iterations: int = _DEFAULT_MAX_ITERATIONS,
234235
initial_conversation_id: str | None = None,
235-
reasoning_effort: str | None = None,
236+
reasoning_effort: ReasoningEffort | None = None,
236237
) -> AgenticMetricSummary:
237238
"""Run the metric-skill agentic evaluation K times and return a summary.
238239
@@ -295,13 +296,13 @@ def evaluate_agentic_metric_skill(
295296
k: int = _DEFAULT_K,
296297
max_iterations: int = _DEFAULT_MAX_ITERATIONS,
297298
initial_conversation_id: str | None = None,
298-
reasoning_effort: str | None = None,
299299
langfuse: object | None = None,
300300
dataset_item_id: str = "",
301301
dataset_name: str = "metric_skill",
302302
run_timestamp: str | None = None,
303303
model_version_override: str | None = None,
304304
run_metadata_extra: dict | None = None,
305+
reasoning_effort: ReasoningEffort | None = None,
305306
) -> None:
306307
"""Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure."""
307308
from datetime import datetime as _dt # noqa: PLC0415

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from dataclasses import dataclass
77

88
from gooddata_eval.core.chat.sse_client import ChatClient
9+
from gooddata_eval.core.config import ReasoningEffort
910
from gooddata_eval.core.models import ToolCallEvent
1011

1112
_DEFAULT_K = 1
@@ -67,7 +68,7 @@ def run_agentic_search_tool(
6768
expected_tool_call: dict,
6869
k: int = _DEFAULT_K,
6970
initial_conversation_id: str | None = None,
70-
reasoning_effort: str | None = None,
71+
reasoning_effort: ReasoningEffort | None = None,
7172
) -> AgenticSearchSummary:
7273
"""Run the search-tool agentic evaluation K times (single-turn each)."""
7374
run_results: list[SearchResult] = []
@@ -139,13 +140,13 @@ def evaluate_agentic_search_tool(
139140
expected_tool_call: dict,
140141
k: int = _DEFAULT_K,
141142
initial_conversation_id: str | None = None,
142-
reasoning_effort: str | None = None,
143143
langfuse: object | None = None,
144144
dataset_item_id: str = "",
145145
dataset_name: str = "search",
146146
run_timestamp: str | None = None,
147147
model_version_override: str | None = None,
148148
run_metadata_extra: dict | None = None,
149+
reasoning_effort: ReasoningEffort | None = None,
149150
) -> None:
150151
"""Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure."""
151152
from datetime import datetime as _dt # noqa: PLC0415

0 commit comments

Comments
 (0)