[bot] Merge master/78904a06 into rel/dev

packages/gooddata-eval/README.md

@@ -92,6 +92,7 @@ Both provider name and provider id are accepted as the prefix.
 |---|---|---|
 | `--runs K` | `2` | Independent runs per item (pass@K). An item passes if any run passes. |
 | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests. Progress output interleaves when K > 1. |
+| `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` items go through the summary endpoint, which has no such option. |
 
 #### Output
 
@@ -104,7 +105,7 @@ Both provider name and provider id are accepted as the prefix.
 
 | Flag | Description |
 |---|---|
-| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. |
+| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`, suffixed `-effort-{level}` when `--reasoning-effort` is set so runs differing only by effort stay separate). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. |
 
 ### JSON report shape
 

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -14,6 +14,7 @@
 from gooddata_eval.core.agentic.metric_skill import evaluate_agentic_metric_skill
 from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 
@@ -24,6 +25,7 @@ class _LfKw(TypedDict, total=False):
     dataset_name: str
     run_timestamp: str
     model_version_override: str | None
+    reasoning_effort: ReasoningEffort | None
 
 
 AGENTIC_TEST_KINDS = frozenset(
@@ -80,6 +82,7 @@ def _dispatch_agentic(
     langfuse: Any,
     run_ts: str,
     model_version_override: str | None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Call the appropriate evaluate_agentic_* function for the item's test_kind."""
     kind = item.test_kind
@@ -90,6 +93,7 @@ def _dispatch_agentic(
         "dataset_name": item.dataset_name,
         "run_timestamp": run_ts,
         "model_version_override": model_version_override,
+        "reasoning_effort": reasoning_effort,
     }
 
     if kind in ("vis_agentic", "agentic_visualization"):
@@ -176,6 +180,7 @@ def run_agentic_items(
     *,
     k: int = 2,
     model_version: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
     use_langfuse: bool = False,
     run_ts: str,
     on_item_start: Any = None,
@@ -202,7 +207,7 @@ def run_agentic_items(
         )
         t0 = time.perf_counter()
         try:
-            _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version)
+            _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort)
             item_report.pass_at_k = True
             item_report.runs = k
         except AssertionError as exc:

packages/gooddata-eval/src/gooddata_eval/cli/main.py

@@ -6,6 +6,7 @@
 import threading
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import get_args
 
 import httpx
 from gooddata_api_client.exceptions import ApiException
@@ -14,7 +15,7 @@
 
 from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items
 from gooddata_eval.core.chat.sse_client import ChatClient
-from gooddata_eval.core.config import RunConfig
+from gooddata_eval.core.config import ReasoningEffort, RunConfig
 from gooddata_eval.core.connection import ConnectionError_, resolve_connection
 from gooddata_eval.core.dataset.local import load_local_dataset
 from gooddata_eval.core.langfuse.sink import LangfuseSink
@@ -104,6 +105,13 @@ def _build_parser() -> argparse.ArgumentParser:
         dest="preserve_failed",
         help="Keep failed conversations on the server for post-mortem inspection.",
     )
+    run.add_argument(
+        "--reasoning-effort",
+        dest="reasoning_effort",
+        choices=list(get_args(ReasoningEffort)),
+        help="Reasoning effort requested per message. Requires the enableGenAiReasoningEffort "
+        "feature flag on the target organization; without it the server ignores the value.",
+    )
     run.add_argument(
         "--langfuse",
         action="store_true",
@@ -292,6 +300,10 @@ def _run(config: RunConfig) -> int:
                     progress_console.print(f"Provider={provider_display}, model={resolved.model_id}{switched}")
 
             run_name = f"gd-eval-{run_ts}-{resolved.model_id}"
+            if config.reasoning_effort:
+                # Without this two runs differing only by effort share a name and are
+                # indistinguishable in the report, which is the comparison this exists for.
+                run_name = f"{run_name}-effort-{config.reasoning_effort.lower()}"
             if progress_console and config.log_to_langfuse:
                 progress_console.print(f"Logging to Langfuse run '{run_name}'...")
 
@@ -307,6 +319,7 @@ def _run(config: RunConfig) -> int:
                     run_name=run_name,
                     model_id=resolved.model_id,
                     provider_type=resolved.provider_type,
+                    reasoning_effort=config.reasoning_effort,
                 )
 
                 def on_langfuse_item_done(
@@ -329,6 +342,7 @@ def on_langfuse_item_done(
                     workspace_id=config.workspace_id,
                     k=config.runs,
                     model_version=resolved.model_id,
+                    reasoning_effort=config.reasoning_effort,
                     use_langfuse=config.log_to_langfuse,
                     run_ts=run_ts,
                     on_item_start=on_item_start,
@@ -342,6 +356,7 @@ def on_langfuse_item_done(
                     token=config.token,
                     workspace_id=config.workspace_id,
                     preserve_failed=config.preserve_failed,
+                    reasoning_effort=config.reasoning_effort,
                 ),
                 SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
             )
@@ -433,6 +448,7 @@ def main(argv: list[str] | None = None) -> int:
             quiet=args.quiet,
             kind=args.kind,
             preserve_failed=args.preserve_failed,
+            reasoning_effort=args.reasoning_effort,
         )
         return _run(config)
     except (

packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py

@@ -15,6 +15,8 @@
 
 import httpx
 
+from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort
+
 _log = logging.getLogger(__name__)
 
 # ---------------------------------------------------------------------------
@@ -384,6 +386,7 @@ def build_run_context(
     run_timestamp: str | None,
     model_version_override: str | None,
     run_metadata_extra: dict[str, Any] | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> tuple[str, dict[str, Any]]:
     """Return (run_name_base, run_metadata) with model version resolved from workspace API.
 
@@ -399,15 +402,24 @@ def build_run_context(
             (e.g. a testing-framework tag or a CI run id for scoping). Default None keeps
             behavior unchanged. The SDK-derived model_version is applied last and cannot
             be overwritten by this dict.
+        reasoning_effort: Effort the run requested, stamped into both the run name and
+            the metadata so effort-varying runs stay comparable side by side.
     """
+    effort = normalize_reasoning_effort(reasoning_effort)
     model = get_model_version(host, token, workspace_id, model_version_override)
     ts = run_timestamp or datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
     base = f"{dataset_name}_{ts}"
     if model:
         base = f"{base}_{model}"
+    # Part of the run name, not just metadata: two runs that differ only by effort would
+    # otherwise collide on the same name and be indistinguishable in the report.
+    if effort:
+        base = f"{base}_effort-{effort.lower()}"
     # Caller supplies its own run tags (e.g. testing_framework); model_version is applied
     # last so the SDK-derived value cannot be overwritten by run_metadata_extra.
     metadata: dict[str, Any] = dict(run_metadata_extra) if run_metadata_extra else {}
+    if effort:
+        metadata["reasoning_effort"] = effort
     if model:
         metadata["model_version"] = model
     return base, metadata

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -13,6 +13,7 @@
 
 from gooddata_eval.core.agentic._catalog import CatalogMetricAlert
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import ToolCallEvent
 
 try:
@@ -342,11 +343,12 @@ def run_agentic_alert_skill(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticAlertSummary:
     """Run the alert-skill agentic evaluation K times and return a summary."""
     expected = _normalize_expected_output(expected_output)
     run_results: list[AlertRunResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     sdk = GoodDataSdk.create(host, token)
 
     def _run_once(conv_id: str) -> AlertRunResult:
@@ -462,6 +464,7 @@ def evaluate_agentic_alert_skill(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -481,6 +484,7 @@ def evaluate_agentic_alert_skill(
         k=k,
         max_iterations=max_iterations,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -500,6 +504,7 @@ def evaluate_agentic_alert_skill(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -14,6 +14,7 @@
 from gooddata_eval.core.agentic.alert_skill import render_alert_proposal
 from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import ChatResult, ToolCallEvent
 from gooddata_eval.core.scoring import (
     check_filters,
@@ -278,14 +279,15 @@ def run_agentic_conversation(
     fixture: ConversationFixture,
     max_clarification_turns: int = 20,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> ConversationResult:
     """Run a multi-turn, multi-skill conversation evaluation (no K-runs).
 
     A single conversation is used for all turns in the fixture.  Each turn may
     trigger up to *max_clarification_turns* additional rounds of simulated-user
     replies before the agent produces the expected output.
     """
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     sdk = GoodDataSdk.create(host, token)
     turn_results: list[TurnResult] = []
     turn_outputs: dict[str, dict] = {}
@@ -403,6 +405,7 @@ def evaluate_agentic_conversation(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run conversation evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -420,6 +423,7 @@ def evaluate_agentic_conversation(
         fixture=fixture,
         max_clarification_turns=max_clarification_turns,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -439,6 +443,7 @@ def evaluate_agentic_conversation(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py

@@ -6,6 +6,7 @@
 from dataclasses import dataclass
 
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import LLMJudge
 
 _DEFAULT_K = 1
@@ -71,10 +72,11 @@ def run_agentic_general_question(
     expected_output: str,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticGeneralQuestionSummary:
     """Run the general-question agentic evaluation K times and return a summary."""
     run_results: list[GeneralQuestionResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     judge = LLMJudge(_GENERAL_QUESTION_EVALUATION_STEPS, model="gpt-4o")
 
     try:
@@ -153,6 +155,7 @@ def evaluate_agentic_general_question(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run general-question evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -171,6 +174,7 @@ def evaluate_agentic_general_question(
         expected_output=expected_output,
         k=k,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -190,6 +194,7 @@ def evaluate_agentic_general_question(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py

@@ -6,6 +6,7 @@
 from dataclasses import dataclass
 
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import LLMJudge
 
 _DEFAULT_K = 1
@@ -68,10 +69,11 @@ def run_agentic_guardrail(
     expected_output: str,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticGuardrailSummary:
     """Run the guardrail agentic evaluation K times and return a summary."""
     run_results: list[GuardrailResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     judge = LLMJudge(_GUARDRAIL_EVALUATION_STEPS, model="gpt-4o")
 
     try:
@@ -150,6 +152,7 @@ def evaluate_agentic_guardrail(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run guardrail evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -168,6 +171,7 @@ def evaluate_agentic_guardrail(
         expected_output=expected_output,
         k=k,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -187,6 +191,7 @@ def evaluate_agentic_guardrail(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,