From 49949a545163f82e4d0b83bd55ca1f94f61d08f2 Mon Sep 17 00:00:00 2001 From: Jan Kadlec Date: Tue, 4 Aug 2026 09:32:40 -0700 Subject: [PATCH] feat(gooddata-eval): let evaluations request a reasoning effort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an optional reasoning effort to `gooddata-eval`, so an evaluation run can request `LOW`, `MEDIUM` or `HIGH` instead of always using the endpoint default. Sent as `options.reasoningEffort` on each chat message and threaded through ChatClient, all seven `evaluate_agentic_*` entry points, the agentic runner and the CLI (`--reasoning-effort`). Per the AI chat OpenAPI schema the field is `SendMessageOptions.reasoningEffort`, typed as an enum of exactly LOW/MEDIUM/HIGH, nullable, and scoped to a single message rather than persisted. The implementation mirrors all four: uppercase `Literal`, camelCase key, re-sent per message, key omitted when unset. Without this there is no way to evaluate at a non-default effort — the value is read from the request body and the SDK never sent one, so effort was the one request-level option the suite could not vary. Both Langfuse writers record it. `build_run_context` suffixes the dataset-run name and adds run metadata; `LangfuseSink` suffixes its run name and reports the effort via trace `tags`, since metadata alone is not a breakdown dimension — the same reason `model_version` already uses first-class fields there. Two runs differing only by effort would otherwise share a name and merge in the report. The value is normalized once at the boundary. The `Literal` only constrains static callers, so without normalization a lowercase value reached the endpoint as an out-of-enum request, and a blank value was sent yet skipped by the writers' truthiness checks — leaving a run whose recorded identity disagreed with what it requested. Blank now means unset and anything outside the enum raises locally. New parameters are appended to the end of the public `evaluate_agentic_*` signatures rather than inserted mid-list, so existing positional callers of this released package keep binding correctly. Default is None, which omits the key and leaves the request payload byte-identical to before. Also drops an unused local in test_cli.py that was failing lint on master. --- packages/gooddata-eval/README.md | 3 +- .../src/gooddata_eval/cli/agentic_runner.py | 7 +- .../src/gooddata_eval/cli/main.py | 18 ++++- .../gooddata_eval/core/agentic/_langfuse.py | 12 +++ .../gooddata_eval/core/agentic/alert_skill.py | 7 +- .../core/agentic/conversation.py | 7 +- .../core/agentic/general_question.py | 7 +- .../gooddata_eval/core/agentic/guardrail.py | 7 +- .../core/agentic/metric_skill.py | 7 +- .../gooddata_eval/core/agentic/search_tool.py | 7 +- .../core/agentic/visualization.py | 7 +- .../src/gooddata_eval/core/chat/sse_client.py | 23 +++++- .../src/gooddata_eval/core/config.py | 24 ++++++ .../src/gooddata_eval/core/langfuse/sink.py | 23 +++++- .../tests/test_agentic_run_context.py | 41 ++++++++++ packages/gooddata-eval/tests/test_cli.py | 80 ++++++++++++++++++- .../gooddata-eval/tests/test_sse_client.py | 79 +++++++++++++++++- 17 files changed, 340 insertions(+), 19 deletions(-) diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index dfbeafd63..b17db60cb 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -92,6 +92,7 @@ Both provider name and provider id are accepted as the prefix. |---|---|---| | `--runs K` | `2` | Independent runs per item (pass@K). An item passes if any run passes. | | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests. Progress output interleaves when K > 1. | +| `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` items go through the summary endpoint, which has no such option. | #### Output @@ -104,7 +105,7 @@ Both provider name and provider id are accepted as the prefix. | Flag | Description | |---|---| -| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. | +| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`, suffixed `-effort-{level}` when `--reasoning-effort` is set so runs differing only by effort stay separate). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. | ### JSON report shape diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 31104660e..f73b44679 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -14,6 +14,7 @@ from gooddata_eval.core.agentic.metric_skill import evaluate_agentic_metric_skill from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization +from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.models import CreatedVisualization, DatasetItem from gooddata_eval.core.runner import EvalReport, ItemReport @@ -24,6 +25,7 @@ class _LfKw(TypedDict, total=False): dataset_name: str run_timestamp: str model_version_override: str | None + reasoning_effort: ReasoningEffort | None AGENTIC_TEST_KINDS = frozenset( @@ -80,6 +82,7 @@ def _dispatch_agentic( langfuse: Any, run_ts: str, model_version_override: str | None, + reasoning_effort: ReasoningEffort | None = None, ) -> None: """Call the appropriate evaluate_agentic_* function for the item's test_kind.""" kind = item.test_kind @@ -90,6 +93,7 @@ def _dispatch_agentic( "dataset_name": item.dataset_name, "run_timestamp": run_ts, "model_version_override": model_version_override, + "reasoning_effort": reasoning_effort, } if kind in ("vis_agentic", "agentic_visualization"): @@ -176,6 +180,7 @@ def run_agentic_items( *, k: int = 2, model_version: str | None = None, + reasoning_effort: ReasoningEffort | None = None, use_langfuse: bool = False, run_ts: str, on_item_start: Any = None, @@ -202,7 +207,7 @@ def run_agentic_items( ) t0 = time.perf_counter() try: - _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version) + _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort) item_report.pass_at_k = True item_report.runs = k except AssertionError as exc: diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 0303270be..9465602c0 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -6,6 +6,7 @@ import threading from datetime import datetime, timezone from pathlib import Path +from typing import get_args import httpx from gooddata_api_client.exceptions import ApiException @@ -14,7 +15,7 @@ from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items from gooddata_eval.core.chat.sse_client import ChatClient -from gooddata_eval.core.config import RunConfig +from gooddata_eval.core.config import ReasoningEffort, RunConfig from gooddata_eval.core.connection import ConnectionError_, resolve_connection from gooddata_eval.core.dataset.local import load_local_dataset from gooddata_eval.core.langfuse.sink import LangfuseSink @@ -104,6 +105,13 @@ def _build_parser() -> argparse.ArgumentParser: dest="preserve_failed", help="Keep failed conversations on the server for post-mortem inspection.", ) + run.add_argument( + "--reasoning-effort", + dest="reasoning_effort", + choices=list(get_args(ReasoningEffort)), + help="Reasoning effort requested per message. Requires the enableGenAiReasoningEffort " + "feature flag on the target organization; without it the server ignores the value.", + ) run.add_argument( "--langfuse", action="store_true", @@ -292,6 +300,10 @@ def _run(config: RunConfig) -> int: progress_console.print(f"Provider={provider_display}, model={resolved.model_id}{switched}") run_name = f"gd-eval-{run_ts}-{resolved.model_id}" + if config.reasoning_effort: + # Without this two runs differing only by effort share a name and are + # indistinguishable in the report, which is the comparison this exists for. + run_name = f"{run_name}-effort-{config.reasoning_effort.lower()}" if progress_console and config.log_to_langfuse: progress_console.print(f"Logging to Langfuse run '{run_name}'...") @@ -307,6 +319,7 @@ def _run(config: RunConfig) -> int: run_name=run_name, model_id=resolved.model_id, provider_type=resolved.provider_type, + reasoning_effort=config.reasoning_effort, ) def on_langfuse_item_done( @@ -329,6 +342,7 @@ def on_langfuse_item_done( workspace_id=config.workspace_id, k=config.runs, model_version=resolved.model_id, + reasoning_effort=config.reasoning_effort, use_langfuse=config.log_to_langfuse, run_ts=run_ts, on_item_start=on_item_start, @@ -342,6 +356,7 @@ def on_langfuse_item_done( token=config.token, workspace_id=config.workspace_id, preserve_failed=config.preserve_failed, + reasoning_effort=config.reasoning_effort, ), SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id), ) @@ -433,6 +448,7 @@ def main(argv: list[str] | None = None) -> int: quiet=args.quiet, kind=args.kind, preserve_failed=args.preserve_failed, + reasoning_effort=args.reasoning_effort, ) return _run(config) except ( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py index fd6e8d4e9..67630ce2f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py @@ -15,6 +15,8 @@ import httpx +from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort + _log = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -384,6 +386,7 @@ def build_run_context( run_timestamp: str | None, model_version_override: str | None, run_metadata_extra: dict[str, Any] | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> tuple[str, dict[str, Any]]: """Return (run_name_base, run_metadata) with model version resolved from workspace API. @@ -399,15 +402,24 @@ def build_run_context( (e.g. a testing-framework tag or a CI run id for scoping). Default None keeps behavior unchanged. The SDK-derived model_version is applied last and cannot be overwritten by this dict. + reasoning_effort: Effort the run requested, stamped into both the run name and + the metadata so effort-varying runs stay comparable side by side. """ + effort = normalize_reasoning_effort(reasoning_effort) model = get_model_version(host, token, workspace_id, model_version_override) ts = run_timestamp or datetime.now().strftime("%Y-%m-%d_%H-%M-%S") base = f"{dataset_name}_{ts}" if model: base = f"{base}_{model}" + # Part of the run name, not just metadata: two runs that differ only by effort would + # otherwise collide on the same name and be indistinguishable in the report. + if effort: + base = f"{base}_effort-{effort.lower()}" # Caller supplies its own run tags (e.g. testing_framework); model_version is applied # last so the SDK-derived value cannot be overwritten by run_metadata_extra. metadata: dict[str, Any] = dict(run_metadata_extra) if run_metadata_extra else {} + if effort: + metadata["reasoning_effort"] = effort if model: metadata["model_version"] = model return base, metadata diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 1a7d2a188..e75a2bed2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -13,6 +13,7 @@ from gooddata_eval.core.agentic._catalog import CatalogMetricAlert from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.models import ToolCallEvent try: @@ -342,11 +343,12 @@ def run_agentic_alert_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> AgenticAlertSummary: """Run the alert-skill agentic evaluation K times and return a summary.""" expected = _normalize_expected_output(expected_output) run_results: list[AlertRunResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) sdk = GoodDataSdk.create(host, token) def _run_once(conv_id: str) -> AlertRunResult: @@ -462,6 +464,7 @@ def evaluate_agentic_alert_skill( run_timestamp: str | None = None, model_version_override: str | None = None, run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> None: """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -481,6 +484,7 @@ def evaluate_agentic_alert_skill( k=k, max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: @@ -500,6 +504,7 @@ def evaluate_agentic_alert_skill( run_timestamp, model_version_override, run_metadata_extra, + reasoning_effort, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 6b79b3279..a7c3034fe 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -14,6 +14,7 @@ from gooddata_eval.core.agentic.alert_skill import render_alert_proposal from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.models import ChatResult, ToolCallEvent from gooddata_eval.core.scoring import ( check_filters, @@ -278,6 +279,7 @@ def run_agentic_conversation( fixture: ConversationFixture, max_clarification_turns: int = 20, initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> ConversationResult: """Run a multi-turn, multi-skill conversation evaluation (no K-runs). @@ -285,7 +287,7 @@ def run_agentic_conversation( trigger up to *max_clarification_turns* additional rounds of simulated-user replies before the agent produces the expected output. """ - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) sdk = GoodDataSdk.create(host, token) turn_results: list[TurnResult] = [] turn_outputs: dict[str, dict] = {} @@ -403,6 +405,7 @@ def evaluate_agentic_conversation( run_timestamp: str | None = None, model_version_override: str | None = None, run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> None: """Run conversation evaluation, log to Langfuse, and raise on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -420,6 +423,7 @@ def evaluate_agentic_conversation( fixture=fixture, max_clarification_turns=max_clarification_turns, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: @@ -439,6 +443,7 @@ def evaluate_agentic_conversation( run_timestamp, model_version_override, run_metadata_extra, + reasoning_effort, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index 653a77956..f2aa494ff 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.evaluators._llm_judge import LLMJudge _DEFAULT_K = 1 @@ -71,10 +72,11 @@ def run_agentic_general_question( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> AgenticGeneralQuestionSummary: """Run the general-question agentic evaluation K times and return a summary.""" run_results: list[GeneralQuestionResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) judge = LLMJudge(_GENERAL_QUESTION_EVALUATION_STEPS, model="gpt-4o") try: @@ -153,6 +155,7 @@ def evaluate_agentic_general_question( run_timestamp: str | None = None, model_version_override: str | None = None, run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> None: """Run general-question evaluation, log to Langfuse, and raise on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -171,6 +174,7 @@ def evaluate_agentic_general_question( expected_output=expected_output, k=k, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: @@ -190,6 +194,7 @@ def evaluate_agentic_general_question( run_timestamp, model_version_override, run_metadata_extra, + reasoning_effort, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index cb61da24a..ae74da5b4 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.evaluators._llm_judge import LLMJudge _DEFAULT_K = 1 @@ -68,10 +69,11 @@ def run_agentic_guardrail( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> AgenticGuardrailSummary: """Run the guardrail agentic evaluation K times and return a summary.""" run_results: list[GuardrailResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) judge = LLMJudge(_GUARDRAIL_EVALUATION_STEPS, model="gpt-4o") try: @@ -150,6 +152,7 @@ def evaluate_agentic_guardrail( run_timestamp: str | None = None, model_version_override: str | None = None, run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> None: """Run guardrail evaluation, log to Langfuse, and raise on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -168,6 +171,7 @@ def evaluate_agentic_guardrail( expected_output=expected_output, k=k, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: @@ -187,6 +191,7 @@ def evaluate_agentic_guardrail( run_timestamp, model_version_override, run_metadata_extra, + reasoning_effort, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index a2758f6a8..2e2b5b9b1 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -11,6 +11,7 @@ from gooddata_sdk import GoodDataSdk from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.models import ToolCallEvent try: @@ -232,6 +233,7 @@ def run_agentic_metric_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> AgenticMetricSummary: """Run the metric-skill agentic evaluation K times and return a summary. @@ -240,7 +242,7 @@ def run_agentic_metric_skill( """ expected_outputs: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output] run_results: list[MetricRunResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) sdk = GoodDataSdk.create(host, token) try: @@ -300,6 +302,7 @@ def evaluate_agentic_metric_skill( run_timestamp: str | None = None, model_version_override: str | None = None, run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> None: """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -319,6 +322,7 @@ def evaluate_agentic_metric_skill( k=k, max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: @@ -338,6 +342,7 @@ def evaluate_agentic_metric_skill( run_timestamp, model_version_override, run_metadata_extra, + reasoning_effort, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index cdbce48ab..25053e17b 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.models import ToolCallEvent _DEFAULT_K = 1 @@ -67,11 +68,12 @@ def run_agentic_search_tool( expected_tool_call: dict, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> AgenticSearchSummary: """Run the search-tool agentic evaluation K times (single-turn each).""" run_results: list[SearchResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: @@ -144,6 +146,7 @@ def evaluate_agentic_search_tool( run_timestamp: str | None = None, model_version_override: str | None = None, run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> None: """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -162,6 +165,7 @@ def evaluate_agentic_search_tool( expected_tool_call=expected_tool_call, k=k, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: @@ -181,6 +185,7 @@ def evaluate_agentic_search_tool( run_timestamp, model_version_override, run_metadata_extra, + reasoning_effort, ) traces_by_conv = find_traces_per_conversation( langfuse, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 12914157a..18656e30c 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -11,6 +11,7 @@ from dataclasses import dataclass from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.evaluators.visualization import ( EvaluationResult, _check_visualization_skill_activated, @@ -203,6 +204,7 @@ def run_agentic_visualization( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> AgenticRunSummary: """Run K independent conversations and return evaluation results. @@ -211,7 +213,7 @@ def run_agentic_visualization( fresh conversations. Caller-supplied conversations are not deleted; all conversations created by this function are deleted on completion. """ - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) run_results: list[RunResult] = [] try: @@ -265,6 +267,7 @@ def evaluate_agentic_visualization( model_version_override: str | None = None, run_metadata_extra: dict | None = None, record_output_path: str | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> None: """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure.""" import json as _json # noqa: PLC0415 @@ -285,6 +288,7 @@ def evaluate_agentic_visualization( k=k, max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: @@ -304,6 +308,7 @@ def evaluate_agentic_visualization( run_timestamp, model_version_override, run_metadata_extra, + reasoning_effort, ) K = len(summary.run_results) traces_by_conv = find_traces_per_conversation( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index 091436b27..2db50d5a2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -22,6 +22,7 @@ import httpx +from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort from gooddata_eval.core.models import ChatResult, DatasetItem _log = logging.getLogger(__name__) @@ -242,12 +243,28 @@ class ChatClient: """Single-turn AI chat client over the GoodData AI conversation endpoints.""" def __init__( - self, host: str, token: str, workspace_id: str, *, timeout: float = 300.0, preserve_failed: bool = False + self, + host: str, + token: str, + workspace_id: str, + *, + timeout: float = 300.0, + preserve_failed: bool = False, + reasoning_effort: ReasoningEffort | None = None, ): + """Create a chat client bound to one workspace. + + ``reasoning_effort`` (``LOW``/``MEDIUM``/``HIGH``) is sent as + ``options.reasoningEffort`` on every message; when None the key is omitted + entirely and the server keeps its own default. The server honours it only + while the ``enableGenAiReasoningEffort`` feature flag is on for the + organization, so setting it is a request rather than a guarantee. + """ self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations" self._auth = {"Authorization": f"Bearer {token}"} self._client = httpx.Client(timeout=timeout) self._preserve_failed = preserve_failed + self._reasoning_effort = normalize_reasoning_effort(reasoning_effort) def create_conversation(self) -> str: def _do() -> str: @@ -271,7 +288,9 @@ def delete_conversation(self, conversation_id: str) -> None: def send_message(self, conversation_id: str, question: str) -> ChatResult: url = f"{self._base}/{conversation_id}/messages" headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json"} - body = {"item": {"role": "user", "content": {"type": "text", "text": question}}} + body: dict[str, Any] = {"item": {"role": "user", "content": {"type": "text", "text": question}}} + if self._reasoning_effort is not None: + body["options"] = {"reasoningEffort": self._reasoning_effort} def _do() -> ChatResult: with self._client.stream("POST", url, json=body, headers=headers) as resp: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index 277785176..8cb07794f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -3,6 +3,29 @@ from dataclasses import dataclass, field from pathlib import Path +from typing import Literal, cast, get_args + +ReasoningEffort = Literal["LOW", "MEDIUM", "HIGH"] +"""Effort values the AI chat endpoint accepts, uppercase as the server enum requires.""" + + +def normalize_reasoning_effort(value: str | None) -> ReasoningEffort | None: + """Canonical effort, or None when unset. + + The `Literal` above only constrains static callers, so normalize once at the + boundary: without it a lowercase value reaches the endpoint and is rejected as + an out-of-enum request, while an empty string is sent yet skipped by the + truthiness checks in the Langfuse writers — leaving a run whose recorded + identity disagrees with what it actually requested. + """ + if value is None: + return None + candidate = value.strip().upper() + if not candidate: + return None + if candidate not in get_args(ReasoningEffort): + raise ValueError(f"Invalid reasoning effort {value!r}; expected one of {', '.join(get_args(ReasoningEffort))}.") + return cast("ReasoningEffort", candidate) @dataclass @@ -20,3 +43,4 @@ class RunConfig: quiet: bool = False kind: str = "visualization" preserve_failed: bool = False + reasoning_effort: ReasoningEffort | None = None diff --git a/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py b/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py index 378ebbbd0..ebb05af33 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py @@ -17,6 +17,7 @@ _SPEED_WEIGHT = 0.2 if TYPE_CHECKING: + from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.runner import ItemReport @@ -48,11 +49,19 @@ def compute_scores( class LangfuseSink: """Posts evaluation results to Langfuse via the ingestion REST API.""" - def __init__(self, dataset_name: str, run_name: str, model_id: str = "", provider_type: str = ""): + def __init__( + self, + dataset_name: str, + run_name: str, + model_id: str = "", + provider_type: str = "", + reasoning_effort: ReasoningEffort | None = None, + ): self._dataset_name = dataset_name self._run_name = run_name self._model_id = model_id self._provider_type = provider_type + self._reasoning_effort = reasoning_effort host = os.environ.get("LANGFUSE_HOST", "https://cloud.langfuse.com").rstrip("/") pub = os.environ.get("LANGFUSE_PUBLIC_KEY", "") sec = os.environ.get("LANGFUSE_SECRET_KEY", "") @@ -101,8 +110,17 @@ def _event(event_type: str, body: dict[str, Any]) -> dict[str, Any]: "item_id": report.id, "model": self._model_id, "provider_type": self._provider_type, + "reasoning_effort": self._reasoning_effort, }, - "tags": [t for t in [report.test_kind, self._provider_type] if t], + "tags": [ + t + for t in [ + report.test_kind, + self._provider_type, + f"effort-{self._reasoning_effort.lower()}" if self._reasoning_effort else None, + ] + if t + ], }, ), ] @@ -168,6 +186,7 @@ def _event(event_type: str, body: dict[str, Any]) -> dict[str, Any]: "metadata": { "model": self._model_id, "provider_type": self._provider_type, + "reasoning_effort": self._reasoning_effort, }, "datasetItemId": dataset_item_id, "traceId": trace_id, diff --git a/packages/gooddata-eval/tests/test_agentic_run_context.py b/packages/gooddata-eval/tests/test_agentic_run_context.py index dc584ebf3..d5638c6aa 100644 --- a/packages/gooddata-eval/tests/test_agentic_run_context.py +++ b/packages/gooddata-eval/tests/test_agentic_run_context.py @@ -1,4 +1,5 @@ # (C) 2026 GoodData Corporation +import pytest from gooddata_eval.core.agentic._langfuse import build_run_context # model_version_override short-circuits get_model_version (no workspace API call). @@ -30,3 +31,43 @@ def test_build_run_context_extra_cannot_override_model_version(): def test_build_run_context_without_extra_has_only_model_version(): _, metadata = build_run_context(**_COMMON) assert metadata == {"model_version": "m"} + + +def test_build_run_context_without_effort_is_unchanged(): + """Runs that do not request an effort keep their existing name and metadata.""" + base, metadata = build_run_context(**_COMMON) + assert base == "ds_2026-07-10_00-00-00_m" + assert "reasoning_effort" not in metadata + + +def test_build_run_context_effort_suffixes_the_run_name(): + """Two runs differing only by effort must not share a name, or the report merges them.""" + low, low_metadata = build_run_context(**_COMMON, reasoning_effort="LOW") + medium, _ = build_run_context(**_COMMON, reasoning_effort="MEDIUM") + assert low == "ds_2026-07-10_00-00-00_m_effort-low" + assert low != medium + assert low_metadata["reasoning_effort"] == "LOW" + + +def test_build_run_context_effort_does_not_displace_model_version(): + _, metadata = build_run_context(**_COMMON, reasoning_effort="HIGH") + assert metadata["model_version"] == "m" + assert metadata["reasoning_effort"] == "HIGH" + + +def test_build_run_context_normalizes_effort_casing(): + """Name suffix and metadata must agree with the canonical value that was sent.""" + base, metadata = build_run_context(**_COMMON, reasoning_effort="low") + assert base.endswith("_effort-low") + assert metadata["reasoning_effort"] == "LOW" + + +def test_build_run_context_blank_effort_is_unset(): + base, metadata = build_run_context(**_COMMON, reasoning_effort=" ") + assert base == "ds_2026-07-10_00-00-00_m" + assert "reasoning_effort" not in metadata + + +def test_build_run_context_rejects_invalid_effort(): + with pytest.raises(ValueError, match="Invalid reasoning effort"): + build_run_context(**_COMMON, reasoning_effort="turbo") diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 28aecb3c4..3f42a3f4e 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -260,7 +260,7 @@ def close(self): ... langfuse_calls: list = [] class _FakeSink: - def __init__(self, dataset_name, run_name, model_id="", provider_type=""): ... + def __init__(self, dataset_name, run_name, model_id="", provider_type="", reasoning_effort=None): ... def log_item(self, report, *, dataset_item_id): langfuse_calls.append(dataset_item_id) @@ -535,8 +535,6 @@ def close(self): ... monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) - original_chat_client = cli_main.ChatClient - def _capture_chat_client(**kwargs): captured_kwargs.update(kwargs) return object() @@ -630,3 +628,79 @@ def _worker(index: int) -> None: output = console.file.getvalue() assert "test-1" in output assert "test-49" in output + + +def test_cli_reasoning_effort_flag_parsed(monkeypatch, fixtures_dir): + """--reasoning-effort reaches RunConfig and is passed on to ChatClient.""" + monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) + captured_kwargs: dict = {} + + class _FakeController: + def __init__(self, *a, **k): ... + def get_active(self): + return ActiveLlmProvider(provider_id="p", default_model_id="gpt-5.2") + + def resolve_and_activate(self, requested, provider=None): + return ResolvedModel(provider_id="p", model_id="gpt-5.2", switched=False, provider_name="P") + + def restore(self, original): ... + def close(self): ... + + monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) + + def _capture_chat_client(**kwargs): + captured_kwargs.update(kwargs) + return object() + + monkeypatch.setattr(cli_main, "ChatClient", _capture_chat_client) + + def _fake_run(items, backend, *, runs, model, workspace_id, **kw): + return EvalReport( + model=model, + workspace_id=workspace_id, + items=[ + ItemReport(id="i1", dataset_name="d", test_kind="visualization", question="q", pass_at_k=True, runs=1) + ], + ) + + monkeypatch.setattr(cli_main, "run_items", _fake_run) + + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--reasoning-effort", + "LOW", + "--quiet", + ] + ) + assert exit_code == 0 + assert captured_kwargs.get("reasoning_effort") == "LOW" + + +def test_cli_rejects_unknown_reasoning_effort(fixtures_dir): + """argparse choices guard the value before it can reach the server as a 422.""" + with pytest.raises(SystemExit) as exc_info: + cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--reasoning-effort", + "low", + ] + ) + assert exc_info.value.code == 2 diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index c5590e428..490dfd57d 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -145,8 +145,8 @@ def test_parse_sse_lines_non_retryable_status_is_chat_error_not_transient(): assert ei.value.status_code == 400 -def _client_with_handler(handler): - client = ChatClient(host="https://example.invalid", token="t", workspace_id="w") +def _client_with_handler(handler, **kwargs): + client = ChatClient(host="https://example.invalid", token="t", workspace_id="w", **kwargs) client._client = httpx.Client(transport=httpx.MockTransport(handler)) return client @@ -378,3 +378,78 @@ def handler(request): with pytest.raises(ChatError) as ei: client.ask(item) assert ei.value.conversation_id == "conv-exc" + + +def _capture_body_client(captured, *, reasoning_effort=None): + def handler(request): + captured.append(json.loads(request.content)) + return httpx.Response(200, content=_OK_SSE) + + client = ChatClient(host="https://example.invalid", token="t", workspace_id="w", reasoning_effort=reasoning_effort) + client._client = httpx.Client(transport=httpx.MockTransport(handler)) + return client + + +def test_send_message_omits_options_when_no_reasoning_effort(): + """Default must stay byte-identical to the pre-feature payload.""" + captured = [] + _capture_body_client(captured).send_message("conv", "q") + assert captured == [{"item": {"role": "user", "content": {"type": "text", "text": "q"}}}] + + +@pytest.mark.parametrize("effort", ["LOW", "MEDIUM", "HIGH"]) +def test_send_message_sends_reasoning_effort(effort): + captured = [] + _capture_body_client(captured, reasoning_effort=effort).send_message("conv", "q") + assert captured[0]["options"] == {"reasoningEffort": effort} + assert captured[0]["item"]["content"]["text"] == "q" + + +def test_reasoning_effort_applies_to_every_message_in_a_conversation(): + """Multi-turn evaluators call send_message repeatedly on one client.""" + captured = [] + client = _capture_body_client(captured, reasoning_effort="LOW") + client.send_message("conv", "first") + client.send_message("conv", "follow-up") + assert [c["options"] for c in captured] == [{"reasoningEffort": "LOW"}] * 2 + + +def test_ask_propagates_reasoning_effort(): + """ask() creates, sends and deletes — the effort must survive that whole path.""" + captured = [] + + def handler(request): + if request.method == "POST" and request.url.path.endswith("/conversations"): + return httpx.Response(201, json={"conversationId": "c1"}) + if request.method == "POST": + captured.append(json.loads(request.content)) + return httpx.Response(200, content=_OK_SSE) + return httpx.Response(204) + + client = _client_with_handler(handler, reasoning_effort="HIGH") + item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={}) + client.ask(item) + assert captured[0]["options"] == {"reasoningEffort": "HIGH"} + + +@pytest.mark.parametrize(("given", "sent"), [("low", "LOW"), (" High ", "HIGH"), ("MEDIUM", "MEDIUM")]) +def test_send_message_normalizes_reasoning_effort(given, sent): + """The endpoint enum is uppercase, so casing is canonicalized before the request.""" + captured = [] + _capture_body_client(captured, reasoning_effort=given).send_message("conv", "q") + assert captured[0]["options"] == {"reasoningEffort": sent} + + +@pytest.mark.parametrize("blank", ["", " "]) +def test_blank_reasoning_effort_is_treated_as_unset(blank): + """Previously a blank value was sent but skipped by the Langfuse writers, so the + request and the recorded run disagreed. It now means 'unset' on both paths.""" + captured = [] + _capture_body_client(captured, reasoning_effort=blank).send_message("conv", "q") + assert "options" not in captured[0] + + +def test_invalid_reasoning_effort_fails_at_construction(): + """Fail locally rather than as an out-of-enum request partway through a run.""" + with pytest.raises(ValueError, match="Invalid reasoning effort"): + ChatClient(host="https://example.invalid", token="t", workspace_id="w", reasoning_effort="maximum")