From 44e3d3c9b5a3fe7ad5ffd7dadc7dc48c0b03112b Mon Sep 17 00:00:00 2001 From: Qiming Bao Date: Thu, 5 Nov 2026 21:14:30 +0000 Subject: [PATCH 1/2] Python: Preserve numeric evaluator scores in LocalEvaluator LocalEvaluator recorded every check score as 1.0 or 0.0 based only on the pass/fail flag, so a function evaluator returning a numeric score (a float, or a dict with "score") lost that value. As a result, EvalResults.assert_score_at_least() gated on the pass flag rather than on the evaluator's score, and a 0.6 judge score passed a 0.8 gate. - Add an optional `score` field to CheckResult (last, with a None default, so existing constructor calls are unaffected). - Set it in _coerce_result() for numeric returns and dicts with "score". - Use it in LocalEvaluator.evaluate() when present; boolean checks keep mapping to 1.0 / 0.0. - Treat NaN scores as below the threshold in assert_score_at_least(). Fixes #10073 --- .../core/agent_framework/_evaluation.py | 17 +++++--- .../core/tests/core/test_local_eval.py | 43 +++++++++++++++++++ 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 787cac17bd3..cafcf20b0b6 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -539,7 +539,8 @@ def _check(results: EvalResults) -> None: if evaluator is not None and score.name != evaluator: continue matching_scores_count += 1 - if score.score < min_score: + # ``not >=`` (rather than ``<``) so that NaN scores count as offenders. + if not score.score >= min_score: offenders.append(f"{item.item_id}/{score.name}={score.score:.3f}") for sub in results.sub_results.values(): _check(sub) @@ -1056,11 +1057,16 @@ class CheckResult: passed: Whether the check passed. reason: Human-readable explanation. check_name: Name of the check that produced this result. + score: Optional numeric score produced by the check (for example, + from a function evaluator that returns a number). When ``None``, + ``LocalEvaluator`` records ``1.0`` for a pass and ``0.0`` for a + failure. """ passed: bool reason: str check_name: str + score: float | None = None EvalCheck = Callable[[EvalItem], CheckResult | Awaitable[CheckResult]] @@ -1388,8 +1394,9 @@ def _coerce_result(value: Any, check_name: str) -> CheckResult: return CheckResult(passed=value, reason="passed" if value else "failed", check_name=check_name) if isinstance(value, (int, float)): - passed = value >= 0.5 - return CheckResult(passed=passed, reason=f"score={value:.3f}", check_name=check_name) + score = float(value) + passed = score >= 0.5 + return CheckResult(passed=passed, reason=f"score={score:.3f}", check_name=check_name, score=score) if isinstance(value, dict): d = cast(dict[str, Any], value) @@ -1409,7 +1416,7 @@ def _coerce_result(value: Any, check_name: str) -> CheckResult: # Honour an explicit 'passed' override; otherwise threshold-based. passed = bool(d["passed"]) if "passed" in d else score >= float(d.get("threshold", 0.5)) reason = str(d.get("reason", f"score={score:.3f}")) - return CheckResult(passed=passed, reason=reason, check_name=check_name) + return CheckResult(passed=passed, reason=reason, check_name=check_name, score=score) if "passed" in d: passed_val = d["passed"] return CheckResult( @@ -1611,7 +1618,7 @@ async def evaluate( item_scores.append( EvalScoreResult( name=result.check_name, - score=1.0 if result.passed else 0.0, + score=result.score if result.score is not None else (1.0 if result.passed else 0.0), passed=result.passed, sample={"reason": result.reason} if result.reason else None, ) diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index 06f39d634a6..3d4550ec750 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -6,6 +6,7 @@ import inspect import json +import math from typing import Any import pytest @@ -828,6 +829,42 @@ def pass_all(response: str) -> bool: assert results.items[1].input_text == "Q2" assert results.items[1].output_text == "R2" + @pytest.mark.asyncio + async def test_numeric_evaluator_scores_are_preserved(self): + @evaluator + def float_judge(response: str) -> float: + return 0.6 + + @evaluator + def dict_judge(response: str) -> dict: + return {"score": 0.3, "threshold": 0.2} + + @evaluator + def bool_check(response: str) -> bool: + return True + + results = await LocalEvaluator(float_judge, dict_judge, bool_check).evaluate([_make_item()]) + + scores = {s.name: s for s in results.items[0].scores} + assert scores["float_judge"].score == pytest.approx(0.6) + assert scores["float_judge"].passed is True + assert scores["dict_judge"].score == pytest.approx(0.3) + assert scores["dict_judge"].passed is True + # Boolean checks keep mapping to 1.0 / 0.0. + assert scores["bool_check"].score == 1.0 + + @pytest.mark.asyncio + async def test_numeric_evaluator_score_below_gate_raises(self): + @evaluator + def judge(response: str) -> float: + return 0.6 + + results = await LocalEvaluator(judge).evaluate([_make_item()]) + + results.assert_score_at_least(0.5) + with pytest.raises(EvalNotPassedError, match="judge=0.600"): + results.assert_score_at_least(0.8) + # --------------------------------------------------------------------------- # num_repetitions validation @@ -1334,6 +1371,12 @@ def test_evaluator_filter_matches_in_sub_results(self) -> None: # Should match "sub_eval" in sub-results and pass without error parent.assert_score_at_least(0.8, evaluator="sub_eval") + def test_nan_score_raises(self) -> None: + results = _score_results([EvalScoreResult(name="relevance", score=math.nan)]) + with pytest.raises(EvalNotPassedError) as exc: + results.assert_score_at_least(0.5) + assert "relevance=nan" in str(exc.value) + class TestAssertNoFailedItems: """Tests for EvalResults.assert_no_failed_items (mirrors .NET coverage).""" From 40acc8042dd8e72f8192a006e7d3143443ebe799 Mon Sep 17 00:00:00 2001 From: Qiming Bao Date: Mon, 16 Nov 2026 04:39:46 +0000 Subject: [PATCH 2/2] Python: Make CheckResult.score keyword-only A positional field with a default breaks dataclass subclasses that add required fields. Add a test for such a subclass. --- python/packages/core/agent_framework/_evaluation.py | 5 +++-- python/packages/core/tests/core/test_local_eval.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index cafcf20b0b6..09d6649430a 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -1060,13 +1060,14 @@ class CheckResult: score: Optional numeric score produced by the check (for example, from a function evaluator that returns a number). When ``None``, ``LocalEvaluator`` records ``1.0`` for a pass and ``0.0`` for a - failure. + failure. Keyword-only, so subclasses can still add required + fields. """ passed: bool reason: str check_name: str - score: float | None = None + score: float | None = field(default=None, kw_only=True) EvalCheck = Callable[[EvalItem], CheckResult | Awaitable[CheckResult]] diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index 3d4550ec750..2b64e6c01be 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -7,6 +7,7 @@ import inspect import json import math +from dataclasses import dataclass from typing import Any import pytest @@ -1371,6 +1372,15 @@ def test_evaluator_filter_matches_in_sub_results(self) -> None: # Should match "sub_eval" in sub-results and pass without error parent.assert_score_at_least(0.8, evaluator="sub_eval") + def test_check_result_subclass_can_add_required_fields(self) -> None: + @dataclass + class LabelledCheckResult(CheckResult): + label: str + + result = LabelledCheckResult(True, "ok", "check", "positive", score=0.7) + assert result.label == "positive" + assert result.score == 0.7 + def test_nan_score_raises(self) -> None: results = _score_results([EvalScoreResult(name="relevance", score=math.nan)]) with pytest.raises(EvalNotPassedError) as exc: