diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 787cac17bd3..09d6649430a 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -539,7 +539,8 @@ def _check(results: EvalResults) -> None: if evaluator is not None and score.name != evaluator: continue matching_scores_count += 1 - if score.score < min_score: + # ``not >=`` (rather than ``<``) so that NaN scores count as offenders. + if not score.score >= min_score: offenders.append(f"{item.item_id}/{score.name}={score.score:.3f}") for sub in results.sub_results.values(): _check(sub) @@ -1056,11 +1057,17 @@ class CheckResult: passed: Whether the check passed. reason: Human-readable explanation. check_name: Name of the check that produced this result. + score: Optional numeric score produced by the check (for example, + from a function evaluator that returns a number). When ``None``, + ``LocalEvaluator`` records ``1.0`` for a pass and ``0.0`` for a + failure. Keyword-only, so subclasses can still add required + fields. """ passed: bool reason: str check_name: str + score: float | None = field(default=None, kw_only=True) EvalCheck = Callable[[EvalItem], CheckResult | Awaitable[CheckResult]] @@ -1388,8 +1395,9 @@ def _coerce_result(value: Any, check_name: str) -> CheckResult: return CheckResult(passed=value, reason="passed" if value else "failed", check_name=check_name) if isinstance(value, (int, float)): - passed = value >= 0.5 - return CheckResult(passed=passed, reason=f"score={value:.3f}", check_name=check_name) + score = float(value) + passed = score >= 0.5 + return CheckResult(passed=passed, reason=f"score={score:.3f}", check_name=check_name, score=score) if isinstance(value, dict): d = cast(dict[str, Any], value) @@ -1409,7 +1417,7 @@ def _coerce_result(value: Any, check_name: str) -> CheckResult: # Honour an explicit 'passed' override; otherwise threshold-based. passed = bool(d["passed"]) if "passed" in d else score >= float(d.get("threshold", 0.5)) reason = str(d.get("reason", f"score={score:.3f}")) - return CheckResult(passed=passed, reason=reason, check_name=check_name) + return CheckResult(passed=passed, reason=reason, check_name=check_name, score=score) if "passed" in d: passed_val = d["passed"] return CheckResult( @@ -1611,7 +1619,7 @@ async def evaluate( item_scores.append( EvalScoreResult( name=result.check_name, - score=1.0 if result.passed else 0.0, + score=result.score if result.score is not None else (1.0 if result.passed else 0.0), passed=result.passed, sample={"reason": result.reason} if result.reason else None, ) diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index 06f39d634a6..2b64e6c01be 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -6,6 +6,8 @@ import inspect import json +import math +from dataclasses import dataclass from typing import Any import pytest @@ -828,6 +830,42 @@ def pass_all(response: str) -> bool: assert results.items[1].input_text == "Q2" assert results.items[1].output_text == "R2" + @pytest.mark.asyncio + async def test_numeric_evaluator_scores_are_preserved(self): + @evaluator + def float_judge(response: str) -> float: + return 0.6 + + @evaluator + def dict_judge(response: str) -> dict: + return {"score": 0.3, "threshold": 0.2} + + @evaluator + def bool_check(response: str) -> bool: + return True + + results = await LocalEvaluator(float_judge, dict_judge, bool_check).evaluate([_make_item()]) + + scores = {s.name: s for s in results.items[0].scores} + assert scores["float_judge"].score == pytest.approx(0.6) + assert scores["float_judge"].passed is True + assert scores["dict_judge"].score == pytest.approx(0.3) + assert scores["dict_judge"].passed is True + # Boolean checks keep mapping to 1.0 / 0.0. + assert scores["bool_check"].score == 1.0 + + @pytest.mark.asyncio + async def test_numeric_evaluator_score_below_gate_raises(self): + @evaluator + def judge(response: str) -> float: + return 0.6 + + results = await LocalEvaluator(judge).evaluate([_make_item()]) + + results.assert_score_at_least(0.5) + with pytest.raises(EvalNotPassedError, match="judge=0.600"): + results.assert_score_at_least(0.8) + # --------------------------------------------------------------------------- # num_repetitions validation @@ -1334,6 +1372,21 @@ def test_evaluator_filter_matches_in_sub_results(self) -> None: # Should match "sub_eval" in sub-results and pass without error parent.assert_score_at_least(0.8, evaluator="sub_eval") + def test_check_result_subclass_can_add_required_fields(self) -> None: + @dataclass + class LabelledCheckResult(CheckResult): + label: str + + result = LabelledCheckResult(True, "ok", "check", "positive", score=0.7) + assert result.label == "positive" + assert result.score == 0.7 + + def test_nan_score_raises(self) -> None: + results = _score_results([EvalScoreResult(name="relevance", score=math.nan)]) + with pytest.raises(EvalNotPassedError) as exc: + results.assert_score_at_least(0.5) + assert "relevance=nan" in str(exc.value) + class TestAssertNoFailedItems: """Tests for EvalResults.assert_no_failed_items (mirrors .NET coverage)."""