Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions python/packages/core/agent_framework/_evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -539,7 +539,8 @@ def _check(results: EvalResults) -> None:
if evaluator is not None and score.name != evaluator:
continue
matching_scores_count += 1
if score.score < min_score:
# ``not >=`` (rather than ``<``) so that NaN scores count as offenders.
if not score.score >= min_score:
offenders.append(f"{item.item_id}/{score.name}={score.score:.3f}")
for sub in results.sub_results.values():
_check(sub)
Expand Down Expand Up @@ -1056,11 +1057,17 @@ class CheckResult:
passed: Whether the check passed.
reason: Human-readable explanation.
check_name: Name of the check that produced this result.
score: Optional numeric score produced by the check (for example,
from a function evaluator that returns a number). When ``None``,
``LocalEvaluator`` records ``1.0`` for a pass and ``0.0`` for a
failure. Keyword-only, so subclasses can still add required
fields.
"""

passed: bool
reason: str
check_name: str
score: float | None = field(default=None, kw_only=True)


EvalCheck = Callable[[EvalItem], CheckResult | Awaitable[CheckResult]]
Expand Down Expand Up @@ -1388,8 +1395,9 @@ def _coerce_result(value: Any, check_name: str) -> CheckResult:
return CheckResult(passed=value, reason="passed" if value else "failed", check_name=check_name)

if isinstance(value, (int, float)):
passed = value >= 0.5
return CheckResult(passed=passed, reason=f"score={value:.3f}", check_name=check_name)
score = float(value)
passed = score >= 0.5
return CheckResult(passed=passed, reason=f"score={score:.3f}", check_name=check_name, score=score)

if isinstance(value, dict):
d = cast(dict[str, Any], value)
Expand All @@ -1409,7 +1417,7 @@ def _coerce_result(value: Any, check_name: str) -> CheckResult:
# Honour an explicit 'passed' override; otherwise threshold-based.
passed = bool(d["passed"]) if "passed" in d else score >= float(d.get("threshold", 0.5))
reason = str(d.get("reason", f"score={score:.3f}"))
return CheckResult(passed=passed, reason=reason, check_name=check_name)
return CheckResult(passed=passed, reason=reason, check_name=check_name, score=score)
if "passed" in d:
passed_val = d["passed"]
return CheckResult(
Expand Down Expand Up @@ -1611,7 +1619,7 @@ async def evaluate(
item_scores.append(
EvalScoreResult(
name=result.check_name,
score=1.0 if result.passed else 0.0,
score=result.score if result.score is not None else (1.0 if result.passed else 0.0),
passed=result.passed,
sample={"reason": result.reason} if result.reason else None,
)
Expand Down
53 changes: 53 additions & 0 deletions python/packages/core/tests/core/test_local_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@

import inspect
import json
import math
from dataclasses import dataclass
from typing import Any

import pytest
Expand Down Expand Up @@ -828,6 +830,42 @@ def pass_all(response: str) -> bool:
assert results.items[1].input_text == "Q2"
assert results.items[1].output_text == "R2"

@pytest.mark.asyncio
async def test_numeric_evaluator_scores_are_preserved(self):
@evaluator
def float_judge(response: str) -> float:
return 0.6

@evaluator
def dict_judge(response: str) -> dict:
return {"score": 0.3, "threshold": 0.2}

@evaluator
def bool_check(response: str) -> bool:
return True

results = await LocalEvaluator(float_judge, dict_judge, bool_check).evaluate([_make_item()])

scores = {s.name: s for s in results.items[0].scores}
assert scores["float_judge"].score == pytest.approx(0.6)
assert scores["float_judge"].passed is True
assert scores["dict_judge"].score == pytest.approx(0.3)
assert scores["dict_judge"].passed is True
# Boolean checks keep mapping to 1.0 / 0.0.
assert scores["bool_check"].score == 1.0

@pytest.mark.asyncio
async def test_numeric_evaluator_score_below_gate_raises(self):
@evaluator
def judge(response: str) -> float:
return 0.6

results = await LocalEvaluator(judge).evaluate([_make_item()])

results.assert_score_at_least(0.5)
with pytest.raises(EvalNotPassedError, match="judge=0.600"):
results.assert_score_at_least(0.8)


# ---------------------------------------------------------------------------
# num_repetitions validation
Expand Down Expand Up @@ -1334,6 +1372,21 @@ def test_evaluator_filter_matches_in_sub_results(self) -> None:
# Should match "sub_eval" in sub-results and pass without error
parent.assert_score_at_least(0.8, evaluator="sub_eval")

def test_check_result_subclass_can_add_required_fields(self) -> None:
@dataclass
class LabelledCheckResult(CheckResult):
label: str

result = LabelledCheckResult(True, "ok", "check", "positive", score=0.7)
assert result.label == "positive"
assert result.score == 0.7

def test_nan_score_raises(self) -> None:
results = _score_results([EvalScoreResult(name="relevance", score=math.nan)])
with pytest.raises(EvalNotPassedError) as exc:
results.assert_score_at_least(0.5)
assert "relevance=nan" in str(exc.value)


class TestAssertNoFailedItems:
"""Tests for EvalResults.assert_no_failed_items (mirrors .NET coverage)."""
Expand Down
Loading