Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion python/packages/core/agent_framework/_evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -524,21 +524,32 @@ def assert_score_at_least(
msg: Optional custom failure message.

Raises:
EvalNotPassedError: When any matching score is below the threshold.
EvalNotPassedError: When any matching score is below the threshold,
or when ``evaluator`` is specified but matches no scores.
"""
offenders: list[str] = []
matching_scores_count = 0
seen_evaluators: set[str] = set()

def _check(results: EvalResults) -> None:
nonlocal matching_scores_count
for item in results.items:
for score in item.scores:
seen_evaluators.add(score.name)
if evaluator is not None and score.name != evaluator:
continue
matching_scores_count += 1
if score.score < min_score:
offenders.append(f"{item.item_id}/{score.name}={score.score:.3f}")
for sub in results.sub_results.values():
_check(sub)

_check(self)
if evaluator is not None and matching_scores_count == 0:
available = f": {', '.join(sorted(seen_evaluators))}" if seen_evaluators else ": none"
detail = msg or f"No scores found for evaluator '{evaluator}'. Available evaluator(s){available}"
raise EvalNotPassedError(detail)

if offenders:
detail = msg or (
f"{len(offenders)} score(s) below threshold {min_score}"
Expand Down
35 changes: 35 additions & 0 deletions python/packages/core/tests/core/test_local_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -1273,6 +1273,41 @@ def test_recursion_into_sub_results(self) -> None:
# Offender from sub-result is surfaced.
assert "0.200" in str(exc.value)

def test_evaluator_filter_missing_evaluator_raises(self) -> None:
results = _score_results(
[
EvalScoreResult(name="judge", score=0.9),
EvalScoreResult(name="relevance", score=0.85),
],
)
with pytest.raises(EvalNotPassedError) as exc:
results.assert_score_at_least(0.8, evaluator="judeg")
msg = str(exc.value)
assert "No scores found for evaluator 'judeg'" in msg
assert "judge" in msg
assert "relevance" in msg

def test_evaluator_filter_empty_scores_raises(self) -> None:
results = _score_results([])
with pytest.raises(EvalNotPassedError) as exc:
results.assert_score_at_least(0.8, evaluator="relevance")
assert "No scores found for evaluator 'relevance'. Available evaluator(s): none" in str(exc.value)

def test_evaluator_filter_custom_msg_on_missing_evaluator(self) -> None:
results = _score_results([EvalScoreResult(name="judge", score=0.9)])
with pytest.raises(EvalNotPassedError) as exc:
results.assert_score_at_least(0.8, evaluator="missing", msg="Custom missing evaluator message")
assert str(exc.value) == "Custom missing evaluator message"

def test_evaluator_filter_matches_in_sub_results(self) -> None:
sub = _score_results([EvalScoreResult(name="sub_eval", score=0.85)])
parent = _score_results(
[EvalScoreResult(name="parent_eval", score=0.9)],
sub_results={"sub": sub},
)
# Should match "sub_eval" in sub-results and pass without error
parent.assert_score_at_least(0.8, evaluator="sub_eval")


class TestAssertNoFailedItems:
"""Tests for EvalResults.assert_no_failed_items (mirrors .NET coverage)."""
Expand Down
Loading