From 1285b0215668215e7ad3e3e6e937f22b6f3f7540 Mon Sep 17 00:00:00 2001 From: Dhruv-Sh-git Date: Mon, 9 Nov 2026 09:04:21 +0530 Subject: [PATCH] Python: fail assert_score_at_least when evaluator filter matches no scores --- .../core/agent_framework/_evaluation.py | 13 ++++++- .../core/tests/core/test_local_eval.py | 35 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 8ee060870c7..e0a3a30c9d2 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -524,21 +524,32 @@ def assert_score_at_least( msg: Optional custom failure message. Raises: - EvalNotPassedError: When any matching score is below the threshold. + EvalNotPassedError: When any matching score is below the threshold, + or when ``evaluator`` is specified but matches no scores. """ offenders: list[str] = [] + matching_scores_count = 0 + seen_evaluators: set[str] = set() def _check(results: EvalResults) -> None: + nonlocal matching_scores_count for item in results.items: for score in item.scores: + seen_evaluators.add(score.name) if evaluator is not None and score.name != evaluator: continue + matching_scores_count += 1 if score.score < min_score: offenders.append(f"{item.item_id}/{score.name}={score.score:.3f}") for sub in results.sub_results.values(): _check(sub) _check(self) + if evaluator is not None and matching_scores_count == 0: + available = f": {', '.join(sorted(seen_evaluators))}" if seen_evaluators else ": none" + detail = msg or f"No scores found for evaluator '{evaluator}'. Available evaluator(s){available}" + raise EvalNotPassedError(detail) + if offenders: detail = msg or ( f"{len(offenders)} score(s) below threshold {min_score}" diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index 9875b2aeeee..f086d80b315 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -1273,6 +1273,41 @@ def test_recursion_into_sub_results(self) -> None: # Offender from sub-result is surfaced. assert "0.200" in str(exc.value) + def test_evaluator_filter_missing_evaluator_raises(self) -> None: + results = _score_results( + [ + EvalScoreResult(name="judge", score=0.9), + EvalScoreResult(name="relevance", score=0.85), + ], + ) + with pytest.raises(EvalNotPassedError) as exc: + results.assert_score_at_least(0.8, evaluator="judeg") + msg = str(exc.value) + assert "No scores found for evaluator 'judeg'" in msg + assert "judge" in msg + assert "relevance" in msg + + def test_evaluator_filter_empty_scores_raises(self) -> None: + results = _score_results([]) + with pytest.raises(EvalNotPassedError) as exc: + results.assert_score_at_least(0.8, evaluator="relevance") + assert "No scores found for evaluator 'relevance'. Available evaluator(s): none" in str(exc.value) + + def test_evaluator_filter_custom_msg_on_missing_evaluator(self) -> None: + results = _score_results([EvalScoreResult(name="judge", score=0.9)]) + with pytest.raises(EvalNotPassedError) as exc: + results.assert_score_at_least(0.8, evaluator="missing", msg="Custom missing evaluator message") + assert str(exc.value) == "Custom missing evaluator message" + + def test_evaluator_filter_matches_in_sub_results(self) -> None: + sub = _score_results([EvalScoreResult(name="sub_eval", score=0.85)]) + parent = _score_results( + [EvalScoreResult(name="parent_eval", score=0.9)], + sub_results={"sub": sub}, + ) + # Should match "sub_eval" in sub-results and pass without error + parent.assert_score_at_least(0.8, evaluator="sub_eval") + class TestAssertNoFailedItems: """Tests for EvalResults.assert_no_failed_items (mirrors .NET coverage)."""