diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index e0a3a30c9d2..ddd679e0f44 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -585,22 +585,32 @@ def assert_dimension_score_at_least( msg: Optional custom failure message. Raises: - EvalNotPassedError: When the dimension fails the threshold. + EvalNotPassedError: When the dimension fails the threshold, or when + ``evaluator``/``dimension_id`` matches nothing (default flags). """ offenders: list[str] = [] missing_items: list[str] = [] + matching_scores_count = 0 + dimension_seen = False + seen_evaluators: set[str] = set() + seen_dimensions: set[str] = set() def _check(results: EvalResults) -> None: + nonlocal matching_scores_count, dimension_seen for item in results.items: found_applicable = False for score in item.scores: + seen_evaluators.add(score.name) if evaluator is not None and score.name != evaluator: continue + matching_scores_count += 1 if not score.dimensions: continue for rs in score.dimensions: + seen_dimensions.add(rs.id) if rs.id != dimension_id: continue + dimension_seen = True if not rs.applicable: continue found_applicable = True @@ -615,6 +625,23 @@ def _check(results: EvalResults) -> None: _check(sub) _check(self) + # A filter that matches nothing must fail the assertion instead of + # vacuously passing (same fix as assert_score_at_least). The checks + # run under the default flags, and also when require_applicable=True + # has no items to report as not-applicable — an empty evaluation must + # not pass the stricter option either. When items are missing, the + # not-applicable report below carries them (wording pinned by tests). + if not require_applicable or not missing_items: + if evaluator is not None and matching_scores_count == 0: + available = f": {', '.join(sorted(seen_evaluators))}" if seen_evaluators else ": none" + raise EvalNotPassedError( + msg or f"No scores found for evaluator '{evaluator}'. Available evaluator(s){available}" + ) + if not dimension_seen: + available = f": {', '.join(sorted(seen_dimensions))}" if seen_dimensions else ": none" + raise EvalNotPassedError( + msg or f"No scores found for dimension '{dimension_id}'. Available dimension(s){available}" + ) problems: list[str] = [] if offenders: problems.append( diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index f086d80b315..b54bd7cdd54 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -1210,6 +1210,55 @@ def test_evaluator_filter_isolates_offenders(self) -> None: # The low-scoring "other" evaluator is filtered out; "policy" passes. results.assert_dimension_score_at_least("clarity", 3, evaluator="policy") + def test_evaluator_filter_missing_evaluator_raises_by_default(self) -> None: + # A filter that matches nothing must fail instead of vacuously + # passing (mirror of the assert_score_at_least fix). + results = _rubric_results( + [ + EvalScoreResult( + name="other", + score=0.9, + dimensions=[RubricScore(id="clarity", score=4, applicable=True, weight=1, reason="")], + ) + ], + ) + with pytest.raises(EvalNotPassedError) as exc: + results.assert_dimension_score_at_least("clarity", 3, evaluator="policy") + msg = str(exc.value) + assert "No scores found for evaluator 'policy'" in msg + assert "other" in msg + + def test_dimension_filter_missing_dimension_raises_by_default(self) -> None: + results = _rubric_results( + [ + EvalScoreResult( + name="policy", + score=0.9, + dimensions=[RubricScore(id="clarity", score=4, applicable=True, weight=1, reason="")], + ) + ], + ) + with pytest.raises(EvalNotPassedError) as exc: + results.assert_dimension_score_at_least("coherence", 3) + msg = str(exc.value) + assert "No scores found for dimension 'coherence'" in msg + assert "clarity" in msg + + def test_dimension_filter_empty_results_raises_by_default(self) -> None: + results = _rubric_results([]) + with pytest.raises(EvalNotPassedError) as exc: + results.assert_dimension_score_at_least("clarity", 3) + assert "No scores found for dimension 'clarity'. Available dimension(s): none" in str(exc.value) + + def test_empty_results_raise_even_with_require_applicable(self) -> None: + # Zero items vacuously passed even the stricter flag (Copilot review): + # with no items, missing_items stays empty and nothing was checked. + results = EvalResults(provider="test") + with pytest.raises(EvalNotPassedError, match="No scores found for dimension"): + results.assert_dimension_score_at_least("clarity", 3, require_applicable=True) + with pytest.raises(EvalNotPassedError, match="No scores found for evaluator 'policy'"): + results.assert_dimension_score_at_least("clarity", 3, evaluator="policy", require_applicable=True) + def _score_results( *scores_per_item: list[EvalScoreResult],