Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion python/packages/core/agent_framework/_evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -585,22 +585,32 @@ def assert_dimension_score_at_least(
msg: Optional custom failure message.

Raises:
EvalNotPassedError: When the dimension fails the threshold.
EvalNotPassedError: When the dimension fails the threshold, or when
``evaluator``/``dimension_id`` matches nothing (default flags).
"""
offenders: list[str] = []
missing_items: list[str] = []
matching_scores_count = 0
dimension_seen = False
seen_evaluators: set[str] = set()
seen_dimensions: set[str] = set()

def _check(results: EvalResults) -> None:
nonlocal matching_scores_count, dimension_seen
for item in results.items:
found_applicable = False
for score in item.scores:
seen_evaluators.add(score.name)
if evaluator is not None and score.name != evaluator:
continue
matching_scores_count += 1
if not score.dimensions:
continue
for rs in score.dimensions:
seen_dimensions.add(rs.id)
if rs.id != dimension_id:
continue
dimension_seen = True
if not rs.applicable:
continue
found_applicable = True
Expand All @@ -615,6 +625,23 @@ def _check(results: EvalResults) -> None:
_check(sub)

_check(self)
# A filter that matches nothing must fail the assertion instead of
# vacuously passing (same fix as assert_score_at_least). The checks
# run under the default flags, and also when require_applicable=True
# has no items to report as not-applicable — an empty evaluation must
# not pass the stricter option either. When items are missing, the
# not-applicable report below carries them (wording pinned by tests).
if not require_applicable or not missing_items:
if evaluator is not None and matching_scores_count == 0:
available = f": {', '.join(sorted(seen_evaluators))}" if seen_evaluators else ": none"
raise EvalNotPassedError(
msg or f"No scores found for evaluator '{evaluator}'. Available evaluator(s){available}"
)
if not dimension_seen:
available = f": {', '.join(sorted(seen_dimensions))}" if seen_dimensions else ": none"
raise EvalNotPassedError(
msg or f"No scores found for dimension '{dimension_id}'. Available dimension(s){available}"
)
problems: list[str] = []
if offenders:
problems.append(
Expand Down
49 changes: 49 additions & 0 deletions python/packages/core/tests/core/test_local_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -1210,6 +1210,55 @@ def test_evaluator_filter_isolates_offenders(self) -> None:
# The low-scoring "other" evaluator is filtered out; "policy" passes.
results.assert_dimension_score_at_least("clarity", 3, evaluator="policy")

def test_evaluator_filter_missing_evaluator_raises_by_default(self) -> None:
# A filter that matches nothing must fail instead of vacuously
# passing (mirror of the assert_score_at_least fix).
results = _rubric_results(
[
EvalScoreResult(
name="other",
score=0.9,
dimensions=[RubricScore(id="clarity", score=4, applicable=True, weight=1, reason="")],
)
],
)
with pytest.raises(EvalNotPassedError) as exc:
results.assert_dimension_score_at_least("clarity", 3, evaluator="policy")
msg = str(exc.value)
assert "No scores found for evaluator 'policy'" in msg
assert "other" in msg

def test_dimension_filter_missing_dimension_raises_by_default(self) -> None:
results = _rubric_results(
[
EvalScoreResult(
name="policy",
score=0.9,
dimensions=[RubricScore(id="clarity", score=4, applicable=True, weight=1, reason="")],
)
],
)
with pytest.raises(EvalNotPassedError) as exc:
results.assert_dimension_score_at_least("coherence", 3)
msg = str(exc.value)
assert "No scores found for dimension 'coherence'" in msg
assert "clarity" in msg

def test_dimension_filter_empty_results_raises_by_default(self) -> None:
results = _rubric_results([])
with pytest.raises(EvalNotPassedError) as exc:
results.assert_dimension_score_at_least("clarity", 3)
assert "No scores found for dimension 'clarity'. Available dimension(s): none" in str(exc.value)

def test_empty_results_raise_even_with_require_applicable(self) -> None:
# Zero items vacuously passed even the stricter flag (Copilot review):
# with no items, missing_items stays empty and nothing was checked.
results = EvalResults(provider="test")
with pytest.raises(EvalNotPassedError, match="No scores found for dimension"):
results.assert_dimension_score_at_least("clarity", 3, require_applicable=True)
with pytest.raises(EvalNotPassedError, match="No scores found for evaluator 'policy'"):
results.assert_dimension_score_at_least("clarity", 3, evaluator="policy", require_applicable=True)


def _score_results(
*scores_per_item: list[EvalScoreResult],
Expand Down
Loading