From 1803a24a22472fabac581155aee8753b63bb59c7 Mon Sep 17 00:00:00 2001 From: Wilson0511 <1795618152@qq.com> Date: Mon, 9 Nov 2026 21:06:21 +0800 Subject: [PATCH 1/2] Fail assert_dimension_score_at_least when a filter matches nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the default require_applicable=False, assert_dimension_score_at_least vacuously passed when the evaluator filter or the dimension_id matched nothing — a typo in either silently skipped the CI gate. Mirror the assert_score_at_least fix: when no score matches the evaluator (or the dimension is never seen among matching scores), raise EvalNotPassedError listing the available evaluators/dimensions. require_applicable=True keeps its existing not-applicable report (and its pinned wording); the new failure only applies to the default flags where the call previously passed silently. Co-Authored-By: ZCode --- .../core/agent_framework/_evaluation.py | 28 ++++++++++++- .../core/tests/core/test_local_eval.py | 40 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index e0a3a30c9d2..5612f142d0a 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -585,22 +585,32 @@ def assert_dimension_score_at_least( msg: Optional custom failure message. Raises: - EvalNotPassedError: When the dimension fails the threshold. + EvalNotPassedError: When the dimension fails the threshold, or when + ``evaluator``/``dimension_id`` matches nothing (default flags). """ offenders: list[str] = [] missing_items: list[str] = [] + matching_scores_count = 0 + dimension_seen = False + seen_evaluators: set[str] = set() + seen_dimensions: set[str] = set() def _check(results: EvalResults) -> None: + nonlocal matching_scores_count, dimension_seen for item in results.items: found_applicable = False for score in item.scores: + seen_evaluators.add(score.name) if evaluator is not None and score.name != evaluator: continue + matching_scores_count += 1 if not score.dimensions: continue for rs in score.dimensions: + seen_dimensions.add(rs.id) if rs.id != dimension_id: continue + dimension_seen = True if not rs.applicable: continue found_applicable = True @@ -615,6 +625,22 @@ def _check(results: EvalResults) -> None: _check(sub) _check(self) + # With the default flags, a filter that matches nothing must fail the + # assertion instead of vacuously passing (same fix as + # assert_score_at_least for evaluator filters). With + # require_applicable=True the not-applicable report below already + # fails the call, and its wording is pinned by existing tests. + if not require_applicable: + if evaluator is not None and matching_scores_count == 0: + available = f": {', '.join(sorted(seen_evaluators))}" if seen_evaluators else ": none" + raise EvalNotPassedError( + msg or f"No scores found for evaluator '{evaluator}'. Available evaluator(s){available}" + ) + if not dimension_seen: + available = f": {', '.join(sorted(seen_dimensions))}" if seen_dimensions else ": none" + raise EvalNotPassedError( + msg or f"No scores found for dimension '{dimension_id}'. Available dimension(s){available}" + ) problems: list[str] = [] if offenders: problems.append( diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index f086d80b315..c9681dea089 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -1210,6 +1210,46 @@ def test_evaluator_filter_isolates_offenders(self) -> None: # The low-scoring "other" evaluator is filtered out; "policy" passes. results.assert_dimension_score_at_least("clarity", 3, evaluator="policy") + def test_evaluator_filter_missing_evaluator_raises_by_default(self) -> None: + # A filter that matches nothing must fail instead of vacuously + # passing (mirror of the assert_score_at_least fix). + results = _rubric_results( + [ + EvalScoreResult( + name="other", + score=0.9, + dimensions=[RubricScore(id="clarity", score=4, applicable=True, weight=1, reason="")], + ) + ], + ) + with pytest.raises(EvalNotPassedError) as exc: + results.assert_dimension_score_at_least("clarity", 3, evaluator="policy") + msg = str(exc.value) + assert "No scores found for evaluator 'policy'" in msg + assert "other" in msg + + def test_dimension_filter_missing_dimension_raises_by_default(self) -> None: + results = _rubric_results( + [ + EvalScoreResult( + name="policy", + score=0.9, + dimensions=[RubricScore(id="clarity", score=4, applicable=True, weight=1, reason="")], + ) + ], + ) + with pytest.raises(EvalNotPassedError) as exc: + results.assert_dimension_score_at_least("coherence", 3) + msg = str(exc.value) + assert "No scores found for dimension 'coherence'" in msg + assert "clarity" in msg + + def test_dimension_filter_empty_results_raises_by_default(self) -> None: + results = _rubric_results([]) + with pytest.raises(EvalNotPassedError) as exc: + results.assert_dimension_score_at_least("clarity", 3) + assert "No scores found for dimension 'clarity'. Available dimension(s): none" in str(exc.value) + def _score_results( *scores_per_item: list[EvalScoreResult], From 33c5c0b0d2474c0e2fcb97e8da035b4a13f030f2 Mon Sep 17 00:00:00 2001 From: Wilson0511 <1795618152@qq.com> Date: Mon, 9 Nov 2026 21:14:33 +0800 Subject: [PATCH 2/2] Fail empty evaluations under require_applicable too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot review: a bare EvalResults (no items) still passed silently with require_applicable=True — missing_items stays empty when there are no items to report, so the stricter option let an empty CI gate succeed. Run the no-match checks whenever missing_items is empty (all-items- applicable runs pass them harmlessly); the not-applicable report keeps carrying non-empty missing sets with its pinned wording. Co-Authored-By: ZCode --- python/packages/core/agent_framework/_evaluation.py | 13 +++++++------ python/packages/core/tests/core/test_local_eval.py | 9 +++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 5612f142d0a..ddd679e0f44 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -625,12 +625,13 @@ def _check(results: EvalResults) -> None: _check(sub) _check(self) - # With the default flags, a filter that matches nothing must fail the - # assertion instead of vacuously passing (same fix as - # assert_score_at_least for evaluator filters). With - # require_applicable=True the not-applicable report below already - # fails the call, and its wording is pinned by existing tests. - if not require_applicable: + # A filter that matches nothing must fail the assertion instead of + # vacuously passing (same fix as assert_score_at_least). The checks + # run under the default flags, and also when require_applicable=True + # has no items to report as not-applicable — an empty evaluation must + # not pass the stricter option either. When items are missing, the + # not-applicable report below carries them (wording pinned by tests). + if not require_applicable or not missing_items: if evaluator is not None and matching_scores_count == 0: available = f": {', '.join(sorted(seen_evaluators))}" if seen_evaluators else ": none" raise EvalNotPassedError( diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index c9681dea089..b54bd7cdd54 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -1250,6 +1250,15 @@ def test_dimension_filter_empty_results_raises_by_default(self) -> None: results.assert_dimension_score_at_least("clarity", 3) assert "No scores found for dimension 'clarity'. Available dimension(s): none" in str(exc.value) + def test_empty_results_raise_even_with_require_applicable(self) -> None: + # Zero items vacuously passed even the stricter flag (Copilot review): + # with no items, missing_items stays empty and nothing was checked. + results = EvalResults(provider="test") + with pytest.raises(EvalNotPassedError, match="No scores found for dimension"): + results.assert_dimension_score_at_least("clarity", 3, require_applicable=True) + with pytest.raises(EvalNotPassedError, match="No scores found for evaluator 'policy'"): + results.assert_dimension_score_at_least("clarity", 3, evaluator="policy", require_applicable=True) + def _score_results( *scores_per_item: list[EvalScoreResult],