diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 8ee060870c7..812af84f9dd 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -1382,6 +1382,12 @@ def _coerce_result(value: Any, check_name: str) -> CheckResult: if isinstance(value, dict): d = cast(dict[str, Any], value) + if "passed" in d: + passed_val = d["passed"] + if not isinstance(passed_val, (bool, int)): + raise TypeError( + f"Function evaluator '{check_name}' returned dict with non-boolean 'passed' value: {passed_val!r}" + ) if "score" in d: try: score = float(d["score"]) @@ -1395,10 +1401,6 @@ def _coerce_result(value: Any, check_name: str) -> CheckResult: return CheckResult(passed=passed, reason=reason, check_name=check_name) if "passed" in d: passed_val = d["passed"] - if not isinstance(passed_val, (bool, int)): - raise TypeError( - f"Function evaluator '{check_name}' returned dict with non-boolean 'passed' value: {passed_val!r}" - ) return CheckResult( passed=bool(passed_val), reason=str(d.get("reason", "passed" if passed_val else "failed")), diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index 9875b2aeeee..55a1ebaac8d 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -267,6 +267,15 @@ def explicit_pass(response: str) -> dict: assert result.passed is True assert result.reason == "all good" + @pytest.mark.asyncio + async def test_dict_with_score_and_string_passed_raises(self): + @evaluator + def bad_passed_with_score(response: str) -> dict: + return {"score": 0.1, "passed": "false", "reason": "judge said no"} + + with pytest.raises(TypeError, match="non-boolean 'passed' value: 'false'"): + await bad_passed_with_score(_make_item()) # type: ignore[misc] # pyrefly: ignore[not-async] # ty: ignore[invalid-await] + @pytest.mark.asyncio async def test_check_result_passthrough(self): @evaluator @@ -985,6 +994,23 @@ def test_none_score_raises(self): with pytest.raises(TypeError, match="non-numeric 'score'"): _coerce_result({"score": None}, "test_check") + def test_non_boolean_passed_with_score_raises(self): + """Dict with score and non-boolean passed string raises TypeError (issue #10165).""" + with pytest.raises(TypeError, match="non-boolean 'passed' value: 'false'"): + _coerce_result({"score": 0.1, "passed": "false"}, "test_check") + + def test_none_passed_with_score_raises(self): + """Dict with score and None passed raises TypeError.""" + with pytest.raises(TypeError, match="non-boolean 'passed' value: None"): + _coerce_result({"score": 0.8, "passed": None}, "test_check") + + def test_boolean_passed_with_score_respected(self): + """Valid boolean passed override is respected when score is present.""" + result_false = _coerce_result({"score": 0.9, "passed": False}, "test_check") + assert result_false.passed is False + result_true = _coerce_result({"score": 0.1, "passed": True}, "test_check") + assert result_true.passed is True + class TestBareCheckViaEvaluateAgent: """Test bare callable check functions through the public evaluate_agent API."""