fix(eval): track unhandled scores per row for the error score handler

py/src/braintrust/framework.py

@@ -1686,7 +1686,6 @@ async def await_or_run_classifier(root_span, classifier, name, **kwargs):
     scorer_names = [_scorer_name(scorer, i) for i, scorer in enumerate(scorers)]
     classifiers = list(evaluator.classifiers or [])
     classifier_names = [_classifier_name(classifier, i) for i, classifier in enumerate(classifiers)]
-    unhandled_scores = scorer_names
 
     if evaluator.parameter_values is not None:
         resolved_evaluator_parameters = evaluator.parameter_values
@@ -1706,6 +1705,7 @@ async def run_evaluator_task(datum, trial_index=0):
         error = None
         exc_info = None
         scores = {}
+        unhandled_scores = scorer_names
         classifications = {}
         tags = datum.tags
 
@@ -1919,7 +1919,6 @@ async def ensure_spans_flushed():
                 if classifications:
                     root_span.log(classifications=classifications)
 
-                nonlocal unhandled_scores
                 unhandled_scores = None
                 if failing_scorers_and_exceptions:
                     scorer_errors = {

py/src/braintrust/test_framework.py

@@ -20,6 +20,7 @@
     Filter,
     _call_user_fn_args,
     await_or_run,
+    default_error_score_handler,
     evaluate_filter,
     parse_filters,
     run_evaluator,
@@ -1134,6 +1135,46 @@ def task_with_hooks(input_value: int, hooks: EvalHooks) -> int:
     assert input_3_trials == [0]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("first_input", ["ok", "scorer_error"])
+async def test_error_score_handler_scores_each_failed_row_independently(first_input):
+    """A failed task gets error scores for every scorer, regardless of how earlier rows finished."""
+
+    async def task(input_value):
+        if input_value == "task_error":
+            raise ValueError("task failed")
+        return input_value
+
+    def exact_match(output, expected):
+        return 1.0 if output == expected else 0.0
+
+    def picky(output):
+        if output == "scorer_error":
+            raise ValueError("scorer failed")
+        return 1.0
+
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-error-score-handler",
+        data=[
+            EvalCase(input=first_input, expected=first_input),
+            EvalCase(input="task_error", expected="task_error"),
+        ],
+        task=task,
+        scores=[exact_match, picky],
+        experiment_name=None,
+        metadata=None,
+        max_concurrency=1,
+        error_score_handler=default_error_score_handler,
+    )
+
+    result = await run_evaluator(experiment=None, evaluator=evaluator, position=None, filters=[])
+
+    failed = next(r for r in result.results if r.input == "task_error")
+    assert isinstance(failed.error, ValueError)
+    assert failed.scores == {"exact_match": 0, "picky": 0}
+
+
 @pytest.mark.vcr
 @pytest.mark.asyncio
 async def test_scorer_spans_have_purpose_attribute(with_memory_logger, with_simulate_login):