diff --git a/src/eva/metrics/utils.py b/src/eva/metrics/utils.py index 13c083aa..405d8075 100644 --- a/src/eva/metrics/utils.py +++ b/src/eva/metrics/utils.py @@ -16,6 +16,15 @@ logger = get_logger(__name__) +# agent_perf_stats.csv stores the full serialized prompt (system prompt + message history) per +# LLM call in a single field. Agents with large system prompts (e.g. medical_hr's ~40-tool, +# long-instruction config) routinely exceed Python's default 131072-byte csv field limit, which +# raises _csv.Error and — since mean_agent_perf_stat is called from turn_taking's compute() — +# takes down the entire turn_taking metric (score forced to 0, details wiped) instead of just the +# token sub-metrics that actually depend on this file. Raise the limit well above any observed +# field size (largest seen so far ~450KB) so legitimate large prompts parse successfully. +csv.field_size_limit(10_000_000) + def parse_judge_response(response_text: str, record_id: str, metric_logger) -> dict | None: """Parse LLM judge response using robust JSON extraction. @@ -524,7 +533,10 @@ def mean_agent_perf_stat( try: with open(csv_path, newline="", encoding="utf-8") as f: rows = list(csv.DictReader(f)) - except OSError: + except (OSError, csv.Error): + # csv.Error covers e.g. a field exceeding csv.field_size_limit() — treat it the same as a + # missing/unreadable file (this sub-metric is best-effort) rather than letting it propagate + # and fail the caller's entire metric. return None if not rows: diff --git a/tests/unit/metrics/test_metrics_utils.py b/tests/unit/metrics/test_metrics_utils.py index 1d21782b..07a415c5 100644 --- a/tests/unit/metrics/test_metrics_utils.py +++ b/tests/unit/metrics/test_metrics_utils.py @@ -11,6 +11,7 @@ compute_aggregation, extract_wer_errors, format_transcript, + mean_agent_perf_stat, normalize_rating, parse_judge_response, parse_judge_response_list, @@ -223,3 +224,58 @@ def test_aggregates(self): assert "top_substitutions" in result assert "top_deletions" in result assert "top_insertions" in result + + +class TestMeanAgentPerfStat: + def _write_csv(self, tmp_path, rows): + import csv + + csv_path = tmp_path / "agent_perf_stats.csv" + with open(csv_path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=["prompt", "output_tokens", "reasoning_tokens"]) + writer.writeheader() + writer.writerows(rows) + return tmp_path + + def test_missing_file_returns_none(self, tmp_path): + assert mean_agent_perf_stat(tmp_path, "output_tokens") is None + + def test_averages_numeric_column(self, tmp_path): + output_dir = self._write_csv( + tmp_path, + [ + {"prompt": "p1", "output_tokens": "10", "reasoning_tokens": "0"}, + {"prompt": "p2", "output_tokens": "20", "reasoning_tokens": "5"}, + ], + ) + assert mean_agent_perf_stat(output_dir, "output_tokens") == pytest.approx(15.0) + assert mean_agent_perf_stat(output_dir, "reasoning_tokens") == pytest.approx(2.5) + + def test_blank_and_non_numeric_values_are_skipped(self, tmp_path): + output_dir = self._write_csv( + tmp_path, + [ + {"prompt": "p1", "output_tokens": "10", "reasoning_tokens": ""}, + {"prompt": "p2", "output_tokens": "not-a-number", "reasoning_tokens": "5"}, + ], + ) + assert mean_agent_perf_stat(output_dir, "output_tokens") == pytest.approx(10.0) + assert mean_agent_perf_stat(output_dir, "reasoning_tokens") == pytest.approx(5.0) + + def test_oversized_prompt_field_does_not_raise(self, tmp_path): + """Regression test. + + A large system prompt (e.g. medical_hr's ~40-tool config) can push + the ``prompt`` field of a single row past Python's default 131072-byte csv field limit. + Before this fix, that raised ``_csv.Error`` out of ``mean_agent_perf_stat`` and, since + turn_taking's ``compute()`` calls it unguarded, zeroed out the entire turn_taking metric + for every record. It must instead either parse successfully (limit was raised) or + degrade gracefully to ``None`` — never raise. + """ + huge_prompt = "x" * 300_000 + output_dir = self._write_csv( + tmp_path, + [{"prompt": huge_prompt, "output_tokens": "42", "reasoning_tokens": "1"}], + ) + result = mean_agent_perf_stat(output_dir, "output_tokens") + assert result == pytest.approx(42.0)