From 5eb8a6f1d36dff06a53117f4353b007172f27f97 Mon Sep 17 00:00:00 2001
From: Shaurya Singh <298017155+shaurya416@users.noreply.github.com>
Date: Fri, 25 Sep 2026 13:16:06 -0700
Subject: [PATCH] Score MultiPL-E pass@k from a per-run temp dir

process_results wrote each problem and its .results.json into
tempfile.gettempdir(), then computed pass@k over every *.results.json
found there. Files left by an earlier run, or by the previous
multiple-<lang> task in the same invocation, were averaged into the
score, because problem names are shared across languages and nothing
removes the files.

Use tempfile.mkdtemp() so each call writes to and globs a directory of
its own. The files are still kept after the run, as before. Add tests
that fail on the old behaviour.
---
 bigcode_eval/tasks/multiple.py |   5 +-
 tests/test_multiple.py         | 106 +++++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+), 2 deletions(-)
 create mode 100644 tests/test_multiple.py

diff --git a/bigcode_eval/tasks/multiple.py b/bigcode_eval/tasks/multiple.py
index 875c648f1..bdf25d05c 100644
--- a/bigcode_eval/tasks/multiple.py
+++ b/bigcode_eval/tasks/multiple.py
@@ -151,8 +151,9 @@ def process_results(self, generations, references):
             for i, doc in enumerate(self.get_dataset())
             if i < len(generations)
         ]
-        # a common temp dir for all the problems
-        temp_dir = tempfile.gettempdir()
+        # a fresh temp dir for the problems of this run, so that results files
+        # left in the shared temp dir by other runs or tasks are not scored
+        temp_dir = tempfile.mkdtemp()
         list_files = []
         for (prompt_name, generation, reference) in zip(
             prompts_names, generations, references
diff --git a/tests/test_multiple.py b/tests/test_multiple.py
new file mode 100644
index 000000000..6f91c7dd7
--- /dev/null
+++ b/tests/test_multiple.py
@@ -0,0 +1,106 @@
+import json
+import os
+import tempfile
+
+import pytest
+
+from bigcode_eval.tasks import multiple
+from bigcode_eval.tasks.custom_metrics.multiple_metrics.evaluation import \
+    get_test_results_json_path
+
+NAMES = [
+    "HumanEval_0_has_close_elements",
+    "HumanEval_1_separate_paren_groups",
+    "HumanEval_2_truncate_number",
+]
+
+
+class DummyMultiPLE(multiple.GeneralMultiPLE):
+    # skips loading the dataset from the hub
+    def __init__(self, language, names):
+        self.language = language
+        self.docs = [
+            {"prompt": f"prompt {name}", "name": name, "language": language}
+            for name in names
+        ]
+
+    def get_dataset(self):
+        return self.docs
+
+
+def fake_evaluate_problem(output_dir, problem_json_path, max_workers):
+    # writes a results file in the same format as the real evaluate_problem,
+    # a completion passes when it ends with "pass"
+    with open(problem_json_path) as f:
+        problem = json.load(f)
+    results = []
+    for completion in problem["completions"]:
+        ok = completion.endswith("pass")
+        status = "OK" if ok else "Exception"
+        results.append({"status": status, "exit_code": 0 if ok else 1})
+    test_results = {"name": problem["name"], "results": results}
+    path = get_test_results_json_path(output_dir, problem_json_path, None)
+    with open(path, "w") as f:
+        json.dump(test_results, f)
+
+
+@pytest.fixture
+def system_tmp(tmp_path, monkeypatch):
+    # stand-in for the system temp dir, shared by every run on the machine
+    monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
+    monkeypatch.setattr(multiple, "evaluate_problem", fake_evaluate_problem)
+    return tmp_path
+
+
+def write_leftover(directory, name, n_ok):
+    results = [{"status": "OK", "exit_code": 0}] * n_ok
+    with open(os.path.join(directory, f"{name}.results.json"), "w") as f:
+        json.dump({"name": name, "results": results}, f)
+
+
+def run(task, generations):
+    references = ["tests"] * len(generations)
+    return task.process_results(generations, references)
+
+
+def test_single_run_mixed(system_tmp):
+    task = DummyMultiPLE("js", NAMES[:2])
+    assert run(task, [["fail"], ["pass"]]) == {"pass@1": 0.5}
+
+
+def test_single_run_all_pass_and_all_fail(system_tmp):
+    task = DummyMultiPLE("js", NAMES)
+    assert run(task, [["pass"]] * 3) == {"pass@1": 1.0}
+    assert run(task, [["fail"]] * 3) == {"pass@1": 0.0}
+
+
+def test_several_samples_per_problem(system_tmp):
+    task = DummyMultiPLE("js", NAMES[:2])
+    generations = [["pass"] * 3 + ["fail"] * 7, ["fail"] * 10]
+    results = run(task, generations)
+    assert set(results) == {"pass@1", "pass@10"}
+    assert results["pass@1"] == pytest.approx(0.15)
+    assert results["pass@10"] == pytest.approx(0.5)
+
+
+def test_leftover_results_file_is_not_scored(system_tmp):
+    write_leftover(system_tmp, "HumanEval_99_leftover", n_ok=20)
+    task = DummyMultiPLE("js", NAMES[:1])
+    assert run(task, [["fail"]]) == {"pass@1": 0.0}
+
+
+def test_leftover_results_file_is_left_alone(system_tmp):
+    write_leftover(system_tmp, "HumanEval_99_leftover", n_ok=20)
+    task = DummyMultiPLE("js", NAMES[:1])
+    run(task, [["fail"]])
+    assert os.path.exists(
+        os.path.join(system_tmp, "HumanEval_99_leftover.results.json")
+    )
+
+
+def test_previous_task_in_same_process_is_not_scored(system_tmp):
+    # problem names are shared across languages, js has more problems than d
+    js = DummyMultiPLE("js", NAMES)
+    d = DummyMultiPLE("d", NAMES[:2])
+    assert run(js, [["pass"]] * 3) == {"pass@1": 1.0}
+    assert run(d, [["fail"]] * 2) == {"pass@1": 0.0}