From 60c61a59d0024b81955b12357e80992e2130dce1 Mon Sep 17 00:00:00 2001 From: Kilian Adriano Lieret Date: Tue, 14 Jul 2026 11:41:36 -0400 Subject: [PATCH 1/3] fix(eval): remove stale ./executable before compile.sh The workspace wipe in _compile_executable runs before the submission tar is extracted, so an ./executable smuggled into the submission archive survived into the compile step and got stashed as if the build had produced it. A stub/no-op compile.sh could then pass with a prebuilt binary. _remove_hashed_files only catches byte-for-byte copies of the gold binary, not arbitrary prebuilts. Add an explicit 'rm -f ./executable' step after extraction and before compile.sh so the build is always forced to regenerate the binary, matching RevEngBench's git-clean-then-compile behavior. --- src/programbench/eval/eval.py | 13 +++++++++++ tests/test_eval.py | 43 +++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/src/programbench/eval/eval.py b/src/programbench/eval/eval.py index ba1f692c..618c2200 100644 --- a/src/programbench/eval/eval.py +++ b/src/programbench/eval/eval.py @@ -520,6 +520,19 @@ def _compile_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> log_buf=log_buf, step_name="seed_git", ) + # Remove any ./executable shipped in the submission archive so the + # build must produce a fresh binary. The workspace wipe above runs + # *before* the tar is extracted, so a prebuilt executable smuggled + # into the submission would otherwise survive into the compile step + # and be stashed as if the build had produced it (a stub/no-op + # compile.sh would then "pass"). _remove_hashed_files only catches + # byte-for-byte copies of the gold binary, not arbitrary prebuilts. + self._run_step( + "rm -f ./executable", + env=env, + log_buf=log_buf, + step_name="clear_stale_executable", + ) # Block internet during compile.sh so a submission can't smuggle # install/download steps into its build. Test-execution containers # are never touched (they may legitimately need network). diff --git a/tests/test_eval.py b/tests/test_eval.py index dcb95f4e..bf769e63 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -6,11 +6,14 @@ """Tests for the evaluation pipeline (data models, XML parsing, batch logic).""" +from pathlib import Path + import pytest from programbench.constants import TASKS_DIR from programbench.eval.eval import ( EvaluationResult, + Evaluator, TestBranchError, TestResult, _process_branch_xml, @@ -79,6 +82,46 @@ """ +class RecordingEnv: + """Fake ContainerEnvironment that records commands and echoes them back.""" + + def __init__(self): + self.commands: list[str] = [] + self.tars_copied: list[Path] = [] + + def execute(self, command: str, *, timeout: int | None = None) -> dict: + self.commands.append(command) + return {"output": command, "returncode": 0, "exception_info": ""} + + def copy_in_tar(self, tar_path: Path, container_path: str) -> None: + self.commands.append(f"__copy_in_tar__ {tar_path}") + self.tars_copied.append(tar_path) + + +class TestCompileClearsStaleExecutable: + def _run_compile(self): + evaluator = Evaluator(tests_branches=["b1"], submission_archive=Path("submission.tar.gz")) + env = RecordingEnv() + evaluator._compile_executable(env, log_buf=[]) + return env + + def test_stale_executable_removed_before_compile(self): + commands = self._run_compile().commands + clear_idx = next(i for i, c in enumerate(commands) if c == "rm -f ./executable") + compile_idx = next(i for i, c in enumerate(commands) if "compile.sh" in c) + extract_idx = next(i for i, c in enumerate(commands) if c.startswith("__copy_in_tar__")) + # The clear must happen after the submission is extracted (otherwise it + # removes nothing) and before compile.sh runs (so the build is forced + # to regenerate the binary). + assert extract_idx < clear_idx < compile_idx + + def test_workspace_wipe_precedes_extraction(self): + commands = self._run_compile().commands + wipe_idx = next(i for i, c in enumerate(commands) if c.startswith("rm -rf ")) + extract_idx = next(i for i, c in enumerate(commands) if c.startswith("__copy_in_tar__")) + assert wipe_idx < extract_idx + + class TestParseTestResults: def test_all_pass(self): result = parse_test_results(JUNIT_XML_ALL_PASS, branch="b1") From 02443357ff552bd77f2c66cfe0fc965b3b7fc251 Mon Sep 17 00:00:00 2001 From: Kilian Adriano Lieret Date: Tue, 14 Jul 2026 11:43:09 -0400 Subject: [PATCH 2/3] test(eval): drop mock-based ordering test It asserted the literal command sequence against a fake env, so it mirrored the implementation rather than exercising real behavior. --- tests/test_eval.py | 43 ------------------------------------------- 1 file changed, 43 deletions(-) diff --git a/tests/test_eval.py b/tests/test_eval.py index bf769e63..dcb95f4e 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -6,14 +6,11 @@ """Tests for the evaluation pipeline (data models, XML parsing, batch logic).""" -from pathlib import Path - import pytest from programbench.constants import TASKS_DIR from programbench.eval.eval import ( EvaluationResult, - Evaluator, TestBranchError, TestResult, _process_branch_xml, @@ -82,46 +79,6 @@ """ -class RecordingEnv: - """Fake ContainerEnvironment that records commands and echoes them back.""" - - def __init__(self): - self.commands: list[str] = [] - self.tars_copied: list[Path] = [] - - def execute(self, command: str, *, timeout: int | None = None) -> dict: - self.commands.append(command) - return {"output": command, "returncode": 0, "exception_info": ""} - - def copy_in_tar(self, tar_path: Path, container_path: str) -> None: - self.commands.append(f"__copy_in_tar__ {tar_path}") - self.tars_copied.append(tar_path) - - -class TestCompileClearsStaleExecutable: - def _run_compile(self): - evaluator = Evaluator(tests_branches=["b1"], submission_archive=Path("submission.tar.gz")) - env = RecordingEnv() - evaluator._compile_executable(env, log_buf=[]) - return env - - def test_stale_executable_removed_before_compile(self): - commands = self._run_compile().commands - clear_idx = next(i for i, c in enumerate(commands) if c == "rm -f ./executable") - compile_idx = next(i for i, c in enumerate(commands) if "compile.sh" in c) - extract_idx = next(i for i, c in enumerate(commands) if c.startswith("__copy_in_tar__")) - # The clear must happen after the submission is extracted (otherwise it - # removes nothing) and before compile.sh runs (so the build is forced - # to regenerate the binary). - assert extract_idx < clear_idx < compile_idx - - def test_workspace_wipe_precedes_extraction(self): - commands = self._run_compile().commands - wipe_idx = next(i for i, c in enumerate(commands) if c.startswith("rm -rf ")) - extract_idx = next(i for i, c in enumerate(commands) if c.startswith("__copy_in_tar__")) - assert wipe_idx < extract_idx - - class TestParseTestResults: def test_all_pass(self): result = parse_test_results(JUNIT_XML_ALL_PASS, branch="b1") From d356135d21fdf5314423d30f22e2d2d74a7614a4 Mon Sep 17 00:00:00 2001 From: Kilian Adriano Lieret Date: Tue, 14 Jul 2026 11:43:52 -0400 Subject: [PATCH 3/3] refactor(eval): clear stale executable right after remove-hashed step --- src/programbench/eval/eval.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/programbench/eval/eval.py b/src/programbench/eval/eval.py index 618c2200..2f17b8cd 100644 --- a/src/programbench/eval/eval.py +++ b/src/programbench/eval/eval.py @@ -492,6 +492,19 @@ def _compile_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> assert self.submission_archive is not None env.copy_in_tar(self.submission_archive, f"{WORKSPACE_DIR}/") self._remove_hashed_files(env, log_buf) + # Remove any ./executable shipped in the submission archive so the + # build must produce a fresh binary. The workspace wipe above runs + # *before* the tar is extracted, so a prebuilt executable smuggled + # into the submission would otherwise survive into the compile step + # and be stashed as if the build had produced it (a stub/no-op + # compile.sh would then "pass"). _remove_hashed_files only catches + # byte-for-byte copies of the gold binary, not arbitrary prebuilts. + self._run_step( + "rm -f ./executable", + env=env, + log_buf=log_buf, + step_name="clear_stale_executable", + ) # Seed a synthetic git repo if the submission didn't ship one. Build # scripts that depend on a working tree (jq submodules, calcurse's # autopoint, cargo+vergen, ...) succeed against this synthetic repo. @@ -520,19 +533,6 @@ def _compile_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> log_buf=log_buf, step_name="seed_git", ) - # Remove any ./executable shipped in the submission archive so the - # build must produce a fresh binary. The workspace wipe above runs - # *before* the tar is extracted, so a prebuilt executable smuggled - # into the submission would otherwise survive into the compile step - # and be stashed as if the build had produced it (a stub/no-op - # compile.sh would then "pass"). _remove_hashed_files only catches - # byte-for-byte copies of the gold binary, not arbitrary prebuilts. - self._run_step( - "rm -f ./executable", - env=env, - log_buf=log_buf, - step_name="clear_stale_executable", - ) # Block internet during compile.sh so a submission can't smuggle # install/download steps into its build. Test-execution containers # are never touched (they may legitimately need network).