From f6a32583279d0abc81eb884044be3037c58b4610 Mon Sep 17 00:00:00 2001 From: Lanre Shittu <136805224+Shizoqua@users.noreply.github.com> Date: Tue, 17 Nov 2026 08:59:17 +0000 Subject: [PATCH] fix(spreadsheetbench): fail a test case whose answer position is empty compare_workbooks skipped blank entries in answer_position and returned ok_all = True when the loop compared nothing. An empty, whitespace-only or comma-only answer_position, including an item with no answer_position field (the rollout defaults it to ""), therefore passed every output workbook, even a wrong one, and gave full hard and soft scores. Count the ranges actually compared and fail with "no answer position to compare" when there are none. Non-empty positions are compared as before, including ones with blank entries between commas. --- CHANGELOG.md | 3 ++ skillopt/envs/spreadsheetbench/evaluator.py | 5 +++ .../test_spreadsheetbench_answer_position.py | 36 +++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 tests/test_spreadsheetbench_answer_position.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cd0fffee..5289cf4b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -209,6 +209,9 @@ All notable changes to SkillOpt are documented here. This project adheres to instead of treating them as missing, and load integral floats such as `1776.0` as `1776`, so a correct integer answer is no longer scored wrong (thanks @Shizoqua). +- Fail a SpreadsheetBench test case when its answer position is empty instead + of passing it with no cells compared, which gave any output workbook, + including a wrong one, a full score (thanks @Shizoqua). ### Security - Prevent managed-identity credentials from being sent to non-Azure or diff --git a/skillopt/envs/spreadsheetbench/evaluator.py b/skillopt/envs/spreadsheetbench/evaluator.py index 3d8b84a69..e603e37a6 100644 --- a/skillopt/envs/spreadsheetbench/evaluator.py +++ b/skillopt/envs/spreadsheetbench/evaluator.py @@ -125,10 +125,12 @@ def compare_workbooks(gt_file: str, proc_file: str, answer_position: str) -> tup try: ok_all = True msg_first = "" + n_compared = 0 for scr in (answer_position or "").split(","): scr = scr.strip() if not scr: continue + n_compared += 1 if "!" in scr: sheet_name, cell_range = scr.split("!", 1) sheet_name = sheet_name.strip().strip("'\"") @@ -141,6 +143,9 @@ def compare_workbooks(gt_file: str, proc_file: str, answer_position: str) -> tup ok_all = False if not msg_first: msg_first = msg + if not n_compared: + # Nothing was compared, so there is no evidence the output is right. + return False, "no answer position to compare" return ok_all, msg_first finally: wb_gt.close() diff --git a/tests/test_spreadsheetbench_answer_position.py b/tests/test_spreadsheetbench_answer_position.py new file mode 100644 index 000000000..90c908163 --- /dev/null +++ b/tests/test_spreadsheetbench_answer_position.py @@ -0,0 +1,36 @@ +"""SpreadsheetBench scoring must not pass a workbook when no cell was compared.""" + +from __future__ import annotations + +import openpyxl +import pytest + +from skillopt.envs.spreadsheetbench.evaluator import evaluate + + +def _workbook(path, value) -> str: + wb = openpyxl.Workbook() + wb.active.title = "Sheet1" + wb.active["A1"] = value + wb.save(path) + return str(path) + + +@pytest.mark.parametrize("answer_position", ["", " ", ",", " , "]) +def test_empty_answer_position_fails_closed(tmp_path, answer_position): + gold = _workbook(tmp_path / "gold.xlsx", 1) + wrong = _workbook(tmp_path / "pred.xlsx", 99) + + result = evaluate(wrong, gold, "Cell-Level Manipulation", answer_position) + + assert result["ok"] is False + assert "answer position" in result["reason"] + + +@pytest.mark.parametrize(("value", "ok"), [(1, True), (99, False)]) +def test_non_empty_answer_position_still_compares(tmp_path, value, ok): + gold = _workbook(tmp_path / "gold.xlsx", 1) + pred = _workbook(tmp_path / "pred.xlsx", value) + + assert evaluate(pred, gold, "Cell-Level Manipulation", "Sheet1!A1")["ok"] is ok + assert evaluate(pred, gold, "Cell-Level Manipulation", " , Sheet1!A1")["ok"] is ok