Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,9 @@ All notable changes to SkillOpt are documented here. This project adheres to
instead of treating them as missing, and load integral floats such as
`1776.0` as `1776`, so a correct integer answer is no longer scored wrong
(thanks @Shizoqua).
- Fail a SpreadsheetBench test case when its answer position is empty instead
of passing it with no cells compared, which gave any output workbook,
including a wrong one, a full score (thanks @Shizoqua).

### Security
- Prevent managed-identity credentials from being sent to non-Azure or
Expand Down
5 changes: 5 additions & 0 deletions skillopt/envs/spreadsheetbench/evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,10 +125,12 @@ def compare_workbooks(gt_file: str, proc_file: str, answer_position: str) -> tup
try:
ok_all = True
msg_first = ""
n_compared = 0
for scr in (answer_position or "").split(","):
scr = scr.strip()
if not scr:
continue
n_compared += 1
if "!" in scr:
sheet_name, cell_range = scr.split("!", 1)
sheet_name = sheet_name.strip().strip("'\"")
Expand All @@ -141,6 +143,9 @@ def compare_workbooks(gt_file: str, proc_file: str, answer_position: str) -> tup
ok_all = False
if not msg_first:
msg_first = msg
if not n_compared:
# Nothing was compared, so there is no evidence the output is right.
return False, "no answer position to compare"
return ok_all, msg_first
finally:
wb_gt.close()
Expand Down
36 changes: 36 additions & 0 deletions tests/test_spreadsheetbench_answer_position.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
"""SpreadsheetBench scoring must not pass a workbook when no cell was compared."""

from __future__ import annotations

import openpyxl
import pytest

from skillopt.envs.spreadsheetbench.evaluator import evaluate


def _workbook(path, value) -> str:
wb = openpyxl.Workbook()
wb.active.title = "Sheet1"
wb.active["A1"] = value
wb.save(path)
return str(path)


@pytest.mark.parametrize("answer_position", ["", " ", ",", " , "])
def test_empty_answer_position_fails_closed(tmp_path, answer_position):
gold = _workbook(tmp_path / "gold.xlsx", 1)
wrong = _workbook(tmp_path / "pred.xlsx", 99)

result = evaluate(wrong, gold, "Cell-Level Manipulation", answer_position)

assert result["ok"] is False
assert "answer position" in result["reason"]


@pytest.mark.parametrize(("value", "ok"), [(1, True), (99, False)])
def test_non_empty_answer_position_still_compares(tmp_path, value, ok):
gold = _workbook(tmp_path / "gold.xlsx", 1)
pred = _workbook(tmp_path / "pred.xlsx", value)

assert evaluate(pred, gold, "Cell-Level Manipulation", "Sheet1!A1")["ok"] is ok
assert evaluate(pred, gold, "Cell-Level Manipulation", " , Sheet1!A1")["ok"] is ok