diff --git a/python/packages/lab/gaia/agent_framework_lab_gaia/gaia.py b/python/packages/lab/gaia/agent_framework_lab_gaia/gaia.py index 28c9b8a07eb..f0191b29c9f 100644 --- a/python/packages/lab/gaia/agent_framework_lab_gaia/gaia.py +++ b/python/packages/lab/gaia/agent_framework_lab_gaia/gaia.py @@ -264,6 +264,88 @@ def _read_jsonl(path: Path) -> Iterable[dict[str, Any]]: yield record +_SHARDED_PARQUET_RE = re.compile(r"(.+)-(\d{5})-of-(\d{5})\.parquet") + + +def _shard_is_readable(path: Path) -> bool: + """Whether the shard's parquet footer can actually be opened. + + A truncated or corrupt file can still carry a complete looking name, and the + loader would skip the failing read and quietly run the benchmark over the + rest of the family, so a shard that cannot be read makes its family + incomplete. Without pyarrow there is nothing to validate and nothing the + loader could read either, so the name alone has to speak for the file. + """ + try: + import pyarrow.parquet as pq + except ImportError: + return True + try: + pq.ParquetFile(path) + except Exception: + return False + return True + + +def _has_complete_parquet_snapshot(data_dir: Path) -> bool: + """Whether every sharded metadata parquet group holds all of its declared shards. + + An interrupted copy can leave only part of a sharded family in the directory. + Treating any matching filename as ready would run the benchmark over a + silently truncated task set, so an incomplete family forces the normal + download path instead. Plain unsharded metadata parquet files count as + complete on their own. + """ + sharded: dict[tuple[Path, str], list[tuple[Path, int, int]]] = {} + unsharded_seen = False + # The shard suffix decides the family, whatever the prefix before it is: the + # loader globs every metadata*.parquet file, so a family named + # metadata-2023-validation-00000-of-00002.parquet must be counted as sharded + # too, not slip through as an unsharded file. + for path in data_dir.rglob("metadata*.parquet"): + match = _SHARDED_PARQUET_RE.fullmatch(path.name) + if match is None: + unsharded_seen = True + continue + key = (path.parent, match.group(1)) + sharded.setdefault(key, []).append((path, int(match.group(2)), int(match.group(3)))) + if not sharded: + return unsharded_seen + # The loader reads every matching parquet file, so a single incomplete sharded + # family disqualifies the directory even when unsharded files also exist. + # A family is complete only when every file declares the same total, the + # indexes are exactly zero through that total minus one, and every declared + # shard can actually be opened: counting files alone passes a family missing + # shard 0, a family with mixed totals, a family holding an index at or beyond + # the declared total, or a family whose corrupt shard the loader would skip. + for shards in sharded.values(): + totals = {total for _, _, total in shards} + if len(totals) != 1: + return False + indexes = {index for _, index, _ in shards} + if indexes != set(range(totals.pop())): + return False + if not all(_shard_is_readable(path) for path, _, _ in shards): + return False + return True + + +def _has_usable_local_data(data_dir: Path) -> bool: + """Whether the directory holds data that is safe to load without a download. + + A JSONL-only layout is usable when ``metadata.jsonl`` is present, and a parquet + layout is usable when every sharded family is complete. A mixed layout must + not pass merely because JSONL exists: ``_load_gaia_local`` reads parquet files + first and only falls back to JSONL when no parquet file is present, so an + incomplete parquet family would silently truncate the task set even though a + complete ``metadata.jsonl`` sits beside it. When any metadata parquet file is + present, the parquet snapshot must therefore be complete. + """ + if any(data_dir.rglob("metadata*.parquet")): + return _has_complete_parquet_snapshot(data_dir) + return any(data_dir.rglob("metadata.jsonl")) + + def _load_gaia_local(repo_dir: Path, wanted_levels: list[int] | None = None, max_n: int | None = None) -> list[Task]: """Load GAIA tasks from local repository directory.""" tasks: list[Task] = [] @@ -405,7 +487,7 @@ async def _default_evaluator(self, task: Task, prediction: Prediction) -> Evalua def _ensure_data(self) -> Path: """Ensure GAIA data is available locally.""" - if self.data_dir.exists() and any(self.data_dir.rglob("metadata.jsonl")): + if self.data_dir.exists() and _has_usable_local_data(self.data_dir): return self.data_dir # Download data if not available diff --git a/python/packages/lab/gaia/tests/test_gaia.py b/python/packages/lab/gaia/tests/test_gaia.py index 2af57b5f9a0..9d920737bd5 100644 --- a/python/packages/lab/gaia/tests/test_gaia.py +++ b/python/packages/lab/gaia/tests/test_gaia.py @@ -2,6 +2,9 @@ """Tests for GAIA benchmark implementation.""" +from pathlib import Path + +import pytest from agent_framework_lab_gaia import gaia_scorer @@ -34,3 +37,222 @@ def test_none_handling(self): """Test handling of None values.""" assert gaia_scorer("None", "test") is False assert gaia_scorer("", "test") is False + + +class TestEnsureData: + """Test local data detection in GAIA._ensure_data.""" + + def test_parquet_only_data_dir_is_accepted(self, tmp_path): + """A data dir in the parquet layout _load_gaia_local supports must not force a download (#9337).""" + pytest.importorskip("pyarrow") + from agent_framework_lab_gaia.gaia import GAIA + + validation = tmp_path / "2023" / "validation" + validation.mkdir(parents=True) + rows = [ + { + "task_id": "v1", + "Question": "What is 2+2?", + "Final answer": "4", + "Level": 1, + "file_name": "", + "Annotator Metadata": "{}", + } + ] + import pyarrow as pa + import pyarrow.parquet as pq + + pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-00000-of-00001.parquet") + + runner = GAIA(data_dir=str(tmp_path)) + assert runner._ensure_data() == tmp_path + + def test_incomplete_sharded_parquet_snapshot_forces_download(self, tmp_path, monkeypatch): + """A partial metadata-...-of-N family must not count as ready local data (#9337).""" + pytest.importorskip("pyarrow") + monkeypatch.delenv("HF_TOKEN", raising=False) + from agent_framework_lab_gaia.gaia import GAIA + + validation = tmp_path / "2023" / "validation" + validation.mkdir(parents=True) + rows = [{"task_id": "v1", "Question": "q", "Final answer": "a", "Level": 1, "file_name": ""}] + + import pyarrow as pa + import pyarrow.parquet as pq + + # interrupted copy: only shard 0 of 2 landed + pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-00000-of-00002.parquet") + + runner = GAIA(data_dir=str(tmp_path)) + with pytest.raises(RuntimeError, match="HF_TOKEN"): + runner._ensure_data() + + # completing the family makes the directory ready again + pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-00001-of-00002.parquet") + assert runner._ensure_data() == tmp_path + + def test_jsonl_does_not_rescue_an_incomplete_parquet_family(self, tmp_path, monkeypatch): + """A mixed layout must not pass on the JSONL shortcut (#9340 review). + + _load_gaia_local reads parquet first and only falls back to JSONL when + no parquet file is present, so metadata.jsonl beside an incomplete + sharded family would silently truncate the task set. + """ + pytest.importorskip("pyarrow") + monkeypatch.delenv("HF_TOKEN", raising=False) + from agent_framework_lab_gaia.gaia import GAIA + + validation = tmp_path / "2023" / "validation" + validation.mkdir(parents=True) + rows = [{"task_id": "v1", "Question": "q", "Final answer": "a", "Level": 1, "file_name": ""}] + + import pyarrow as pa + import pyarrow.parquet as pq + + (validation / "metadata.jsonl").write_text("{}\n", encoding="utf-8") + # interrupted copy: only shard 0 of 2 landed beside the JSONL file + pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-00000-of-00002.parquet") + + runner = GAIA(data_dir=str(tmp_path)) + with pytest.raises(RuntimeError, match="HF_TOKEN"): + runner._ensure_data() + + # completing the family makes the mixed directory ready again + pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-00001-of-00002.parquet") + assert runner._ensure_data() == tmp_path + + def test_jsonl_only_layout_is_still_accepted(self, tmp_path, monkeypatch): + """A JSONL-only directory keeps working without any parquet files (#9340 review).""" + monkeypatch.delenv("HF_TOKEN", raising=False) + from agent_framework_lab_gaia.gaia import GAIA + + validation = tmp_path / "2023" / "validation" + validation.mkdir(parents=True) + (validation / "metadata.jsonl").write_text("{}\n", encoding="utf-8") + + runner = GAIA(data_dir=str(tmp_path)) + assert runner._ensure_data() == tmp_path + + def _write_shard(self, validation: Path, index: int, total: int) -> None: + import pyarrow as pa + import pyarrow.parquet as pq + + rows = [{"task_id": "v1", "Question": "q", "Final answer": "a", "Level": 1, "file_name": ""}] + pq.write_table(pa.Table.from_pylist(rows), validation / f"metadata-{index:05d}-of-{total:05d}.parquet") + + def _write_named_shard(self, validation: Path, name: str) -> None: + import pyarrow as pa + import pyarrow.parquet as pq + + rows = [{"task_id": "v1", "Question": "q", "Final answer": "a", "Level": 1, "file_name": ""}] + pq.write_table(pa.Table.from_pylist(rows), validation / name) + + def test_sharded_family_with_a_dashed_prefix_is_still_counted(self, tmp_path, monkeypatch): + """A family whose prefix carries extra name parts must not slip past as unsharded. + + The loader globs every metadata*.parquet file, so + metadata-2023-validation-00000-of-00002.parquet belongs to a sharded + family even though its prefix has more than one dash separated part. + """ + pytest.importorskip("pyarrow") + monkeypatch.delenv("HF_TOKEN", raising=False) + from agent_framework_lab_gaia.gaia import GAIA + + validation = tmp_path / "2023" / "validation" + validation.mkdir(parents=True) + self._write_named_shard(validation, "metadata-2023-validation-00000-of-00002.parquet") + + runner = GAIA(data_dir=str(tmp_path)) + with pytest.raises(RuntimeError, match="HF_TOKEN"): + runner._ensure_data() + + self._write_named_shard(validation, "metadata-2023-validation-00001-of-00002.parquet") + assert runner._ensure_data() == tmp_path + + def test_corrupt_shard_disqualifies_a_complete_looking_family(self, tmp_path, monkeypatch): + """A shard that cannot be read must force the download path. + + The loader skips a shard whose read fails and would run the benchmark + over the remaining shards, so a complete looking name is not enough. + """ + pytest.importorskip("pyarrow") + monkeypatch.delenv("HF_TOKEN", raising=False) + from agent_framework_lab_gaia.gaia import GAIA + + validation = tmp_path / "2023" / "validation" + validation.mkdir(parents=True) + self._write_shard(validation, 0, 2) + (validation / "metadata-00001-of-00002.parquet").write_bytes(b"this is not a parquet file") + + runner = GAIA(data_dir=str(tmp_path)) + with pytest.raises(RuntimeError, match="HF_TOKEN"): + runner._ensure_data() + + def test_sharded_family_missing_shard_zero_forces_download(self, tmp_path, monkeypatch): + """A family holding every index except zero must not count as complete.""" + pytest.importorskip("pyarrow") + monkeypatch.delenv("HF_TOKEN", raising=False) + from agent_framework_lab_gaia.gaia import GAIA + + validation = tmp_path / "2023" / "validation" + validation.mkdir(parents=True) + self._write_shard(validation, 1, 2) + self._write_shard(validation, 2, 2) + + runner = GAIA(data_dir=str(tmp_path)) + with pytest.raises(RuntimeError, match="HF_TOKEN"): + runner._ensure_data() + + def test_sharded_family_with_mixed_totals_forces_download(self, tmp_path, monkeypatch): + """Files in one family declaring different totals describe an incoherent copy.""" + pytest.importorskip("pyarrow") + monkeypatch.delenv("HF_TOKEN", raising=False) + from agent_framework_lab_gaia.gaia import GAIA + + validation = tmp_path / "2023" / "validation" + validation.mkdir(parents=True) + # Three distinct indexes against a largest declared total of three, so a + # count against the max total passes while the family disagrees on N. + self._write_shard(validation, 0, 3) + self._write_shard(validation, 1, 3) + self._write_shard(validation, 2, 3) + self._write_shard(validation, 2, 2) + + runner = GAIA(data_dir=str(tmp_path)) + with pytest.raises(RuntimeError, match="HF_TOKEN"): + runner._ensure_data() + + def test_sharded_family_with_an_out_of_range_index_forces_download(self, tmp_path, monkeypatch): + """An index at or beyond the declared total must not stand in for a missing shard.""" + pytest.importorskip("pyarrow") + monkeypatch.delenv("HF_TOKEN", raising=False) + from agent_framework_lab_gaia.gaia import GAIA + + validation = tmp_path / "2023" / "validation" + validation.mkdir(parents=True) + self._write_shard(validation, 0, 2) + self._write_shard(validation, 2, 2) + + runner = GAIA(data_dir=str(tmp_path)) + with pytest.raises(RuntimeError, match="HF_TOKEN"): + runner._ensure_data() + + def test_unsharded_file_does_not_rescue_an_incomplete_family(self, tmp_path, monkeypatch): + """An unsharded file must not mask a partial sharded family (#9337 review).""" + pytest.importorskip("pyarrow") + monkeypatch.delenv("HF_TOKEN", raising=False) + from agent_framework_lab_gaia.gaia import GAIA + + validation = tmp_path / "2023" / "validation" + validation.mkdir(parents=True) + rows = [{"task_id": "v1", "Question": "q", "Final answer": "a", "Level": 1, "file_name": ""}] + + import pyarrow as pa + import pyarrow.parquet as pq + + pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-00000-of-00002.parquet") + pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-extra.parquet") + + runner = GAIA(data_dir=str(tmp_path)) + with pytest.raises(RuntimeError, match="HF_TOKEN"): + runner._ensure_data()