Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 83 additions & 1 deletion python/packages/lab/gaia/agent_framework_lab_gaia/gaia.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,88 @@ def _read_jsonl(path: Path) -> Iterable[dict[str, Any]]:
yield record


_SHARDED_PARQUET_RE = re.compile(r"(.+)-(\d{5})-of-(\d{5})\.parquet")


def _shard_is_readable(path: Path) -> bool:
"""Whether the shard's parquet footer can actually be opened.

A truncated or corrupt file can still carry a complete looking name, and the
loader would skip the failing read and quietly run the benchmark over the
rest of the family, so a shard that cannot be read makes its family
incomplete. Without pyarrow there is nothing to validate and nothing the
loader could read either, so the name alone has to speak for the file.
"""
try:
import pyarrow.parquet as pq
except ImportError:
return True
try:
pq.ParquetFile(path)
except Exception:
return False
return True


def _has_complete_parquet_snapshot(data_dir: Path) -> bool:
"""Whether every sharded metadata parquet group holds all of its declared shards.

An interrupted copy can leave only part of a sharded family in the directory.
Treating any matching filename as ready would run the benchmark over a
silently truncated task set, so an incomplete family forces the normal
download path instead. Plain unsharded metadata parquet files count as
complete on their own.
"""
sharded: dict[tuple[Path, str], list[tuple[Path, int, int]]] = {}
unsharded_seen = False
# The shard suffix decides the family, whatever the prefix before it is: the
# loader globs every metadata*.parquet file, so a family named
# metadata-2023-validation-00000-of-00002.parquet must be counted as sharded
# too, not slip through as an unsharded file.
for path in data_dir.rglob("metadata*.parquet"):
match = _SHARDED_PARQUET_RE.fullmatch(path.name)
if match is None:
unsharded_seen = True
continue
key = (path.parent, match.group(1))
sharded.setdefault(key, []).append((path, int(match.group(2)), int(match.group(3))))
if not sharded:
return unsharded_seen
# The loader reads every matching parquet file, so a single incomplete sharded
# family disqualifies the directory even when unsharded files also exist.
# A family is complete only when every file declares the same total, the
# indexes are exactly zero through that total minus one, and every declared
# shard can actually be opened: counting files alone passes a family missing
# shard 0, a family with mixed totals, a family holding an index at or beyond
# the declared total, or a family whose corrupt shard the loader would skip.
for shards in sharded.values():
totals = {total for _, _, total in shards}
if len(totals) != 1:
return False
indexes = {index for _, index, _ in shards}
if indexes != set(range(totals.pop())):
return False
if not all(_shard_is_readable(path) for path, _, _ in shards):
return False
return True

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Completeness here is filename-only. If all declared files exist but one shard is truncated or corrupt, this returns true; _load_gaia_local then catches that shard's read failure and runs with tasks from the remaining shards, silently producing a partial benchmark. Please validate that every accepted shard is readable before reporting the snapshot as usable, and fail the family if any shard cannot be loaded.



def _has_usable_local_data(data_dir: Path) -> bool:
"""Whether the directory holds data that is safe to load without a download.

A JSONL-only layout is usable when ``metadata.jsonl`` is present, and a parquet
layout is usable when every sharded family is complete. A mixed layout must
not pass merely because JSONL exists: ``_load_gaia_local`` reads parquet files
first and only falls back to JSONL when no parquet file is present, so an
incomplete parquet family would silently truncate the task set even though a
complete ``metadata.jsonl`` sits beside it. When any metadata parquet file is
present, the parquet snapshot must therefore be complete.
"""
if any(data_dir.rglob("metadata*.parquet")):
return _has_complete_parquet_snapshot(data_dir)
return any(data_dir.rglob("metadata.jsonl"))


def _load_gaia_local(repo_dir: Path, wanted_levels: list[int] | None = None, max_n: int | None = None) -> list[Task]:
"""Load GAIA tasks from local repository directory."""
tasks: list[Task] = []
Expand Down Expand Up @@ -405,7 +487,7 @@ async def _default_evaluator(self, task: Task, prediction: Prediction) -> Evalua

def _ensure_data(self) -> Path:
"""Ensure GAIA data is available locally."""
if self.data_dir.exists() and any(self.data_dir.rglob("metadata.jsonl")):
if self.data_dir.exists() and _has_usable_local_data(self.data_dir):
return self.data_dir

# Download data if not available
Expand Down
222 changes: 222 additions & 0 deletions python/packages/lab/gaia/tests/test_gaia.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

"""Tests for GAIA benchmark implementation."""

from pathlib import Path

import pytest
from agent_framework_lab_gaia import gaia_scorer


Expand Down Expand Up @@ -34,3 +37,222 @@ def test_none_handling(self):
"""Test handling of None values."""
assert gaia_scorer("None", "test") is False
assert gaia_scorer("", "test") is False


class TestEnsureData:
"""Test local data detection in GAIA._ensure_data."""

def test_parquet_only_data_dir_is_accepted(self, tmp_path):
"""A data dir in the parquet layout _load_gaia_local supports must not force a download (#9337)."""
pytest.importorskip("pyarrow")
from agent_framework_lab_gaia.gaia import GAIA

validation = tmp_path / "2023" / "validation"
validation.mkdir(parents=True)
rows = [
{
"task_id": "v1",
"Question": "What is 2+2?",
"Final answer": "4",
"Level": 1,
"file_name": "",
"Annotator Metadata": "{}",
}
]
import pyarrow as pa
import pyarrow.parquet as pq

pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-00000-of-00001.parquet")

runner = GAIA(data_dir=str(tmp_path))
assert runner._ensure_data() == tmp_path

def test_incomplete_sharded_parquet_snapshot_forces_download(self, tmp_path, monkeypatch):
"""A partial metadata-...-of-N family must not count as ready local data (#9337)."""
pytest.importorskip("pyarrow")
monkeypatch.delenv("HF_TOKEN", raising=False)
from agent_framework_lab_gaia.gaia import GAIA

validation = tmp_path / "2023" / "validation"
validation.mkdir(parents=True)
rows = [{"task_id": "v1", "Question": "q", "Final answer": "a", "Level": 1, "file_name": ""}]

import pyarrow as pa
import pyarrow.parquet as pq

# interrupted copy: only shard 0 of 2 landed
pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-00000-of-00002.parquet")

runner = GAIA(data_dir=str(tmp_path))
with pytest.raises(RuntimeError, match="HF_TOKEN"):
runner._ensure_data()

# completing the family makes the directory ready again
pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-00001-of-00002.parquet")
assert runner._ensure_data() == tmp_path

def test_jsonl_does_not_rescue_an_incomplete_parquet_family(self, tmp_path, monkeypatch):
"""A mixed layout must not pass on the JSONL shortcut (#9340 review).

_load_gaia_local reads parquet first and only falls back to JSONL when
no parquet file is present, so metadata.jsonl beside an incomplete
sharded family would silently truncate the task set.
"""
pytest.importorskip("pyarrow")
monkeypatch.delenv("HF_TOKEN", raising=False)
from agent_framework_lab_gaia.gaia import GAIA

validation = tmp_path / "2023" / "validation"
validation.mkdir(parents=True)
rows = [{"task_id": "v1", "Question": "q", "Final answer": "a", "Level": 1, "file_name": ""}]

import pyarrow as pa
import pyarrow.parquet as pq

(validation / "metadata.jsonl").write_text("{}\n", encoding="utf-8")
# interrupted copy: only shard 0 of 2 landed beside the JSONL file
pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-00000-of-00002.parquet")

runner = GAIA(data_dir=str(tmp_path))
with pytest.raises(RuntimeError, match="HF_TOKEN"):
runner._ensure_data()

# completing the family makes the mixed directory ready again
pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-00001-of-00002.parquet")
assert runner._ensure_data() == tmp_path

def test_jsonl_only_layout_is_still_accepted(self, tmp_path, monkeypatch):
"""A JSONL-only directory keeps working without any parquet files (#9340 review)."""
monkeypatch.delenv("HF_TOKEN", raising=False)
from agent_framework_lab_gaia.gaia import GAIA

validation = tmp_path / "2023" / "validation"
validation.mkdir(parents=True)
(validation / "metadata.jsonl").write_text("{}\n", encoding="utf-8")

runner = GAIA(data_dir=str(tmp_path))
assert runner._ensure_data() == tmp_path

def _write_shard(self, validation: Path, index: int, total: int) -> None:
import pyarrow as pa
import pyarrow.parquet as pq

rows = [{"task_id": "v1", "Question": "q", "Final answer": "a", "Level": 1, "file_name": ""}]
pq.write_table(pa.Table.from_pylist(rows), validation / f"metadata-{index:05d}-of-{total:05d}.parquet")

def _write_named_shard(self, validation: Path, name: str) -> None:
import pyarrow as pa
import pyarrow.parquet as pq

rows = [{"task_id": "v1", "Question": "q", "Final answer": "a", "Level": 1, "file_name": ""}]
pq.write_table(pa.Table.from_pylist(rows), validation / name)

def test_sharded_family_with_a_dashed_prefix_is_still_counted(self, tmp_path, monkeypatch):
"""A family whose prefix carries extra name parts must not slip past as unsharded.

The loader globs every metadata*.parquet file, so
metadata-2023-validation-00000-of-00002.parquet belongs to a sharded
family even though its prefix has more than one dash separated part.
"""
pytest.importorskip("pyarrow")
monkeypatch.delenv("HF_TOKEN", raising=False)
from agent_framework_lab_gaia.gaia import GAIA

validation = tmp_path / "2023" / "validation"
validation.mkdir(parents=True)
self._write_named_shard(validation, "metadata-2023-validation-00000-of-00002.parquet")

runner = GAIA(data_dir=str(tmp_path))
with pytest.raises(RuntimeError, match="HF_TOKEN"):
runner._ensure_data()

self._write_named_shard(validation, "metadata-2023-validation-00001-of-00002.parquet")
assert runner._ensure_data() == tmp_path

def test_corrupt_shard_disqualifies_a_complete_looking_family(self, tmp_path, monkeypatch):
"""A shard that cannot be read must force the download path.

The loader skips a shard whose read fails and would run the benchmark
over the remaining shards, so a complete looking name is not enough.
"""
pytest.importorskip("pyarrow")
monkeypatch.delenv("HF_TOKEN", raising=False)
from agent_framework_lab_gaia.gaia import GAIA

validation = tmp_path / "2023" / "validation"
validation.mkdir(parents=True)
self._write_shard(validation, 0, 2)
(validation / "metadata-00001-of-00002.parquet").write_bytes(b"this is not a parquet file")

runner = GAIA(data_dir=str(tmp_path))
with pytest.raises(RuntimeError, match="HF_TOKEN"):
runner._ensure_data()

def test_sharded_family_missing_shard_zero_forces_download(self, tmp_path, monkeypatch):
"""A family holding every index except zero must not count as complete."""
pytest.importorskip("pyarrow")
monkeypatch.delenv("HF_TOKEN", raising=False)
from agent_framework_lab_gaia.gaia import GAIA

validation = tmp_path / "2023" / "validation"
validation.mkdir(parents=True)
self._write_shard(validation, 1, 2)
self._write_shard(validation, 2, 2)

runner = GAIA(data_dir=str(tmp_path))
with pytest.raises(RuntimeError, match="HF_TOKEN"):
runner._ensure_data()

def test_sharded_family_with_mixed_totals_forces_download(self, tmp_path, monkeypatch):
"""Files in one family declaring different totals describe an incoherent copy."""
pytest.importorskip("pyarrow")
monkeypatch.delenv("HF_TOKEN", raising=False)
from agent_framework_lab_gaia.gaia import GAIA

validation = tmp_path / "2023" / "validation"
validation.mkdir(parents=True)
# Three distinct indexes against a largest declared total of three, so a
# count against the max total passes while the family disagrees on N.
self._write_shard(validation, 0, 3)
self._write_shard(validation, 1, 3)
self._write_shard(validation, 2, 3)
self._write_shard(validation, 2, 2)

runner = GAIA(data_dir=str(tmp_path))
with pytest.raises(RuntimeError, match="HF_TOKEN"):
runner._ensure_data()

def test_sharded_family_with_an_out_of_range_index_forces_download(self, tmp_path, monkeypatch):
"""An index at or beyond the declared total must not stand in for a missing shard."""
pytest.importorskip("pyarrow")
monkeypatch.delenv("HF_TOKEN", raising=False)
from agent_framework_lab_gaia.gaia import GAIA

validation = tmp_path / "2023" / "validation"
validation.mkdir(parents=True)
self._write_shard(validation, 0, 2)
self._write_shard(validation, 2, 2)

runner = GAIA(data_dir=str(tmp_path))
with pytest.raises(RuntimeError, match="HF_TOKEN"):
runner._ensure_data()

def test_unsharded_file_does_not_rescue_an_incomplete_family(self, tmp_path, monkeypatch):
"""An unsharded file must not mask a partial sharded family (#9337 review)."""
pytest.importorskip("pyarrow")
monkeypatch.delenv("HF_TOKEN", raising=False)
from agent_framework_lab_gaia.gaia import GAIA

validation = tmp_path / "2023" / "validation"
validation.mkdir(parents=True)
rows = [{"task_id": "v1", "Question": "q", "Final answer": "a", "Level": 1, "file_name": ""}]

import pyarrow as pa
import pyarrow.parquet as pq

pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-00000-of-00002.parquet")
pq.write_table(pa.Table.from_pylist(rows), validation / "metadata-extra.parquet")

runner = GAIA(data_dir=str(tmp_path))
with pytest.raises(RuntimeError, match="HF_TOKEN"):
runner._ensure_data()
Loading