From 0601f55c91e0ff341b1d9a03e3c0b2279f130033 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Wed, 15 Jul 2026 14:56:31 +0200 Subject: [PATCH 1/2] feat: versioned DatasetStorage for reading/writing datasets across protocol versions Mirror the syft-job migration/storage architecture in syft-datasets: - DatasetStorage: single choke point for all dataset filesystem IO. Reads select a ProtocolCodec by on-disk protocol version and upgrade to the latest object version in memory; writes downgrade to the target protocol and the codec persists it. Nests a MigrationRegistry, peer_schemas, and codecs. - On-disk protocol version: new datasets can live under a v path segment (public/syft_datasets/v//). Protocol 0 = the last release (syft-client 0.1.117 / syft-dataset 0.1.20): flat layout, no v, no canonical_name/version fields; read + written back byte-compatibly. - Versioned models: Dataset / PrivateDatasetConfig are now MigratableObjects under models//v1.py. Object schemas are unchanged since 0.1.117, so only v1 exists; the protocol bump is purely path-layout + identity fields. - DatasetRef: path-derived identity (owner, name, protocol_version). - Broadcast write: a public dataset is one copy read by the whole audience, so the write protocol(s) come from the audience's versions (target_protocol_versions_for_peers), defaulting to the widest-compatible (oldest) protocol when no peers are known. peer_schemas is injectable but not yet populated by syft-client, so today datasets are written in protocol 0 and the v1 layout ships dormant. - SyftDatasetManager routes all reads/writes through DatasetStorage and supports multi-version write (one copy per protocol version in the audience); get_all dedupes copies across layouts preferring the newest protocol. - migrations/history + release scripts (export_release_artifact.py, generate_release_fixture.py); protocol-0/0.1.20 artifacts and fixtures derived as if that release had emitted them. - Tests mirror syft-job/tests/migrations (unit + p2p, incl. byte-exact cross-release round-trip and a multi-version-write test); CI + justfile wired. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/unit-tests.yml | 19 ++ .pre-commit-config.yaml | 2 +- Justfile | 4 + packages/syft-datasets/pyproject.toml | 3 +- .../scripts/export_release_artifact.py | 41 +++ .../scripts/generate_release_fixture.py | 126 ++++++++ .../src/syft_datasets/__init__.py | 26 +- .../syft-datasets/src/syft_datasets/config.py | 126 +++++++- .../src/syft_datasets/dataset.py | 252 +-------------- .../src/syft_datasets/dataset_manager.py | 296 +++++++++--------- .../src/syft_datasets/dataset_ref.py | 23 ++ .../src/syft_datasets/dataset_storage.py | 216 +++++++++++++ .../src/syft_datasets/migrations/__init__.py | 5 + .../src/syft_datasets/migrations/history.py | 32 ++ .../syft-dataset-0.1.20.json | 149 +++++++++ .../syft-dataset-0.1.21.json | 149 +++++++++ .../history/protocols/protocol-0.json | 144 +++++++++ .../history/protocols/protocol-1.json | 144 +++++++++ .../src/syft_datasets/migrations/registry.py | 22 ++ .../src/syft_datasets/models/__init__.py | 11 + .../syft_datasets/models/dataset/__init__.py | 9 + .../src/syft_datasets/models/dataset/v1.py | 214 +++++++++++++ .../models/private_dataset_config/__init__.py | 9 + .../models/private_dataset_config/v1.py | 24 ++ .../syft_datasets/protocolcodecs/__init__.py | 16 + .../src/syft_datasets/protocolcodecs/base.py | 53 ++++ .../src/syft_datasets/protocolcodecs/v0.py | 52 +++ .../src/syft_datasets/protocolcodecs/v1.py | 47 +++ .../src/syft_datasets/version.py | 10 + .../tests/migrations/__init__.py | 0 .../tests/migrations/p2p/__init__.py | 0 .../private/syft_datasets/demo/private.csv | 3 + .../syft_datasets/demo/private_metadata.yaml | 2 + .../private/syft_datasets/demo/syft.pub.yaml | 8 + .../public/syft_datasets/demo/dataset.yaml | 13 + .../public/syft_datasets/demo/mock.csv | 3 + .../public/syft_datasets/demo/readme.md | 1 + .../public/syft_datasets/demo/syft.pub.yaml | 8 + .../private/syft_datasets/v1/demo/private.csv | 3 + .../v1/demo/private_metadata.yaml | 4 + .../syft_datasets/v1/demo/syft.pub.yaml | 8 + .../public/syft_datasets/v1/demo/dataset.yaml | 15 + .../public/syft_datasets/v1/demo/mock.csv | 3 + .../public/syft_datasets/v1/demo/readme.md | 1 + .../syft_datasets/v1/demo/syft.pub.yaml | 8 + .../migrations/p2p/test_current_protocol.py | 109 +++++++ .../p2p/test_older_protocol_compatibility.py | 122 ++++++++ .../migrations/p2p/test_protocol_codecs.py | 143 +++++++++ .../p2p/test_protocol_sanity_checks.py | 63 ++++ .../tests/migrations/unit/__init__.py | 0 .../migrations/unit/fixtures/Dataset/v1.yaml | 15 + .../fixtures/PrivateDatasetConfig/v1.yaml | 4 + .../tests/migrations/unit/mocks.py | 35 +++ .../migrations/unit/test_history_artifacts.py | 100 ++++++ .../tests/migrations/unit/test_migrations.py | 72 +++++ .../migrations/unit/test_mock_migrations.py | 107 +++++++ .../unit/test_objects_registered.py | 58 ++++ .../migrations/unit/test_serialization.py | 40 +++ .../migrations/unit/test_upgrade_paths.py | 28 ++ uv.lock | 4 +- 60 files changed, 2789 insertions(+), 415 deletions(-) create mode 100644 packages/syft-datasets/scripts/export_release_artifact.py create mode 100644 packages/syft-datasets/scripts/generate_release_fixture.py create mode 100644 packages/syft-datasets/src/syft_datasets/dataset_ref.py create mode 100644 packages/syft-datasets/src/syft_datasets/dataset_storage.py create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/__init__.py create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/history.py create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.20.json create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.21.json create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-0.json create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-1.json create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/registry.py create mode 100644 packages/syft-datasets/src/syft_datasets/models/__init__.py create mode 100644 packages/syft-datasets/src/syft_datasets/models/dataset/__init__.py create mode 100644 packages/syft-datasets/src/syft_datasets/models/dataset/v1.py create mode 100644 packages/syft-datasets/src/syft_datasets/models/private_dataset_config/__init__.py create mode 100644 packages/syft-datasets/src/syft_datasets/models/private_dataset_config/v1.py create mode 100644 packages/syft-datasets/src/syft_datasets/protocolcodecs/__init__.py create mode 100644 packages/syft-datasets/src/syft_datasets/protocolcodecs/base.py create mode 100644 packages/syft-datasets/src/syft_datasets/protocolcodecs/v0.py create mode 100644 packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py create mode 100644 packages/syft-datasets/src/syft_datasets/version.py create mode 100644 packages/syft-datasets/tests/migrations/__init__.py create mode 100644 packages/syft-datasets/tests/migrations/p2p/__init__.py create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private.csv create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private_metadata.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/syft.pub.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/dataset.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/mock.csv create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/readme.md create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/syft.pub.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private.csv create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private_metadata.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/syft.pub.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/dataset.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/mock.csv create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/readme.md create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/syft.pub.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py create mode 100644 packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py create mode 100644 packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py create mode 100644 packages/syft-datasets/tests/migrations/p2p/test_protocol_sanity_checks.py create mode 100644 packages/syft-datasets/tests/migrations/unit/__init__.py create mode 100644 packages/syft-datasets/tests/migrations/unit/fixtures/Dataset/v1.yaml create mode 100644 packages/syft-datasets/tests/migrations/unit/fixtures/PrivateDatasetConfig/v1.yaml create mode 100644 packages/syft-datasets/tests/migrations/unit/mocks.py create mode 100644 packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py create mode 100644 packages/syft-datasets/tests/migrations/unit/test_migrations.py create mode 100644 packages/syft-datasets/tests/migrations/unit/test_mock_migrations.py create mode 100644 packages/syft-datasets/tests/migrations/unit/test_objects_registered.py create mode 100644 packages/syft-datasets/tests/migrations/unit/test_serialization.py create mode 100644 packages/syft-datasets/tests/migrations/unit/test_upgrade_paths.py diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 2f0a61d816a..bce442931be 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -111,6 +111,25 @@ jobs: - name: Run job tests run: uv run pytest packages/syft-job/tests/ -v + dataset-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install the project + run: uv sync --all-extras + + - name: Run dataset tests + run: uv run pytest packages/syft-datasets/tests/ -v + enclave-tests: runs-on: ubuntu-latest steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6522797eb1b..5519bc46191 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,4 +47,4 @@ repos: rev: 'v3.0.0-alpha.9-for-vscode' hooks: - id: prettier - exclude: (\.min\.(js|css)$|^ci/|^test_outputs/) + exclude: (\.min\.(js|css)$|^ci/|^test_outputs/|^packages/syft-datasets/tests/migrations/(p2p|unit)/fixtures/) diff --git a/Justfile b/Justfile index 132ada82358..74fcb344912 100644 --- a/Justfile +++ b/Justfile @@ -30,6 +30,10 @@ test-unit-job: #!/bin/bash uv run pytest -v ./packages/syft-job/tests +test-unit-datasets: + #!/bin/bash + uv run pytest -n auto ./packages/syft-datasets/tests + test-unit-migration: #!/bin/bash uv run pytest -n auto ./packages/syft-migration/tests diff --git a/packages/syft-datasets/pyproject.toml b/packages/syft-datasets/pyproject.toml index 03af882f014..8ed83693736 100644 --- a/packages/syft-datasets/pyproject.toml +++ b/packages/syft-datasets/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "syft-dataset" -version = "0.1.20" +version = "0.1.21" description = "Syft Datasets" authors = [{ name = "OpenMined", email = "info@openmined.org" }] license = { text = "Apache-2.0" } @@ -10,6 +10,7 @@ dependencies = [ "pyyaml>=6.0.3", "syft-notebook-ui", "syft-perms==0.1.14", + "syft-migration", ] diff --git a/packages/syft-datasets/scripts/export_release_artifact.py b/packages/syft-datasets/scripts/export_release_artifact.py new file mode 100644 index 00000000000..f6a97db3bbd --- /dev/null +++ b/packages/syft-datasets/scripts/export_release_artifact.py @@ -0,0 +1,41 @@ +"""Export the release artifacts for the current syft-dataset version. + +Run on EVERY release (uv run python scripts/export_release_artifact.py): +always writes the package release info; additionally writes the protocol +artifact when this release introduces a new protocol version. +""" + +import sys + +from syft_datasets.migrations.history import PACKAGE_ARTIFACTS_DIR, PROTOCOLS_DIR +from syft_datasets.migrations.registry import DATASET_PROTOCOL_VERSION, dataset_registry +from syft_datasets.version import __version__ + + +def main() -> None: + # Import the models so every versioned object is registered. + import syft_datasets # noqa: F401 + + if dataset_registry.protocol_changed_without_bump(): + sys.exit( + "The dataset protocol changed compared to the released " + f"protocol-{DATASET_PROTOCOL_VERSION}.json; bump " + "DATASET_PROTOCOL_VERSION in syft_datasets/migrations/registry.py " + "before releasing." + ) + + PACKAGE_ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True) + PROTOCOLS_DIR.mkdir(parents=True, exist_ok=True) + + info_path = PACKAGE_ARTIFACTS_DIR / f"syft-dataset-{__version__}.json" + dataset_registry.compute_released_package_protocol_info().save(info_path) + print(f"Wrote {info_path}") + + protocol_path = PROTOCOLS_DIR / f"protocol-{DATASET_PROTOCOL_VERSION}.json" + if not protocol_path.exists(): + dataset_registry.compute_released_protocol().save(protocol_path) + print(f"Wrote {protocol_path} (new protocol version)") + + +if __name__ == "__main__": + main() diff --git a/packages/syft-datasets/scripts/generate_release_fixture.py b/packages/syft-datasets/scripts/generate_release_fixture.py new file mode 100644 index 00000000000..594fda59c01 --- /dev/null +++ b/packages/syft-datasets/scripts/generate_release_fixture.py @@ -0,0 +1,126 @@ +"""Generate a p2p backward-compatibility fixture for a syft-dataset release. + +Run on EVERY release, after bumping the version: + + uv run python scripts/generate_release_fixture.py + +Writes a SyftBox tree exactly as this release serializes datasets to disk, into + + tests/migrations/p2p/fixtures/syft_dataset--protocol

_syftbox/ + +Future releases loop over these fixtures (test_older_protocol_compatibility.py) +to prove they can still read and round-trip older on-disk data. + +Protocol 0 / release 0.1.20 predates this script; its fixture +(syft_dataset-0.1.20-protocol0_syftbox) is generated by passing +``--protocol 0 --version 0.1.20`` and is treated as hand-authored, like +protocol-0.json. +""" + +import argparse +import shutil +import tempfile +from pathlib import Path + +from syft_datasets.dataset_manager import SyftDatasetManager +from syft_datasets.migrations import dataset_registry +from syft_datasets.migrations.registry import DATASET_PROTOCOL_VERSION +from syft_datasets.version import __version__ + +DO_EMAIL = "do@test.org" +DS_EMAIL = "ds@test.org" + +FIXTURES_DIR = ( + Path(__file__).resolve().parents[1] / "tests" / "migrations" / "p2p" / "fixtures" +) + + +def _seed_syftbox(syftbox: Path, src_dir: Path, protocol_version: str) -> None: + """One dataset written in the given protocol layout (mock + private + readme).""" + mgr = SyftDatasetManager(syftbox_folder_path=syftbox, email=DO_EMAIL) + # Force the target protocol by advertising a peer at that version. + mgr.storage.peer_schemas = { + DS_EMAIL: dataset_registry.schema_for_protocol_version(protocol_version) + } + + mock_path = src_dir / "mock.csv" + mock_path.write_text("id,value\n1,10\n2,20\n") + private_path = src_dir / "private.csv" + private_path.write_text("id,secret\n1,alpha\n2,beta\n") + readme_path = src_dir / "readme.md" + readme_path.write_text("# demo dataset\n") + + mgr.create( + name="demo", + mock_path=mock_path, + private_path=private_path, + readme_path=readme_path, + summary="demo dataset", + tags=["demo"], + users=[DS_EMAIL], + ) + + +def _normalize(target: Path) -> None: + """Strip machine-specific values so the committed fixture is portable. + + Drop the executable bit (fixtures are data, not scripts) and blank the + private config's absolute ``data_dir`` (not part of the on-disk format + peers see). + """ + import yaml + + for path in target.rglob("*"): + if path.is_file(): + path.chmod(0o644) + + for config in target.rglob("private_metadata.yaml"): + data = yaml.safe_load(config.read_text()) or {} + if data.get("data_dir"): + data["data_dir"] = "" + config.write_text(yaml.safe_dump(data, indent=2, sort_keys=False)) + + +def _build_fixture(target: Path, protocol_version: str) -> None: + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + src_dir = tmp_path / "src" + src_dir.mkdir() + _seed_syftbox(syftbox, src_dir, protocol_version) + shutil.copytree(syftbox, target) + _normalize(target) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--force", action="store_true", help="overwrite an existing fixture" + ) + parser.add_argument( + "--protocol", + default=DATASET_PROTOCOL_VERSION, + help="protocol version to write (default: current)", + ) + parser.add_argument( + "--version", + default=__version__, + help="release version to stamp in the fixture name (default: current)", + ) + args = parser.parse_args() + + target = ( + FIXTURES_DIR / f"syft_dataset-{args.version}-protocol{args.protocol}_syftbox" + ) + if target.exists(): + if not args.force: + raise SystemExit(f"{target} already exists; pass --force to regenerate.") + shutil.rmtree(target) + + _build_fixture(target, args.protocol) + print(f"Wrote {target}") + + +if __name__ == "__main__": + main() diff --git a/packages/syft-datasets/src/syft_datasets/__init__.py b/packages/syft-datasets/src/syft_datasets/__init__.py index 405e3e4fe68..6d2b75b44e0 100644 --- a/packages/syft-datasets/src/syft_datasets/__init__.py +++ b/packages/syft-datasets/src/syft_datasets/__init__.py @@ -1,4 +1,26 @@ +# __version__ comes from the installed distribution metadata (see version.py). +from .version import __version__ + from .config import SyftBoxConfig +from .dataset_ref import DatasetRef +from .dataset_storage import DatasetStorage +from .migrations import dataset_registry +from .migrations.history import register_historic_schemas +from .models import Dataset, DatasetV1, PrivateDatasetConfig, PrivateDatasetConfigV1 + +# Historic schemas list object versions that must already be registered, which +# happens when the models above are imported. +register_historic_schemas() -__version__ = "0.1.7" -__all__ = ["SyftBoxConfig"] +__all__ = [ + "__version__", + "SyftBoxConfig", + "DatasetRef", + "DatasetStorage", + "dataset_registry", + # Models + "Dataset", + "DatasetV1", + "PrivateDatasetConfig", + "PrivateDatasetConfigV1", +] diff --git a/packages/syft-datasets/src/syft_datasets/config.py b/packages/syft-datasets/src/syft_datasets/config.py index 28c3ae4251c..52674ebcce1 100644 --- a/packages/syft-datasets/src/syft_datasets/config.py +++ b/packages/syft-datasets/src/syft_datasets/config.py @@ -1,9 +1,27 @@ +import re +from typing import Optional + from pydantic import BaseModel, Field from pathlib import Path from .url import SyftBoxURL +from .migrations.registry import DATASET_PROTOCOL_VERSION SYFT_DATASETS_FOLDER_NAME = "syft_datasets" METADATA_FILENAME = "dataset.yaml" +PRIVATE_METADATA_FILENAME = "private_metadata.yaml" + +# Datasets under public/syft_datasets/ and private/syft_datasets/ live inside a +# protocol-version segment ("v1", "v2", ...); protocol 0 (<= 0.1.20) had none. +PROTOCOL_DIR_RE = re.compile(r"^v\d+$") + + +def is_protocol_dir_name(name: str) -> bool: + return PROTOCOL_DIR_RE.match(name) is not None + + +def protocol_dir_name(protocol_version: str) -> Optional[str]: + """The path segment for a protocol version; None for protocol 0.""" + return None if protocol_version == "0" else f"v{protocol_version}" class SyftBoxConfig(BaseModel): @@ -20,45 +38,119 @@ def private_dir(self) -> Path: def public_dir(self) -> Path: return self.syftbox_folder / self.email / "public" + def public_datasets_root_for_datasite(self, datasite: str) -> Path: + """The syft_datasets scan root for a datasite (no protocol segment).""" + return self.syftbox_folder / datasite / "public" / SYFT_DATASETS_FOLDER_NAME + + def private_datasets_root(self) -> Path: + """The private syft_datasets scan root (no protocol segment).""" + return self.private_dir / SYFT_DATASETS_FOLDER_NAME + + def private_datasets_root_for_owner(self, owner: str) -> Path: + """The private syft_datasets scan root under an owner's datasite.""" + return self.syftbox_folder / owner / "private" / SYFT_DATASETS_FOLDER_NAME + + def get_private_dataset_dir( + self, + owner: str, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> Path: + return self._with_segment( + self.private_datasets_root_for_owner(owner), protocol_version, dataset_name + ) + def public_datasets_dir_for_datasite(self, datasite: str) -> Path: - dir = self.syftbox_folder / datasite / "public" / SYFT_DATASETS_FOLDER_NAME + # Backwards-compatible accessor; ensures the scan root exists. + dir = self.public_datasets_root_for_datasite(datasite) dir.mkdir(parents=True, exist_ok=True) return dir - def private_dir_for_my_dataset(self, dataset_name: str) -> Path: - return self.private_dir / SYFT_DATASETS_FOLDER_NAME / dataset_name + def _with_segment(self, root: Path, protocol_version: str, name: str) -> Path: + segment = protocol_dir_name(protocol_version) + return (root / segment / name) if segment else (root / name) - def get_my_mock_dataset_dir(self, dataset_name: str) -> Path: - return self.public_datasets_dir_for_datasite(self.email) / dataset_name + def private_dir_for_my_dataset( + self, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> Path: + return self._with_segment( + self.private_datasets_root(), protocol_version, dataset_name + ) - def get_mock_dataset_dir(self, dataset_name: str, datasite: str) -> Path: - return self.public_datasets_dir_for_datasite(datasite) / dataset_name + def get_my_mock_dataset_dir( + self, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> Path: + return self.get_mock_dataset_dir( + dataset_name=dataset_name, + datasite=self.email, + protocol_version=protocol_version, + ) + + def get_mock_dataset_dir( + self, + dataset_name: str, + datasite: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> Path: + return self._with_segment( + self.public_datasets_root_for_datasite(datasite), + protocol_version, + dataset_name, + ) - def get_mock_url_for_my_dataset(self, dataset_name: str) -> SyftBoxURL: + def get_mock_url_for_my_dataset( + self, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> SyftBoxURL: return SyftBoxURL.from_path( - path=self.get_my_mock_dataset_dir(dataset_name), + path=self.get_my_mock_dataset_dir(dataset_name, protocol_version), syftbox_folder=self.syftbox_folder, ) - def get_private_url_for_my_dataset(self, dataset_name: str) -> SyftBoxURL: + def get_private_url_for_my_dataset( + self, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> SyftBoxURL: return SyftBoxURL.from_path( - path=self.private_dir_for_my_dataset(dataset_name), + path=self.private_dir_for_my_dataset(dataset_name, protocol_version), syftbox_folder=self.syftbox_folder, ) def get_readme_url_for_my_dataset( - self, dataset_name: str, readme_name: str + self, + dataset_name: str, + readme_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, ) -> SyftBoxURL: return SyftBoxURL.from_path( - path=self.get_my_mock_dataset_dir(dataset_name) / readme_name, + path=self.get_my_mock_dataset_dir(dataset_name, protocol_version) + / readme_name, syftbox_folder=self.syftbox_folder, ) - def public_metadata_filename_for_my_dataset(self, dataset_name: str) -> str: + def public_metadata_filename_for_my_dataset( + self, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> Path: # TODO: not sure why the absolute is needed here return ( - self.get_my_mock_dataset_dir(dataset_name) / METADATA_FILENAME + self.get_my_mock_dataset_dir(dataset_name, protocol_version) + / METADATA_FILENAME ).absolute() - def private_metadata_filename_for_my_dataset(self, dataset_name: str) -> str: - return self.private_dir_for_my_dataset(dataset_name) / "private_metadata.yaml" + def private_metadata_filename_for_my_dataset( + self, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> Path: + return ( + self.private_dir_for_my_dataset(dataset_name, protocol_version) + / PRIVATE_METADATA_FILENAME + ) diff --git a/packages/syft-datasets/src/syft_datasets/dataset.py b/packages/syft-datasets/src/syft_datasets/dataset.py index ba5eea9e5eb..20a94ffa999 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset.py +++ b/packages/syft-datasets/src/syft_datasets/dataset.py @@ -1,243 +1,15 @@ -from datetime import datetime, timezone -from functools import cached_property -from pathlib import Path -from typing import ClassVar -from syft_permissions.spec.ruleset import PERMISSION_FILE_NAME -from typing_extensions import Self -from uuid import UUID, uuid4 +"""Backwards-compatible re-exports. -import yaml -from pydantic import BaseModel, Field -from syft_notebook_ui.formatter_mixin import ( - ANSIPydanticFormatter, - PydanticFormatter, - PydanticFormatterMixin, -) +The dataset models now live under ``syft_datasets.models`` as versioned +``MigratableObject``s. This module keeps the historical import paths +(``from syft_datasets.dataset import Dataset, PrivateDatasetConfig``) working. +""" -from .types import PathLike, to_path -from .url import SyftBoxURL -from .config import SyftBoxConfig +from .models import Dataset, DatasetV1, PrivateDatasetConfig, PrivateDatasetConfigV1 - -def _utcnow(): - return datetime.now(tz=timezone.utc) - - -class DatasetBase(BaseModel): - __display_formatter__: ClassVar[PydanticFormatter] = ANSIPydanticFormatter() - _syftbox_config: SyftBoxConfig | None = None - - def save(self, filepath: PathLike) -> None: - filepath = to_path(filepath) - if not filepath.suffix == ".yaml": - raise ValueError("Model must be saved as a .yaml file.") - - if not filepath.parent.exists(): - filepath.parent.mkdir(parents=True, exist_ok=True) - - data = self.model_dump(mode="json") - yaml_dump = yaml.safe_dump(data, indent=2, sort_keys=False) - filepath.write_text(yaml_dump) - - @classmethod - def load( - cls, filepath: PathLike, syftbox_config: SyftBoxConfig | None = None - ) -> Self: - filepath = to_path(filepath) - if not filepath.exists(): - raise FileNotFoundError(f"Config file not found: {filepath}") - - data = yaml.safe_load(filepath.read_text()) - res = cls.model_validate(data) - res._syftbox_config = syftbox_config - return res - - def __str__(self) -> str: - return self.__display_formatter__.format_str(self) - - def __repr__(self) -> str: - return self.__display_formatter__.format_repr(self) - - def _repr_html_(self) -> str: - return self.__display_formatter__.format_html(self) - - def _repr_markdown_(self) -> str: - return self.__display_formatter__.format_markdown(self) - - -class PrivateDatasetConfig(DatasetBase, PydanticFormatterMixin): - """Used to store private dataset metadata, outside of the sync folder.""" - - uid: UUID # id for this dataset - data_dir: Path - - -class Dataset(DatasetBase, PydanticFormatterMixin): - __table_extra_fields__ = [ - "name", - "owner", - ] - - uid: UUID = Field(default_factory=uuid4) - created_at: datetime = Field(default_factory=_utcnow) - updated_at: datetime = Field(default_factory=_utcnow) - name: str - summary: str | None = None - tags: list[str] = [] - location: str | None = None - - mock_url: SyftBoxURL - private_url: SyftBoxURL - readme_url: SyftBoxURL | None = None - - # URLs to uploaded files (excluding metadata files) - mock_files_urls: list[SyftBoxURL] = Field(default_factory=list) - - @property - def owner(self) -> str: - return self.mock_url.host - - @property - def syftbox_config(self) -> SyftBoxConfig: - if self._syftbox_config is None: - raise ValueError("SyftBox config is not set.") - return self._syftbox_config - - def _url_to_path(self, url: SyftBoxURL) -> Path: - return url.to_local_path( - syftbox_folder=self.syftbox_config.syftbox_folder, - ) - - @property - def readme_path(self) -> Path | None: - if self.readme_url is None: - return None - return self._url_to_path(self.readme_url) - - def get_readme(self) -> str | None: - """Get the content of the README file.""" - if self.readme_path and self.readme_path.exists(): - return self.readme_path.read_text() - return None - - @property - def mock_dir(self) -> Path: - return self._url_to_path(self.mock_url) - - @property - def private_config_path(self) -> Path: - if self.syftbox_config.email != self.owner: - raise ValueError( - "Cannot access private config for a dataset owned by another user." - ) - return self._private_metadata_dir / "private_metadata.yaml" - - @cached_property - def private_config(self) -> PrivateDatasetConfig: - config_path = self.private_config_path - if not config_path.exists(): - raise FileNotFoundError( - f"Private dataset config not found at {config_path}" - ) - - return PrivateDatasetConfig.load( - filepath=config_path, syftbox_config=self._syftbox_config - ) - - @property - def private_dir(self) -> Path: - if self._syftbox_config is None: - raise ValueError("SyftBox config is not set.") - private_dir = ( - self.syftbox_config.syftbox_folder - / self.owner - / "private" - / "syft_datasets" - / self.name - ) - return private_dir - - @property - def _private_metadata_dir(self) -> Path: - if self.syftbox_config.email != self.owner: - raise ValueError( - "Cannot access private data for a dataset owned by another user." - ) - return self.private_dir - - @property - def mock_files(self) -> list[Path]: - """ - Get absolute paths to all mock files uploaded during dataset.create. - Excludes dataset.yaml and readme.md files. - """ - return [self._url_to_path(url) for url in self.mock_files_urls] - - @property - def private_files(self) -> list[Path]: - """ - Get absolute paths to all private files. - - For owners: returns paths from dataset.create (private_files_paths). - For non-owners (e.g. enclave): returns files from shared_private_dir. - """ - return [ - f - for f in self.private_dir.iterdir() - if f.is_file() - and f.name not in (PERMISSION_FILE_NAME, "private_metadata.yaml") - ] - - @property - def files(self) -> list[Path]: - """ - Get absolute paths to all files (both mock and private) uploaded during dataset.create. - """ - return self.mock_files + self.private_files - - def _generate_description_html(self) -> str: - from syft_notebook_ui.pydantic_html_repr import create_html_repr - - fields_to_include = ["name", "created_at", "summary", "tags", "location"] - - paths_to_include = [] - try: - paths_to_include.append("mock_dir") - except Exception: - fields_to_include.append("mock_url") - - try: - private_dir = self.private_dir - if private_dir.is_dir(): - paths_to_include.append("private_dir") - except Exception: - pass - - try: - readme_path = self.readme_path - if readme_path and readme_path.exists(): - paths_to_include.append("readme_path") - except Exception: - fields_to_include.append("readme_url") - - description = create_html_repr( - obj=self, - fields=fields_to_include, - display_paths=paths_to_include, - ) - - hint = ( - '

' - "💡 Use .mock_files to access mock data" - "
" - ) - return description + hint - - def describe(self) -> None: - from IPython.display import HTML, display - - description = self._generate_description_html() - display(HTML(description)) - - def _repr_html_(self) -> str: - return self._generate_description_html() +__all__ = [ + "Dataset", + "DatasetV1", + "PrivateDatasetConfig", + "PrivateDatasetConfigV1", +] diff --git a/packages/syft-datasets/src/syft_datasets/dataset_manager.py b/packages/syft-datasets/src/syft_datasets/dataset_manager.py index 13cb012dde8..2060eb6fde4 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset_manager.py +++ b/packages/syft-datasets/src/syft_datasets/dataset_manager.py @@ -11,12 +11,13 @@ from typing_extensions import Literal from syft_datasets.dataset import Dataset, PrivateDatasetConfig +from syft_datasets.dataset_ref import DatasetRef +from syft_datasets.dataset_storage import DatasetStorage from syft_datasets.file_utils import copy_dir_contents, copy_paths, is_empty_dir -from .url import SyftBoxURL -from .config import SyftBoxConfig +from .config import METADATA_FILENAME, PRIVATE_METADATA_FILENAME, SyftBoxConfig from .permissions import set_mock_dataset_permissions, set_private_dataset_permissions -from .config import METADATA_FILENAME +from .url import SyftBoxURL DATASET_COLLECTION_PREFIX = "syft_datasetcollection" PRIVATE_DATASET_COLLECTION_PREFIX = "syft_privatecollection" @@ -28,6 +29,10 @@ def __init__(self, syftbox_folder_path: PathLike, email: str): self.syftbox_config = SyftBoxConfig( syftbox_folder=to_path(syftbox_folder_path), email=email ) + # peer_schemas (peer email -> dataset ProtocolSchema) will be filled in by + # syft-client later; until then every peer resolves to the widest- + # compatible protocol, so datasets are written in that layout. + self.storage = DatasetStorage(config=self.syftbox_config) @classmethod def from_config(cls, config: SyftBoxConfig) -> Self: @@ -39,11 +44,10 @@ def _validate_dataset_name(self, dataset_name: str) -> None: raise ValueError( f"Invalid dataset name '{dataset_name}'. Only alphanumeric characters, underscores, and hyphens are allowed." ) + # Reject names that collide with a protocol-version directory (v1, v2, ...). + self.storage.validate_dataset_name(dataset_name) - def _prepare_mock_data(self, dataset_name: str, src_path: Path) -> list[Path]: - target_mock_dir = self.syftbox_config.get_my_mock_dataset_dir( - dataset_name=dataset_name - ) + def _prepare_mock_data(self, target_mock_dir: Path, src_path: Path) -> list[Path]: # Validate src data if not src_path.exists(): raise FileNotFoundError(f"Could not find mock data at {src_path}") @@ -80,14 +84,7 @@ def _prepare_mock_data(self, dataset_name: str, src_path: Path) -> list[Path]: return copied_files - def _prepare_private_data( - self, - dataset_name: str, - src_path: Path, - ) -> list[Path]: - private_dir = self.syftbox_config.private_dir_for_my_dataset( - dataset_name=dataset_name - ) + def _prepare_private_data(self, private_dir: Path, src_path: Path) -> list[Path]: private_dir.mkdir(parents=True, exist_ok=True) copied_files = [] @@ -111,36 +108,9 @@ def _prepare_private_data( return copied_files - def _prepare_private_config( - self, - dataset_name: str, - dataset_uid: UUID, - ) -> None: - """ - The private dataset config is used to store private metadata separately from the public dataset metadata. - """ - private_metadata_path: Path = ( - self.syftbox_config.private_metadata_filename_for_my_dataset( - dataset_name=dataset_name - ) - ) - if private_metadata_path.exists(): - raise FileExistsError( - f"Private metadata file {private_metadata_path} already exists." - ) - - private_config = PrivateDatasetConfig( - uid=dataset_uid, - data_dir=private_metadata_path.parent, - ) - - private_metadata_path.parent.mkdir(parents=True, exist_ok=True) - private_config.save(filepath=private_metadata_path) - - def _prepare_readme(self, dataset_name: str, src_file: Path | None) -> list[Path]: - target_mock_dir = self.syftbox_config.get_my_mock_dataset_dir( - dataset_name=dataset_name - ) + def _prepare_readme( + self, target_mock_dir: Path, src_file: Path | None + ) -> list[Path]: copied_files = [] if src_file is not None: if not src_file.is_file(): @@ -166,86 +136,106 @@ def create( users: list[str] | str | None = None, # copy_private_data: bool = True, # TODO ) -> Dataset: - """_summary_ + """Create a dataset, writing it in each protocol version its audience can read. Args: - name (str): Unique of the dataset to create. - mock_path (PathLike): Path to the existing mock data. This can be a file or a directory. - private_path (PathLike): Path to the existing private data. This can be a file or a directory. + name (str): Unique name of the dataset to create. + mock_path (PathLike): Path to the existing mock data (file or directory). + private_path (PathLike): Path to the existing private data (file or directory). summary (str | None, optional): Short summary of the dataset. Defaults to None. - readme_path (Path | None, optional): Markdown README in the public dataset. Defaults to None. - location (str | None, optional): Location identifier for the dataset, e.g. 'high-side-1234'. - Only required for datasets that are hosted on a remote location and require manual syncing. - Defaults to None. - tags (list[str] | None, optional): Optional tags for the dataset. Defaults to None. - users (list[str] | str | None, optional): Users to share dataset with. Can be list of emails, SHARE_WITH_ANY, or None (default, share with no one). + readme_path (Path | None, optional): Markdown README in the public dataset. + location (str | None, optional): Location identifier for the dataset. + tags (list[str] | None, optional): Optional tags for the dataset. + users (list[str] | str | None, optional): Users to share dataset with. Can be + a list of emails, SHARE_WITH_ANY, or None (default, share with no one). Returns: - Dataset: The created Dataset object. + Dataset: The created Dataset object (the newest protocol version written). """ - mock_path: Path = to_path(mock_path) - private_path: Path = to_path(private_path) - readme_path: Path | None = to_path(readme_path) if readme_path else None + # Reject names that collide with a protocol-version directory (v1, v2, ...). + # (The stricter _validate_dataset_name regex is intentionally not enforced + # here to preserve historical behavior, e.g. names with spaces.) + self.storage.validate_dataset_name(name) + mock_path = to_path(mock_path) + private_path = to_path(private_path) + readme_path = to_path(readme_path) if readme_path else None tags = tags or [] - mock_url = self.syftbox_config.get_mock_url_for_my_dataset(dataset_name=name) + target_versions = self._target_protocol_versions(users) + dataset_uid = uuid4() + + created: dict[str, Dataset] = {} + for protocol_version in sorted(target_versions, key=int): + created[protocol_version] = self._create_one_version( + name=name, + protocol_version=protocol_version, + dataset_uid=dataset_uid, + mock_path=mock_path, + private_path=private_path, + readme_path=readme_path, + summary=summary, + location=location, + tags=tags, + users=users, + ) + + # Return the newest protocol version written (richest layout). + newest = max(created, key=int) + return created[newest] + + def _target_protocol_versions(self, users: list[str] | str | None) -> set[str]: + """The protocol versions to write so the audience can read the dataset.""" + peer_emails = None if users in (None, SHARE_WITH_ANY) else list(users) + return self.storage.target_protocol_versions_for_peers(peer_emails) + + def _create_one_version( + self, + name: str, + protocol_version: str, + dataset_uid: UUID, + mock_path: Path, + private_path: Path, + readme_path: Path | None, + summary: str | None, + location: str | None, + tags: list[str], + users: list[str] | str | None, + ) -> Dataset: + ref = self.storage.new_dataset_ref(name, protocol_version) + + mock_url = self.syftbox_config.get_mock_url_for_my_dataset( + name, protocol_version + ) readme_url = ( self.syftbox_config.get_readme_url_for_my_dataset( - dataset_name=name, readme_name=readme_path.name + name, readme_path.name, protocol_version ) if readme_path else None ) - - # Generate private_url for the dataset Private URLs use a simple path format private_url = self.syftbox_config.get_private_url_for_my_dataset( - dataset_name=name - ) - - # Prepare mock data and collect file paths - all_mock_file_paths = self._prepare_mock_data( - dataset_name=name, - src_path=mock_path, + name, protocol_version ) - # Mock files exclude dataset.yaml and readme.md - # Convert absolute paths to SyftBoxURLs - - # Prepare readme and collect file paths - readme_files = self._prepare_readme( - dataset_name=name, - src_file=readme_path, - ) - - public_metadata_path = ( - self.syftbox_config.public_metadata_filename_for_my_dataset( - dataset_name=name - ) - ) + target_mock_dir = self.storage.public_dataset_dir(ref) + all_mock_file_paths = self._prepare_mock_data(target_mock_dir, mock_path) + readme_files = self._prepare_readme(target_mock_dir, readme_path) + public_metadata_path = target_mock_dir / METADATA_FILENAME mock_file_paths = [ f for f in all_mock_file_paths if f != public_metadata_path and f not in readme_files ] - mock_files_urls = [ SyftBoxURL.from_path( - path=file_path, - syftbox_folder=self.syftbox_config.syftbox_folder, + path=file_path, syftbox_folder=self.syftbox_config.syftbox_folder ) for file_path in mock_file_paths ] - self._prepare_private_data( - dataset_name=name, - src_path=private_path, - ) - - # TODO enable adding private data without copying to SyftBox - # e.g. private_data_dir = dataset._private_metadata_dir if copy_private_data else private_path - dataset_uid = uuid4() - self._prepare_private_config(dataset_uid=dataset_uid, dataset_name=name) + private_dir = self.storage.private_dataset_dir(ref) + self._prepare_private_data(private_dir, private_path) dataset = Dataset( uid=dataset_uid, @@ -257,14 +247,14 @@ def create( location=location, tags=tags, mock_files_urls=mock_files_urls, - _syftbox_config=self.syftbox_config, ) - # needs to set since its a private attr dataset._syftbox_config = self.syftbox_config + dataset._protocol_version = protocol_version - # Save dataset metadata - dataset.save(filepath=public_metadata_path) - # Set permissions on mock and private directories + self.storage.write_dataset(ref, dataset) + self.storage.write_private_config( + ref, PrivateDatasetConfig(uid=dataset_uid, data_dir=private_dir) + ) self._set_new_dataset_permissions(dataset=dataset, users=users) return dataset @@ -289,24 +279,11 @@ def _set_new_dataset_permissions( dataset.private_dir, ) - def _load_dataset_from_dir(self, dataset_dir: Path) -> Dataset: - metadata_path = dataset_dir / METADATA_FILENAME - if not metadata_path.exists(): - raise FileNotFoundError(f"Dataset metadata not found at {metadata_path}") - - return Dataset.load( - filepath=metadata_path, - syftbox_config=self.syftbox_config, - ) - def get(self, name: str, datasite: str | None = None) -> Dataset: datasite = datasite or self.syftbox_config.email - mock_dir = self.syftbox_config.get_mock_dataset_dir( - dataset_name=name, - datasite=datasite, - ) - - if not mock_dir.exists(): + try: + ref = self.storage.find_dataset_ref(datasite, name) + except FileNotFoundError: available = self.get_all() if available: listing = "\n".join( @@ -323,7 +300,7 @@ def get(self, name: str, datasite: str | None = None) -> Dataset: f" Available datasets:\n" f"{listing}" ) - return self._load_dataset_from_dir(mock_dir) + return self.storage.read_dataset(ref) def __getitem__(self, key: str | int) -> Dataset: if isinstance(key, int): @@ -345,6 +322,15 @@ def _repr_html_(self) -> str: return dataset_manager_repr_html(self.get_all()) + def _datasites_to_check(self, datasite: str | None) -> list[str]: + if datasite: + return [datasite] + syftbox_folder = self.syftbox_config.syftbox_folder + # All directories with "@" in the name are peer/owner email directories + return [ + d.name for d in syftbox_folder.iterdir() if d.is_dir() and "@" in d.name + ] + def get_all( self, datasite: str | None = None, @@ -353,30 +339,24 @@ def get_all( order_by: str | None = None, sort_order: Literal["asc", "desc"] = "asc", ) -> list[Dataset]: - all_datasets = [] + # A dataset may exist on disk in more than one protocol layout; keep only + # the newest protocol per (owner, name). + best_ref: dict[tuple[str, str], DatasetRef] = {} + for datasite in self._datasites_to_check(datasite): + for ref in self.storage.iter_dataset_refs(datasite): + key = (ref.owner, ref.name) + current = best_ref.get(key) + if current is None or int(ref.protocol_version) > int( + current.protocol_version + ): + best_ref[key] = ref - if datasite: - datasites_to_check = [datasite] - else: - syftbox_folder = self.syftbox_config.syftbox_folder - # All directories with "@" in the name are peer/owner email directories - datasites_to_check = [ - d.name for d in syftbox_folder.iterdir() if d.is_dir() and "@" in d.name - ] - - for datasite in datasites_to_check: - public_datasets_dir = self.syftbox_config.public_datasets_dir_for_datasite( - datasite - ) - if not public_datasets_dir.exists(): + all_datasets = [] + for ref in best_ref.values(): + try: + all_datasets.append(self.storage.read_dataset(ref)) + except Exception: continue - for dataset_dir in public_datasets_dir.iterdir(): - if dataset_dir.is_dir(): - try: - dataset = self._load_dataset_from_dir(dataset_dir) - all_datasets.append(dataset) - except Exception: - continue if order_by is not None: all_datasets.sort( @@ -447,22 +427,27 @@ def delete( def get_private_dataset_files(self, name: str) -> dict[Path, bytes]: """Get private dataset files as {path_in_datasite: content}. - Returns paths relative to the datasite (e.g. private/syft_datasets/{name}/{file}). - For private_metadata.yaml, clears data_dir before including it. + Returns paths relative to the datasite (e.g. + private/syft_datasets/[v/]{name}/{file}). For private_metadata.yaml, + clears data_dir before including it. """ - dataset = self.get(name=name, datasite=self.syftbox_config.email) - private_dir = dataset.private_dir + datasite = self.syftbox_config.email + ref = self.storage.find_dataset_ref(datasite, name) + private_dir = self.storage.private_dataset_dir(ref) if not private_dir.exists(): raise ValueError(f"Private data directory not found: {private_dir}") + datasite_root = self.syftbox_config.syftbox_folder / datasite + private_rel_root = private_dir.relative_to(datasite_root) + files = {} for f in private_dir.rglob("*"): if not f.is_file(): continue rel = f.relative_to(private_dir) - path_in_datasite = Path(f"private/syft_datasets/{name}") / rel - if f.name == "private_metadata.yaml" and rel == Path(f.name): - files[path_in_datasite] = self._private_config_without_data_dir(f) + path_in_datasite = private_rel_root / rel + if f.name == PRIVATE_METADATA_FILENAME and rel == Path(f.name): + files[path_in_datasite] = self._private_config_without_data_dir(ref) else: files[path_in_datasite] = f.read_bytes() @@ -470,10 +455,15 @@ def get_private_dataset_files(self, name: str) -> dict[Path, bytes]: raise ValueError(f"No private files found for dataset '{name}'") return files - def _private_config_without_data_dir(self, config_path: Path) -> bytes: - """Load private_metadata.yaml and return it with data_dir cleared.""" - config = PrivateDatasetConfig.load(filepath=config_path) + def _private_config_without_data_dir(self, ref: DatasetRef) -> bytes: + """Serialize the dataset's private_metadata.yaml with data_dir cleared. + + Written in the dataset's on-disk protocol format so peers can read it. + """ + config = self.storage.read_private_config(ref) config.data_dir = Path("") - return yaml.safe_dump( - config.model_dump(mode="json"), indent=2, sort_keys=False - ).encode() + data = config.disk_dict() + if ref.protocol_version == "0": + data.pop("canonical_name", None) + data.pop("version", None) + return yaml.safe_dump(data, indent=2, sort_keys=False).encode() diff --git a/packages/syft-datasets/src/syft_datasets/dataset_ref.py b/packages/syft-datasets/src/syft_datasets/dataset_ref.py new file mode 100644 index 00000000000..a1bcca87ee3 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/dataset_ref.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass + + +class DatasetNotFoundError(FileNotFoundError): + """Raised when a dataset's metadata (dataset.yaml) does not exist.""" + + +class PrivateConfigNotFoundError(FileNotFoundError): + """Raised when a dataset's private_metadata.yaml does not exist.""" + + +@dataclass(frozen=True) +class DatasetRef: + """One dataset on disk: who owns it and its protocol layout. + + A dataset is single-owner and broadcast-read (one public copy read by the + whole audience), so there is no per-reader field: identity is + (owner, name, protocol_version). + """ + + owner: str # datasite email of the DO who owns the dataset + name: str + protocol_version: str # "0" (no path segment) or "1"+ (v segment) diff --git a/packages/syft-datasets/src/syft_datasets/dataset_storage.py b/packages/syft-datasets/src/syft_datasets/dataset_storage.py new file mode 100644 index 00000000000..9ee96dbb515 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/dataset_storage.py @@ -0,0 +1,216 @@ +from pathlib import Path +from typing import Iterator, Optional + +from syft_migration import ( + MigratableObject, + MigrationError, + MigrationRegistry, + MigrationService, + ProtocolSchema, +) + +from .config import SyftBoxConfig, is_protocol_dir_name +from .dataset_ref import DatasetNotFoundError, DatasetRef, PrivateConfigNotFoundError +from .migrations.registry import DATASET_PROTOCOL_VERSION, dataset_registry +from .models import Dataset, PrivateDatasetConfig +from .protocolcodecs import CODECS, ProtocolCodec + +__all__ = [ + "DatasetRef", + "DatasetNotFoundError", + "PrivateConfigNotFoundError", + "DatasetStorage", +] + + +class DatasetStorage: + """Dataset filesystem IO and path resolution, delegating disk layout to codecs. + + DatasetStorage owns migration: reads upgrade objects to the latest registered + version in memory; writes downgrade them to what a reading peer understands. + Each on-disk storage format lives behind a ProtocolCodec, selected by the + protocol version (from a dataset's on-disk layout for reads; the negotiated + version for new datasets). The last codec is the current protocol. + + Unlike jobs (each copy targets one peer), a public dataset is a single copy + read by the whole audience, so new datasets are written at the version(s) the + audience can read (see ``target_protocol_versions_for_peers``), defaulting to + the widest-compatible (oldest) protocol when no peers are known. + """ + + def __init__( + self, + config: SyftBoxConfig, + registry: MigrationRegistry = dataset_registry, + peer_schemas: Optional[dict[str, ProtocolSchema]] = None, + ) -> None: + self.config = config + self.registry = registry + self.service = MigrationService(registry=registry) + # peer email -> dataset ProtocolSchema; filled in by syft-client later. + # Peers without an entry cannot be assumed to read the current layout, so + # they resolve to the widest-compatible (oldest) protocol. + self.peer_schemas: dict[str, ProtocolSchema] = peer_schemas or {} + self.codecs = [cls(config) for cls in CODECS] + + @property + def _codec_by_protocol_version(self) -> dict[str, ProtocolCodec]: + # protocol version -> codec; one codec may serve several versions. + return { + protocol_version: codec + for codec in self.codecs + for protocol_version in codec.protocol_versions + } + + def _codec_for(self, protocol_version: str) -> ProtocolCodec: + codec = self._codec_by_protocol_version.get(protocol_version) + if codec is None: + raise MigrationError( + f"No codec for dataset protocol version {protocol_version!r}" + ) + return codec + + @property + def _widest_protocol_version(self) -> str: + """The oldest protocol version any current client can read (widest compat).""" + return min(self._codec_by_protocol_version, key=int) + + # -- peers / protocol ---------------------------------------------------- + def negotiated_protocol_version_for_peer( + self, peer_email: str, raise_on_unknown: bool = True + ) -> str: + """The dataset protocol version to speak with ``peer_email``. + + Negotiated as the minimum of our own protocol version and the peer's, so + both sides use a version they can read. A peer without a known schema + raises by default; with ``raise_on_unknown=False`` it is assumed to run + the current protocol. + """ + schema = self.peer_schemas.get(peer_email) + if schema is not None: + return min(DATASET_PROTOCOL_VERSION, schema.version, key=int) + if raise_on_unknown: + raise MigrationError( + f"No dataset protocol schema known for peer {peer_email!r}" + ) + return DATASET_PROTOCOL_VERSION + + def target_protocol_versions_for_peers( + self, peer_emails: Optional[list[str]] = None + ) -> set[str]: + """The set of protocol versions to write so every peer can read a copy. + + A dataset is written once per distinct version in the audience. A known + peer contributes ``min(ours, theirs)``; an unknown peer (or no audience) + contributes the widest-compatible protocol, since we cannot assume it can + read a newer layout. + """ + if not peer_emails: + return {self._widest_protocol_version} + versions: set[str] = set() + for email in peer_emails: + schema = self.peer_schemas.get(email) + if schema is not None: + versions.add(min(DATASET_PROTOCOL_VERSION, schema.version, key=int)) + else: + versions.add(self._widest_protocol_version) + return versions + + def new_dataset_ref(self, name: str, protocol_version: str) -> DatasetRef: + """A ref for a new dataset owned by the current user.""" + return DatasetRef( + owner=self.config.email, name=name, protocol_version=protocol_version + ) + + # -- naming -------------------------------------------------------------- + @staticmethod + def validate_dataset_name(dataset_name: str) -> None: + if is_protocol_dir_name(dataset_name): + raise ValueError( + f"Dataset name {dataset_name!r} is reserved for protocol version directories" + ) + + # -- path resolution ----------------------------------------------------- + def public_dataset_dir(self, ref: DatasetRef) -> Path: + return self._codec_for(ref.protocol_version).public_dataset_dir(ref) + + def private_dataset_dir(self, ref: DatasetRef) -> Path: + return self._codec_for(ref.protocol_version).private_dataset_dir(ref) + + def metadata_path(self, ref: DatasetRef) -> Path: + return self._codec_for(ref.protocol_version).metadata_path(ref) + + def private_metadata_path(self, ref: DatasetRef) -> Path: + return self._codec_for(ref.protocol_version).private_metadata_path(ref) + + # -- scanning (union over all protocol layouts) -------------------------- + def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: + """Yield a ref per dataset under public/syft_datasets/, across all codecs.""" + for codec in self.codecs: + yield from codec.iter_dataset_refs(datasite_email) + + def find_dataset_ref(self, datasite_email: str, name: str) -> DatasetRef: + """The ref for ``name`` in a datasite; prefers the highest protocol layout.""" + matches = [ + ref for ref in self.iter_dataset_refs(datasite_email) if ref.name == name + ] + if not matches: + raise DatasetNotFoundError(f"Dataset '{name}' not found") + return max(matches, key=lambda r: int(r.protocol_version)) + + # -- model IO ------------------------------------------------------------ + def read_dataset(self, ref: DatasetRef) -> Dataset: + """Load a dataset's dataset.yaml, upgraded to the latest version.""" + codec = self._codec_for(ref.protocol_version) + data = codec.read(codec.metadata_path(ref), "Dataset") + dataset = self._upgrade(data, "Dataset") + # Name comes from the path (the dataset dir), never a spoofable file. + dataset.name = ref.name + dataset._syftbox_config = self.config + dataset._protocol_version = ref.protocol_version + return dataset + + def read_private_config(self, ref: DatasetRef) -> PrivateDatasetConfig: + """Load a dataset's private_metadata.yaml, upgraded to the latest version. + + Raises PrivateConfigNotFoundError if the file does not exist. + """ + codec = self._codec_for(ref.protocol_version) + path = codec.private_metadata_path(ref) + if not path.exists(): + raise PrivateConfigNotFoundError( + f"Private config not found for dataset {ref.name}" + ) + return self._upgrade( + codec.read(path, "PrivateDatasetConfig"), "PrivateDatasetConfig" + ) + + def write_dataset(self, ref: DatasetRef, dataset: Dataset) -> Path: + """Write dataset.yaml in the version/format for this ref's protocol.""" + codec = self._codec_for(ref.protocol_version) + return self._write(codec, codec.metadata_path(ref), dataset, ref) + + def write_private_config( + self, ref: DatasetRef, config: PrivateDatasetConfig + ) -> Path: + """Write private_metadata.yaml in the version/format for this ref's protocol.""" + codec = self._codec_for(ref.protocol_version) + return self._write(codec, codec.private_metadata_path(ref), config, ref) + + # -- internals ----------------------------------------------------------- + def _upgrade(self, data: dict, canonical_name: str) -> MigratableObject: + obj = self.service.load(data) + return self.service.migrate(obj, self.registry.latest_version(canonical_name)) + + def _write( + self, + codec: ProtocolCodec, + path: Path, + obj: MigratableObject, + ref: DatasetRef, + ) -> Path: + """Downgrade ``obj`` to the ref's protocol schema, then let the codec persist it.""" + schema = self.registry.schema_for_protocol_version(ref.protocol_version) + downgraded = self.service.migrate_to_schema(obj, schema) + codec.write(path, downgraded) + return path diff --git a/packages/syft-datasets/src/syft_datasets/migrations/__init__.py b/packages/syft-datasets/src/syft_datasets/migrations/__init__.py new file mode 100644 index 00000000000..22b0db6f7ad --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/__init__.py @@ -0,0 +1,5 @@ +from .registry import dataset_registry + +__all__ = [ + "dataset_registry", +] diff --git a/packages/syft-datasets/src/syft_datasets/migrations/history.py b/packages/syft-datasets/src/syft_datasets/migrations/history.py new file mode 100644 index 00000000000..583f67e9689 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/history.py @@ -0,0 +1,32 @@ +from pathlib import Path + +from syft_migration import ReleasedPackageProtocolInfo, ReleasedProtocol + +from .registry import dataset_registry + +# Release artifacts of past syft-dataset releases: +# package-artifacts/syft-dataset-.json (every release) +# protocols/protocol-.json (only when the protocol changed) +# Generated by export_release_artifact.py; 0.1.20 / protocol 0 predate the +# artifact mechanism, so their files are hardcoded as if that release had +# emitted them. +HISTORY_DIR = Path(__file__).parent / "history" +PACKAGE_ARTIFACTS_DIR = HISTORY_DIR / "package-artifacts" +PROTOCOLS_DIR = HISTORY_DIR / "protocols" + + +def register_historic_schemas() -> None: + """Register the release artifacts of past releases into the dataset registry. + + Must run after the versioned models are imported: with + ``raise_for_unknown_objects`` an artifact listing an object version this + release cannot load fails at import time instead of at migration time. + """ + for path in sorted(PACKAGE_ARTIFACTS_DIR.glob("*.json")): + dataset_registry.register_released_package_protocol_info( + ReleasedPackageProtocolInfo.load(path), raise_for_unknown_objects=True + ) + for path in sorted(PROTOCOLS_DIR.glob("*.json")): + dataset_registry.register_released_protocol( + ReleasedProtocol.load(path), raise_for_unknown_objects=True + ) diff --git a/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.20.json b/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.20.json new file mode 100644 index 00000000000..cd2f31cc7e3 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.20.json @@ -0,0 +1,149 @@ +{ + "package_info": { + "package_name": "syft-dataset", + "version": "0.1.20", + "protocol_version": "0" + }, + "protocol_schema": { + "protocol_name": "syft-dataset", + "version": "0", + "supported_versions": { + "PrivateDatasetConfig": ["1"], + "Dataset": ["1"] + }, + "current_object_schemas": { + "PrivateDatasetConfig": { + "description": "Private dataset metadata, stored as private_metadata.yaml outside the sync folder.", + "properties": { + "canonical_name": { + "default": "PrivateDatasetConfig", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "data_dir": { + "format": "path", + "title": "Data Dir", + "type": "string" + } + }, + "required": ["uid", "data_dir"], + "title": "PrivateDatasetConfigV1", + "type": "object" + }, + "Dataset": { + "description": "Public dataset metadata, stored as dataset.yaml under\nSyftBox//public/syft_datasets/[v/]/.", + "properties": { + "canonical_name": { + "default": "Dataset", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Summary" + }, + "tags": { + "default": [], + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "mock_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Mock Url", + "type": "string" + }, + "private_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Private Url", + "type": "string" + }, + "readme_url": { + "anyOf": [ + { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Readme Url" + }, + "mock_files_urls": { + "items": { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + "title": "Mock Files Urls", + "type": "array" + } + }, + "required": ["name", "mock_url", "private_url"], + "title": "DatasetV1", + "type": "object" + } + } + } +} diff --git a/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.21.json b/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.21.json new file mode 100644 index 00000000000..dbcf520d6a7 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.21.json @@ -0,0 +1,149 @@ +{ + "package_info": { + "package_name": "syft-dataset", + "version": "0.1.21", + "protocol_version": "1" + }, + "protocol_schema": { + "protocol_name": "syft-dataset", + "version": "1", + "supported_versions": { + "PrivateDatasetConfig": ["1"], + "Dataset": ["1"] + }, + "current_object_schemas": { + "PrivateDatasetConfig": { + "description": "Private dataset metadata, stored as private_metadata.yaml outside the sync folder.", + "properties": { + "canonical_name": { + "default": "PrivateDatasetConfig", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "data_dir": { + "format": "path", + "title": "Data Dir", + "type": "string" + } + }, + "required": ["uid", "data_dir"], + "title": "PrivateDatasetConfigV1", + "type": "object" + }, + "Dataset": { + "description": "Public dataset metadata, stored as dataset.yaml under\nSyftBox//public/syft_datasets/[v/]/.", + "properties": { + "canonical_name": { + "default": "Dataset", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Summary" + }, + "tags": { + "default": [], + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "mock_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Mock Url", + "type": "string" + }, + "private_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Private Url", + "type": "string" + }, + "readme_url": { + "anyOf": [ + { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Readme Url" + }, + "mock_files_urls": { + "items": { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + "title": "Mock Files Urls", + "type": "array" + } + }, + "required": ["name", "mock_url", "private_url"], + "title": "DatasetV1", + "type": "object" + } + } + } +} diff --git a/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-0.json b/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-0.json new file mode 100644 index 00000000000..c01c931dfc9 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-0.json @@ -0,0 +1,144 @@ +{ + "protocol_schema": { + "protocol_name": "syft-dataset", + "version": "0", + "supported_versions": { + "PrivateDatasetConfig": ["1"], + "Dataset": ["1"] + }, + "current_object_schemas": { + "PrivateDatasetConfig": { + "description": "Private dataset metadata, stored as private_metadata.yaml outside the sync folder.", + "properties": { + "canonical_name": { + "default": "PrivateDatasetConfig", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "data_dir": { + "format": "path", + "title": "Data Dir", + "type": "string" + } + }, + "required": ["uid", "data_dir"], + "title": "PrivateDatasetConfigV1", + "type": "object" + }, + "Dataset": { + "description": "Public dataset metadata, stored as dataset.yaml under\nSyftBox//public/syft_datasets/[v/]/.", + "properties": { + "canonical_name": { + "default": "Dataset", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Summary" + }, + "tags": { + "default": [], + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "mock_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Mock Url", + "type": "string" + }, + "private_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Private Url", + "type": "string" + }, + "readme_url": { + "anyOf": [ + { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Readme Url" + }, + "mock_files_urls": { + "items": { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + "title": "Mock Files Urls", + "type": "array" + } + }, + "required": ["name", "mock_url", "private_url"], + "title": "DatasetV1", + "type": "object" + } + } + } +} diff --git a/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-1.json b/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-1.json new file mode 100644 index 00000000000..b81ab76a29b --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-1.json @@ -0,0 +1,144 @@ +{ + "protocol_schema": { + "protocol_name": "syft-dataset", + "version": "1", + "supported_versions": { + "PrivateDatasetConfig": ["1"], + "Dataset": ["1"] + }, + "current_object_schemas": { + "PrivateDatasetConfig": { + "description": "Private dataset metadata, stored as private_metadata.yaml outside the sync folder.", + "properties": { + "canonical_name": { + "default": "PrivateDatasetConfig", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "data_dir": { + "format": "path", + "title": "Data Dir", + "type": "string" + } + }, + "required": ["uid", "data_dir"], + "title": "PrivateDatasetConfigV1", + "type": "object" + }, + "Dataset": { + "description": "Public dataset metadata, stored as dataset.yaml under\nSyftBox//public/syft_datasets/[v/]/.", + "properties": { + "canonical_name": { + "default": "Dataset", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Summary" + }, + "tags": { + "default": [], + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "mock_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Mock Url", + "type": "string" + }, + "private_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Private Url", + "type": "string" + }, + "readme_url": { + "anyOf": [ + { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Readme Url" + }, + "mock_files_urls": { + "items": { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + "title": "Mock Files Urls", + "type": "array" + } + }, + "required": ["name", "mock_url", "private_url"], + "title": "DatasetV1", + "type": "object" + } + } + } +} diff --git a/packages/syft-datasets/src/syft_datasets/migrations/registry.py b/packages/syft-datasets/src/syft_datasets/migrations/registry.py new file mode 100644 index 00000000000..661be107d98 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/registry.py @@ -0,0 +1,22 @@ +from syft_migration import MigrationRegistry + +from ..version import PACKAGE_NAME, __version__ + +# Hardcoded, language-agnostic identifier for the syft-dataset protocol; +# intentionally distinct from the package name. +PROTOCOL_NAME = "syft-dataset" + +# Incrementing version of the dataset protocol. Protocol 0 is the last release +# without versioning (syft-client 0.1.117 / syft-dataset 0.1.20, no v path +# segment); protocol >= 1 stores datasets under a v segment after the +# syft_datasets folder (see config.protocol_dir_name). +DATASET_PROTOCOL_VERSION = "1" + +# Package-local registry for all versioned syft-dataset objects. The current +# protocol schema is computed from the objects registered into it. +dataset_registry = MigrationRegistry( + protocol_name=PROTOCOL_NAME, + package_name=PACKAGE_NAME, + package_version=__version__, + protocol_version=DATASET_PROTOCOL_VERSION, +) diff --git a/packages/syft-datasets/src/syft_datasets/models/__init__.py b/packages/syft-datasets/src/syft_datasets/models/__init__.py new file mode 100644 index 00000000000..427185b28e9 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/models/__init__.py @@ -0,0 +1,11 @@ +from .dataset import Dataset, DatasetV1 +from .private_dataset_config import PrivateDatasetConfig, PrivateDatasetConfigV1 + +__all__ = [ + # Current-version aliases + "Dataset", + "PrivateDatasetConfig", + # Versioned objects + "DatasetV1", + "PrivateDatasetConfigV1", +] diff --git a/packages/syft-datasets/src/syft_datasets/models/dataset/__init__.py b/packages/syft-datasets/src/syft_datasets/models/dataset/__init__.py new file mode 100644 index 00000000000..afbb706ab44 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/models/dataset/__init__.py @@ -0,0 +1,9 @@ +from .v1 import DatasetV1 + +# The current version of the dataset object. +Dataset = DatasetV1 + +__all__ = [ + "Dataset", + "DatasetV1", +] diff --git a/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py b/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py new file mode 100644 index 00000000000..2978a35d051 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from functools import cached_property +from pathlib import Path +from typing import ClassVar +from uuid import UUID, uuid4 + +import yaml +from pydantic import Field +from syft_migration import MigratableObject +from syft_notebook_ui.formatter_mixin import ( + ANSIPydanticFormatter, + PydanticFormatter, + PydanticFormatterMixin, +) +from syft_permissions.spec.ruleset import PERMISSION_FILE_NAME + +from ...config import ( + PRIVATE_METADATA_FILENAME, + SYFT_DATASETS_FOLDER_NAME, + SyftBoxConfig, + protocol_dir_name, +) +from ...migrations import dataset_registry +from ...migrations.registry import DATASET_PROTOCOL_VERSION +from ...url import SyftBoxURL +from ..private_dataset_config.v1 import PrivateDatasetConfigV1 + + +def _utcnow() -> datetime: + return datetime.now(tz=timezone.utc) + + +class DatasetV1(MigratableObject, PydanticFormatterMixin, registry=dataset_registry): + """Public dataset metadata, stored as dataset.yaml under + SyftBox//public/syft_datasets/[v/]/.""" + + __display_formatter__: ClassVar[PydanticFormatter] = ANSIPydanticFormatter() + __table_extra_fields__: ClassVar[list[str]] = ["name", "owner"] + + canonical_name: str = "Dataset" + version: str = "1" + + uid: UUID = Field(default_factory=uuid4) + created_at: datetime = Field(default_factory=_utcnow) + updated_at: datetime = Field(default_factory=_utcnow) + name: str + summary: str | None = None + tags: list[str] = [] + location: str | None = None + + mock_url: SyftBoxURL + private_url: SyftBoxURL + readme_url: SyftBoxURL | None = None + + # URLs to uploaded files (excluding metadata files) + mock_files_urls: list[SyftBoxURL] = Field(default_factory=list) + + # Runtime-only: set by DatasetStorage / the manager, never serialized. The + # on-disk protocol layout the dataset lives in (governs the v segment). + _syftbox_config: SyftBoxConfig | None = None + _protocol_version: str = DATASET_PROTOCOL_VERSION + + def disk_dict(self) -> dict: + """The on-disk form of the dataset metadata.""" + return self.model_dump(mode="json") + + @property + def owner(self) -> str: + return self.mock_url.host + + @property + def syftbox_config(self) -> SyftBoxConfig: + if self._syftbox_config is None: + raise ValueError("SyftBox config is not set.") + return self._syftbox_config + + def _url_to_path(self, url: SyftBoxURL) -> Path: + return url.to_local_path(syftbox_folder=self.syftbox_config.syftbox_folder) + + @property + def readme_path(self) -> Path | None: + if self.readme_url is None: + return None + return self._url_to_path(self.readme_url) + + def get_readme(self) -> str | None: + """Get the content of the README file.""" + if self.readme_path and self.readme_path.exists(): + return self.readme_path.read_text() + return None + + @property + def mock_dir(self) -> Path: + return self._url_to_path(self.mock_url) + + @property + def private_config_path(self) -> Path: + if self.syftbox_config.email != self.owner: + raise ValueError( + "Cannot access private config for a dataset owned by another user." + ) + return self._private_metadata_dir / PRIVATE_METADATA_FILENAME + + @cached_property + def private_config(self) -> PrivateDatasetConfigV1: + config_path = self.private_config_path + if not config_path.exists(): + raise FileNotFoundError( + f"Private dataset config not found at {config_path}" + ) + data = yaml.safe_load(config_path.read_text()) or {} + data.setdefault("canonical_name", "PrivateDatasetConfig") + data.setdefault("version", "1") + return PrivateDatasetConfigV1(**data) + + @property + def private_dir(self) -> Path: + """The private data dir for this dataset, under the dataset's protocol layout. + + Derived from the path (owner + name + protocol) rather than the stored + URL so it stays correct across on-disk layouts. + """ + segment = protocol_dir_name(self._protocol_version) + root = ( + self.syftbox_config.syftbox_folder + / self.owner + / "private" + / SYFT_DATASETS_FOLDER_NAME + ) + return (root / segment / self.name) if segment else (root / self.name) + + @property + def _private_metadata_dir(self) -> Path: + if self.syftbox_config.email != self.owner: + raise ValueError( + "Cannot access private data for a dataset owned by another user." + ) + return self.private_dir + + @property + def mock_files(self) -> list[Path]: + """Absolute paths to all mock files uploaded during dataset.create. + + Excludes dataset.yaml and readme.md files. + """ + return [self._url_to_path(url) for url in self.mock_files_urls] + + @property + def private_files(self) -> list[Path]: + """Absolute paths to all private files. + + For owners: returns paths from dataset.create (private_files_paths). + For non-owners (e.g. enclave): returns files from shared_private_dir. + """ + return [ + f + for f in self.private_dir.iterdir() + if f.is_file() + and f.name not in (PERMISSION_FILE_NAME, PRIVATE_METADATA_FILENAME) + ] + + @property + def files(self) -> list[Path]: + """Absolute paths to all files (both mock and private).""" + return self.mock_files + self.private_files + + def _generate_description_html(self) -> str: + from syft_notebook_ui.pydantic_html_repr import create_html_repr + + fields_to_include = ["name", "created_at", "summary", "tags", "location"] + + paths_to_include = [] + try: + paths_to_include.append("mock_dir") + except Exception: + fields_to_include.append("mock_url") + + try: + private_dir = self.private_dir + if private_dir.is_dir(): + paths_to_include.append("private_dir") + except Exception: + pass + + try: + readme_path = self.readme_path + if readme_path and readme_path.exists(): + paths_to_include.append("readme_path") + except Exception: + fields_to_include.append("readme_url") + + description = create_html_repr( + obj=self, + fields=fields_to_include, + display_paths=paths_to_include, + ) + + hint = ( + '
' + "💡 Use .mock_files to access mock data" + "
" + ) + return description + hint + + def describe(self) -> None: + from IPython.display import HTML, display + + description = self._generate_description_html() + display(HTML(description)) + + def _repr_html_(self) -> str: + return self._generate_description_html() diff --git a/packages/syft-datasets/src/syft_datasets/models/private_dataset_config/__init__.py b/packages/syft-datasets/src/syft_datasets/models/private_dataset_config/__init__.py new file mode 100644 index 00000000000..d391d1f9f79 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/models/private_dataset_config/__init__.py @@ -0,0 +1,9 @@ +from .v1 import PrivateDatasetConfigV1 + +# The current version of the private dataset config object. +PrivateDatasetConfig = PrivateDatasetConfigV1 + +__all__ = [ + "PrivateDatasetConfig", + "PrivateDatasetConfigV1", +] diff --git a/packages/syft-datasets/src/syft_datasets/models/private_dataset_config/v1.py b/packages/syft-datasets/src/syft_datasets/models/private_dataset_config/v1.py new file mode 100644 index 00000000000..fbc7f080dfd --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/models/private_dataset_config/v1.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from pathlib import Path +from uuid import UUID + +from syft_migration import MigratableObject +from syft_notebook_ui.formatter_mixin import PydanticFormatterMixin + +from ...migrations import dataset_registry + + +class PrivateDatasetConfigV1( + MigratableObject, PydanticFormatterMixin, registry=dataset_registry +): + """Private dataset metadata, stored as private_metadata.yaml outside the sync folder.""" + + canonical_name: str = "PrivateDatasetConfig" + version: str = "1" + + uid: UUID # id for this dataset + data_dir: Path + + def disk_dict(self) -> dict: + return self.model_dump(mode="json") diff --git a/packages/syft-datasets/src/syft_datasets/protocolcodecs/__init__.py b/packages/syft-datasets/src/syft_datasets/protocolcodecs/__init__.py new file mode 100644 index 00000000000..50c78dccd44 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/protocolcodecs/__init__.py @@ -0,0 +1,16 @@ +from .base import ProtocolCodec +from .v0 import ProtocolCodecV0 +from .v1 import ProtocolCodecV1 + +# Ordered oldest -> newest. The last codec is the one the current dataset +# protocol uses (CODECS[-1].version == DATASET_PROTOCOL_VERSION); adding a new +# on-disk format means appending a ProtocolCodecV here and bumping +# DATASET_PROTOCOL_VERSION. +CODECS = [ProtocolCodecV0, ProtocolCodecV1] + +__all__ = [ + "ProtocolCodec", + "ProtocolCodecV0", + "ProtocolCodecV1", + "CODECS", +] diff --git a/packages/syft-datasets/src/syft_datasets/protocolcodecs/base.py b/packages/syft-datasets/src/syft_datasets/protocolcodecs/base.py new file mode 100644 index 00000000000..1e21dc2174a --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/protocolcodecs/base.py @@ -0,0 +1,53 @@ +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Iterator + +from syft_migration import MigratableObject + +from ..config import METADATA_FILENAME, PRIVATE_METADATA_FILENAME, SyftBoxConfig +from ..dataset_ref import DatasetRef + + +class ProtocolCodec(ABC): + """Interface for reading/writing/listing datasets in one on-disk storage format. + + A codec owns raw disk layout + serialization for the protocol versions it + lists in ``protocol_versions`` (one codec can serve several — the layout is + derived from each dataset's ``protocol_version``). DatasetStorage owns + migration and selects a codec by protocol version; a codec never maps back + to one. The codec carries its own ``version`` independent of the protocols + it handles. + + A dataset is a directory with a public ``dataset.yaml`` (the discovery + marker, synced to peers) plus mock files, and a separate + ``private_metadata.yaml`` under the owner's private/ tree (never synced). + """ + + version: str # the codec's own version ("0", "1", ...) + protocol_versions: list[str] # protocol versions this codec reads/writes + metadata_marker = METADATA_FILENAME + private_metadata_marker = PRIVATE_METADATA_FILENAME + + def __init__(self, config: SyftBoxConfig) -> None: + self.config = config + + @abstractmethod + def public_dataset_dir(self, ref: DatasetRef) -> Path: ... + + @abstractmethod + def private_dataset_dir(self, ref: DatasetRef) -> Path: ... + + def metadata_path(self, ref: DatasetRef) -> Path: + return self.public_dataset_dir(ref) / self.metadata_marker + + def private_metadata_path(self, ref: DatasetRef) -> Path: + return self.private_dataset_dir(ref) / self.private_metadata_marker + + @abstractmethod + def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: ... + + @abstractmethod + def read(self, path: Path, canonical_name: str) -> dict: ... + + @abstractmethod + def write(self, path: Path, obj: MigratableObject) -> None: ... diff --git a/packages/syft-datasets/src/syft_datasets/protocolcodecs/v0.py b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v0.py new file mode 100644 index 00000000000..d2bb91920aa --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v0.py @@ -0,0 +1,52 @@ +from pathlib import Path +from typing import Iterator + +import yaml +from syft_migration import MigratableObject + +from ..config import is_protocol_dir_name +from ..dataset_ref import DatasetRef +from .base import ProtocolCodec + + +class ProtocolCodecV0(ProtocolCodec): + """Pre-versioning layout (<= 0.1.20): datasets flat under syft_datasets/, no identity fields.""" + + version = "0" + protocol_versions = ["0"] + + def public_dataset_dir(self, ref: DatasetRef) -> Path: + return self.config.get_mock_dataset_dir( + ref.name, ref.owner, ref.protocol_version + ) + + def private_dataset_dir(self, ref: DatasetRef) -> Path: + return self.config.get_private_dataset_dir( + ref.owner, ref.name, ref.protocol_version + ) + + def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: + root = self.config.public_datasets_root_for_datasite(datasite_email) + if not root.exists(): + return + for entry in sorted(p for p in root.iterdir() if p.is_dir()): + # v/ segments belong to a versioned codec, not the flat layout. + if is_protocol_dir_name(entry.name): + continue + if (entry / self.metadata_marker).exists(): + yield DatasetRef(datasite_email, entry.name, "0") + + def read(self, path: Path, canonical_name: str) -> dict: + data = yaml.safe_load(path.read_text()) or {} + # Protocol-0 files predate the identity fields; they are all version 1. + data.setdefault("canonical_name", canonical_name) + data.setdefault("version", "1") + return data + + def write(self, path: Path, obj: MigratableObject) -> None: + data = obj.disk_dict() + # Byte-match the pre-versioning (<= 0.1.20) on-disk format. + data.pop("canonical_name", None) + data.pop("version", None) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(data, indent=2, sort_keys=False)) diff --git a/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py new file mode 100644 index 00000000000..1c3119a38c3 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py @@ -0,0 +1,47 @@ +from pathlib import Path +from typing import Iterator + +import yaml +from syft_migration import MigratableObject + +from ..config import protocol_dir_name +from ..dataset_ref import DatasetRef +from .base import ProtocolCodec + + +class ProtocolCodecV1(ProtocolCodec): + """Versioned layout (>= 0.1.21): datasets under syft_datasets/v/, identity fields on disk.""" + + version = "1" + protocol_versions = ["1"] + + def public_dataset_dir(self, ref: DatasetRef) -> Path: + return self.config.get_mock_dataset_dir( + ref.name, ref.owner, ref.protocol_version + ) + + def private_dataset_dir(self, ref: DatasetRef) -> Path: + return self.config.get_private_dataset_dir( + ref.owner, ref.name, ref.protocol_version + ) + + def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: + root = self.config.public_datasets_root_for_datasite(datasite_email) + if not root.exists(): + return + for protocol_version in self.protocol_versions: + version_dir = root / protocol_dir_name(protocol_version) + if not version_dir.is_dir(): + continue + for entry in sorted(p for p in version_dir.iterdir() if p.is_dir()): + if (entry / self.metadata_marker).exists(): + yield DatasetRef(datasite_email, entry.name, protocol_version) + + def read(self, path: Path, canonical_name: str) -> dict: + # Files already carry canonical_name/version on disk. + return yaml.safe_load(path.read_text()) or {} + + def write(self, path: Path, obj: MigratableObject) -> None: + data = obj.disk_dict() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(data, indent=2, sort_keys=False)) diff --git a/packages/syft-datasets/src/syft_datasets/version.py b/packages/syft-datasets/src/syft_datasets/version.py new file mode 100644 index 00000000000..d2941ebc140 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/version.py @@ -0,0 +1,10 @@ +from importlib.metadata import version + +# Distribution name from pyproject ([project].name). The import package is +# "syft_datasets" but the distribution is "syft-dataset" (singular), so this is +# hardcoded rather than derived from __package__. +PACKAGE_NAME = "syft-dataset" + +# Derived from the installed distribution metadata (pyproject.toml's version) +# so it cannot drift from the released package version. +__version__ = version(PACKAGE_NAME) diff --git a/packages/syft-datasets/tests/migrations/__init__.py b/packages/syft-datasets/tests/migrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/syft-datasets/tests/migrations/p2p/__init__.py b/packages/syft-datasets/tests/migrations/p2p/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private.csv b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private.csv new file mode 100644 index 00000000000..f5c2e52a5a1 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private.csv @@ -0,0 +1,3 @@ +id,secret +1,alpha +2,beta diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private_metadata.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private_metadata.yaml new file mode 100644 index 00000000000..e30855a6492 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private_metadata.yaml @@ -0,0 +1,2 @@ +uid: 4362bc0f-000f-44db-88c0-be508aa814fa +data_dir: '' diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/syft.pub.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/syft.pub.yaml new file mode 100644 index 00000000000..c362f440602 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/syft.pub.yaml @@ -0,0 +1,8 @@ +rules: +- access: + admin: [] + read: + - do@test.org + write: [] + pattern: '**' +terminal: false diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/dataset.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/dataset.yaml new file mode 100644 index 00000000000..96f5fcf8ef3 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/dataset.yaml @@ -0,0 +1,13 @@ +uid: 4362bc0f-000f-44db-88c0-be508aa814fa +created_at: '2026-07-15T12:54:05.124932Z' +updated_at: '2026-07-15T12:54:05.125289Z' +name: demo +summary: demo dataset +tags: +- demo +location: null +mock_url: syft://do@test.org/public/syft_datasets/demo +private_url: syft://do@test.org/private/syft_datasets/demo +readme_url: syft://do@test.org/public/syft_datasets/demo/readme.md +mock_files_urls: +- syft://do@test.org/public/syft_datasets/demo/mock.csv diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/mock.csv b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/mock.csv new file mode 100644 index 00000000000..ea6be9205d3 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/mock.csv @@ -0,0 +1,3 @@ +id,value +1,10 +2,20 diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/readme.md b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/readme.md new file mode 100644 index 00000000000..c5b7a1f37d7 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/readme.md @@ -0,0 +1 @@ +# demo dataset diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/syft.pub.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/syft.pub.yaml new file mode 100644 index 00000000000..676a3c8f2e3 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/syft.pub.yaml @@ -0,0 +1,8 @@ +rules: +- access: + admin: [] + read: + - ds@test.org + write: [] + pattern: '**' +terminal: false diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private.csv b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private.csv new file mode 100644 index 00000000000..f5c2e52a5a1 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private.csv @@ -0,0 +1,3 @@ +id,secret +1,alpha +2,beta diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private_metadata.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private_metadata.yaml new file mode 100644 index 00000000000..9362a9fd6cf --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private_metadata.yaml @@ -0,0 +1,4 @@ +canonical_name: PrivateDatasetConfig +version: '1' +uid: fddcb47f-1d56-46d7-bf54-99be3ccc1e24 +data_dir: '' diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/syft.pub.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/syft.pub.yaml new file mode 100644 index 00000000000..c362f440602 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/syft.pub.yaml @@ -0,0 +1,8 @@ +rules: +- access: + admin: [] + read: + - do@test.org + write: [] + pattern: '**' +terminal: false diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/dataset.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/dataset.yaml new file mode 100644 index 00000000000..503efd923a4 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/dataset.yaml @@ -0,0 +1,15 @@ +canonical_name: Dataset +version: '1' +uid: fddcb47f-1d56-46d7-bf54-99be3ccc1e24 +created_at: '2026-07-15T12:54:04.399452Z' +updated_at: '2026-07-15T12:54:04.399619Z' +name: demo +summary: demo dataset +tags: +- demo +location: null +mock_url: syft://do@test.org/public/syft_datasets/v1/demo +private_url: syft://do@test.org/private/syft_datasets/v1/demo +readme_url: syft://do@test.org/public/syft_datasets/v1/demo/readme.md +mock_files_urls: +- syft://do@test.org/public/syft_datasets/v1/demo/mock.csv diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/mock.csv b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/mock.csv new file mode 100644 index 00000000000..ea6be9205d3 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/mock.csv @@ -0,0 +1,3 @@ +id,value +1,10 +2,20 diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/readme.md b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/readme.md new file mode 100644 index 00000000000..c5b7a1f37d7 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/readme.md @@ -0,0 +1 @@ +# demo dataset diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/syft.pub.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/syft.pub.yaml new file mode 100644 index 00000000000..676a3c8f2e3 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/syft.pub.yaml @@ -0,0 +1,8 @@ +rules: +- access: + admin: [] + read: + - ds@test.org + write: [] + pattern: '**' +terminal: false diff --git a/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py new file mode 100644 index 00000000000..b8573eb74b9 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py @@ -0,0 +1,109 @@ +"""End-to-end dataset flow through SyftDatasetManager. + +Datasets are broadcast (one public copy read by the whole audience), so by +default they are written in the widest-compatible (oldest) protocol so every +current peer can read them; the new v layout is written only for peers that +advertise support for it. +""" + +from pathlib import Path + +import yaml +from syft_datasets.dataset_manager import SyftDatasetManager +from syft_datasets.migrations import dataset_registry + +DO_EMAIL = "do@test.org" +DS0 = "old@test.org" +DS1 = "new@test.org" + + +def _src(tmp_path: Path) -> tuple[Path, Path, Path]: + src = tmp_path / "src" + src.mkdir() + mock = src / "mock.csv" + mock.write_text("id,value\n1,10\n") + private = src / "private.csv" + private.write_text("id,secret\n1,x\n") + readme = src / "readme.md" + readme.write_text("# demo\n") + return mock, private, readme + + +def _manager(tmp_path: Path, peer_schemas=None) -> SyftDatasetManager: + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + mgr = SyftDatasetManager(syftbox_folder_path=syftbox, email=DO_EMAIL) + if peer_schemas is not None: + mgr.storage.peer_schemas = peer_schemas + return mgr + + +def test_default_create_writes_protocol_0(tmp_path: Path): + mgr = _manager(tmp_path) + mock, private, readme = _src(tmp_path) + + dataset = mgr.create( + name="demo", mock_path=mock, private_path=private, readme_path=readme + ) + # Flat layout, no v, no identity fields (byte-compatible with 0.1.20). + assert dataset.mock_dir.parent.name == "syft_datasets" + raw = yaml.safe_load((dataset.mock_dir / "dataset.yaml").read_text()) + assert "canonical_name" not in raw + + got = mgr.get("demo") + assert got.name == "demo" + assert got.version == dataset_registry.latest_version("Dataset") + assert [p.name for p in got.mock_files] == ["mock.csv"] + assert [p.name for p in got.private_files] == ["private.csv"] + + +def test_create_for_protocol1_peer_writes_v1(tmp_path: Path): + schema1 = dataset_registry.schema_for_protocol_version("1") + mgr = _manager(tmp_path, peer_schemas={DS1: schema1}) + mock, private, readme = _src(tmp_path) + + dataset = mgr.create( + name="demo", + mock_path=mock, + private_path=private, + readme_path=readme, + users=[DS1], + ) + # Versioned layout, identity fields present. + assert dataset.mock_dir.parent.name == "v1" + raw = yaml.safe_load((dataset.mock_dir / "dataset.yaml").read_text()) + assert raw["canonical_name"] == "Dataset" and raw["version"] == "1" + + # The shared peer has read access to the (v1) mock dir. + perm = dataset.mock_dir / "syft.pub.yaml" + assert perm.exists() + assert DS1 in perm.read_text() + + got = mgr.get("demo") + assert got.name == "demo" and got._protocol_version == "1" + + +def test_multi_version_write_for_mixed_audience(tmp_path: Path): + schema0 = dataset_registry.schema_for_protocol_version("0") + schema1 = dataset_registry.schema_for_protocol_version("1") + mgr = _manager(tmp_path, peer_schemas={DS0: schema0, DS1: schema1}) + mock, private, readme = _src(tmp_path) + + mgr.create( + name="demo", + mock_path=mock, + private_path=private, + readme_path=readme, + users=[DS0, DS1], + ) + + root = mgr.syftbox_config.public_datasets_root_for_datasite(DO_EMAIL) + # Both layouts written: flat (protocol 0) and v1 (protocol 1). + assert (root / "demo" / "dataset.yaml").exists() + assert (root / "v1" / "demo" / "dataset.yaml").exists() + + # get_all() dedupes the two on-disk copies to one, preferring the newest. + all_datasets = mgr.get_all() + assert len(all_datasets) == 1 + assert all_datasets[0]._protocol_version == "1" + assert mgr.get("demo")._protocol_version == "1" diff --git a/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py b/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py new file mode 100644 index 00000000000..5840603ba5f --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py @@ -0,0 +1,122 @@ +"""Reading/writing datasets against every released syft-dataset on-disk layout. + +Each fixture tree under fixtures/ is named +``syft_dataset--protocol

_syftbox`` and holds a SyftBox exactly as +that release wrote a dataset to disk (a ``demo`` dataset: mock files + readme + +dataset.yaml, plus a private_metadata.yaml). The current code must still read, +upgrade and byte-exactly round-trip all of them. + +Protocol 0 is the last release (0.1.20) without a v path segment or +canonical_name/version fields; protocol >= 1 stores datasets under a v +segment and writes those identity fields. Fixtures are generated by +scripts/generate_release_fixture.py. +""" + +import re +import shutil +from pathlib import Path + +import pytest +from syft_datasets.config import SyftBoxConfig +from syft_datasets.dataset_manager import SyftDatasetManager +from syft_datasets.dataset_storage import DatasetStorage +from syft_datasets.migrations import dataset_registry + +DO_EMAIL = "do@test.org" +DATASET = "demo" + +FIXTURES_DIR = Path(__file__).parent / "fixtures" +RELEASE_FIXTURES = sorted(FIXTURES_DIR.glob("syft_dataset-*-protocol*_syftbox")) + +released_syftbox_fixtures = pytest.mark.parametrize( + "fixture", RELEASE_FIXTURES, ids=lambda f: f.name +) + + +def _protocol_of(fixture: Path) -> str: + return re.search(r"-protocol(\d+)_syftbox$", fixture.name).group(1) + + +def _has_identity(protocol: str) -> bool: + """canonical_name/version are written only from protocol 1 onwards.""" + return protocol != "0" + + +def _syftbox_w_old_datasets(fixture: Path, tmp_path: Path) -> Path: + syftbox = tmp_path / "SyftBox" + shutil.copytree(fixture, syftbox) + return syftbox + + +def _manager(syftbox: Path) -> SyftDatasetManager: + return SyftDatasetManager(syftbox_folder_path=syftbox, email=DO_EMAIL) + + +def _storage(syftbox: Path) -> DatasetStorage: + return DatasetStorage(config=SyftBoxConfig(syftbox_folder=syftbox, email=DO_EMAIL)) + + +@released_syftbox_fixtures +def test_old_datasets_load_and_upgrade(fixture: Path, tmp_path: Path): + syftbox = _syftbox_w_old_datasets(fixture, tmp_path) + mgr = _manager(syftbox) + + datasets = mgr.get_all() + assert {d.name for d in datasets} == {DATASET} + dataset = datasets[0] + # Loads into the latest registered version in memory. + assert dataset.version == dataset_registry.latest_version("Dataset") + assert dataset.owner == DO_EMAIL + assert dataset._protocol_version == _protocol_of(fixture) + # Private config also loads/upgrades to latest. + ref = mgr.storage.find_dataset_ref(DO_EMAIL, DATASET) + private_config = mgr.storage.read_private_config(ref) + assert private_config.version == dataset_registry.latest_version( + "PrivateDatasetConfig" + ) + + +@released_syftbox_fixtures +def test_write_back_is_byte_exact(fixture: Path, tmp_path: Path): + protocol = _protocol_of(fixture) + syftbox = _syftbox_w_old_datasets(fixture, tmp_path) + storage = _storage(syftbox) + + ref = storage.find_dataset_ref(DO_EMAIL, DATASET) + metadata_path = storage.metadata_path(ref) + original_bytes = metadata_path.read_text() + + # Read (upgrade in memory) then write back in that release's layout/format. + dataset = storage.read_dataset(ref) + storage.write_dataset(ref, dataset) + + assert metadata_path.read_text() == original_bytes + raw = metadata_path.read_text() + assert ("canonical_name" in raw) is _has_identity(protocol) + + +@released_syftbox_fixtures +def test_mixed_protocol_listing(fixture: Path, tmp_path: Path): + syftbox = _syftbox_w_old_datasets(fixture, tmp_path) + mgr = _manager(syftbox) + + # A newly created dataset (default protocol) coexists with the old one. + src = tmp_path / "src" + src.mkdir() + mock = src / "mock.csv" + mock.write_text("id\n1\n") + private = src / "private.csv" + private.write_text("id\n1\n") + mgr.create(name="fresh", mock_path=mock, private_path=private) + + assert {d.name for d in mgr.get_all()} == {DATASET, "fresh"} + + +@released_syftbox_fixtures +def test_dataset_reprs_do_not_error(fixture: Path, tmp_path: Path): + syftbox = _syftbox_w_old_datasets(fixture, tmp_path) + datasets = _manager(syftbox).get_all() + + for render in (repr, str, lambda o: o._repr_html_()): + assert render(datasets) + assert render(datasets[0]) diff --git a/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py b/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py new file mode 100644 index 00000000000..556e16d69f4 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py @@ -0,0 +1,143 @@ +"""Structure, coverage, and behavior of the ProtocolCodecs behind DatasetStorage. + +A codec is selected by PROTOCOL version and may serve several of them +(``protocol_versions``); it also carries its own ``version``. The invariants +here are about protocol-version coverage: every protocol version the registry +understands is handled by exactly one codec. +""" + +import importlib +import inspect +import pkgutil +from pathlib import Path + +import yaml +from syft_datasets import protocolcodecs +from syft_datasets.config import SyftBoxConfig +from syft_datasets.dataset_storage import DatasetRef, DatasetStorage +from syft_datasets.migrations import dataset_registry +from syft_datasets.migrations.registry import DATASET_PROTOCOL_VERSION +from syft_datasets.models import Dataset +from syft_datasets.protocolcodecs import ProtocolCodec + +DO_EMAIL = "do@test.org" + + +def _storage(tmp_path: Path) -> DatasetStorage: + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + config = SyftBoxConfig(syftbox_folder=syftbox, email=DO_EMAIL) + return DatasetStorage(config=config) + + +def _mock_dataset(storage: DatasetStorage, ref: DatasetRef) -> Dataset: + cfg = storage.config + return Dataset( + name=ref.name, + mock_url=cfg.get_mock_url_for_my_dataset(ref.name, ref.protocol_version), + private_url=cfg.get_private_url_for_my_dataset(ref.name, ref.protocol_version), + ) + + +# -- structural invariant ------------------------------------------------------ +def test_codec_versions_unique_and_protocol_versions_disjoint(tmp_path: Path): + codecs = _storage(tmp_path).codecs + + codec_versions = [c.version for c in codecs] + assert len(codec_versions) == len(set(codec_versions)), ( + "codec versions must be unique" + ) + + seen: set[str] = set() + for codec in codecs: + for protocol_version in codec.protocol_versions: + assert protocol_version not in seen, ( + f"protocol {protocol_version} handled by more than one codec" + ) + seen.add(protocol_version) + + # The current protocol is handled by the last (newest) codec. + assert DATASET_PROTOCOL_VERSION in codecs[-1].protocol_versions + + +def _all_concrete_codec_classes() -> set[type[ProtocolCodec]]: + for module_info in pkgutil.iter_modules(protocolcodecs.__path__): + importlib.import_module(f"{protocolcodecs.__name__}.{module_info.name}") + + def descendants(cls: type) -> set[type]: + subs = set(cls.__subclasses__()) + return subs.union(*(descendants(s) for s in subs)) + + return {cls for cls in descendants(ProtocolCodec) if not inspect.isabstract(cls)} + + +# -- registration -------------------------------------------------------------- +def test_all_codecs_in_codebase_are_registered_in_dataset_storage(tmp_path: Path): + registered_codecs = {type(codec) for codec in _storage(tmp_path).codecs} + defined_codecs = _all_concrete_codec_classes() + + missing = defined_codecs - registered_codecs + assert not missing, ( + f"codec(s) defined but not registered in DatasetStorage/CODECS: " + f"{sorted(cls.__name__ for cls in missing)}" + ) + + +# -- coverage ------------------------------------------------------------------ +def test_every_known_protocol_version_is_covered_by_exactly_one_codec(tmp_path: Path): + storage = _storage(tmp_path) + + known = set(dataset_registry.protocol_version_history) | {DATASET_PROTOCOL_VERSION} + covered = {pv for codec in storage.codecs for pv in codec.protocol_versions} + assert covered == known + + for protocol_version in known: + assert storage._codec_for(protocol_version) is not None + + +# -- behavior 1: on-disk format per codec -------------------------------------- +def test_v0_writes_flat_no_identity_v1_nests_with_identity(tmp_path: Path): + storage = _storage(tmp_path) + + ref0 = DatasetRef(DO_EMAIL, "flat", "0") + ref1 = DatasetRef(DO_EMAIL, "nested", "1") + p0 = storage.write_dataset(ref0, _mock_dataset(storage, ref0)) + p1 = storage.write_dataset(ref1, _mock_dataset(storage, ref1)) + + # protocol 0: public/syft_datasets//dataset.yaml (no v), identity stripped + assert p0.parent.parent.name == "syft_datasets" + raw0 = yaml.safe_load(p0.read_text()) + assert "canonical_name" not in raw0 and "version" not in raw0 + + # protocol 1: public/syft_datasets/v1//dataset.yaml, identity present + assert p1.parent.parent.name == "v1" + raw1 = yaml.safe_load(p1.read_text()) + assert raw1["canonical_name"] == "Dataset" and raw1["version"] == "1" + + for ref in (ref0, ref1): + loaded = storage.read_dataset(ref) + assert loaded.name == ref.name + assert loaded.version == dataset_registry.latest_version("Dataset") + + +# -- behavior 2: each codec scans only its own layout -------------------------- +def test_scan_partitions_by_layout(tmp_path: Path): + storage = _storage(tmp_path) + v0_codec, v1_codec = storage._codec_for("0"), storage._codec_for("1") + + ref0 = DatasetRef(DO_EMAIL, "flat", "0") + ref1 = DatasetRef(DO_EMAIL, "nested", "1") + storage.write_dataset(ref0, _mock_dataset(storage, ref0)) + storage.write_dataset(ref1, _mock_dataset(storage, ref1)) + + v0_refs = list(v0_codec.iter_dataset_refs(DO_EMAIL)) + v1_refs = list(v1_codec.iter_dataset_refs(DO_EMAIL)) + assert [(r.name, r.protocol_version) for r in v0_refs] == [("flat", "0")] + assert [(r.name, r.protocol_version) for r in v1_refs] == [("nested", "1")] + + all_refs = list(storage.iter_dataset_refs(DO_EMAIL)) + assert {(r.name, r.protocol_version) for r in all_refs} == { + ("flat", "0"), + ("nested", "1"), + } + assert len(all_refs) == 2 diff --git a/packages/syft-datasets/tests/migrations/p2p/test_protocol_sanity_checks.py b/packages/syft-datasets/tests/migrations/p2p/test_protocol_sanity_checks.py new file mode 100644 index 00000000000..34b3a437ea4 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/test_protocol_sanity_checks.py @@ -0,0 +1,63 @@ +"""Sanity checks around the protocol-versioned dataset layout.""" + +from pathlib import Path + +import pytest +from syft_datasets.config import SyftBoxConfig +from syft_datasets.dataset_storage import DatasetStorage +from syft_datasets.migrations import dataset_registry +from syft_datasets.migrations.registry import DATASET_PROTOCOL_VERSION +from syft_migration import MigrationError + +DO_EMAIL = "do@test.org" +DS_EMAIL = "ds@test.org" + + +def _storage(tmp_path: Path, peer_schemas=None) -> DatasetStorage: + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + config = SyftBoxConfig(syftbox_folder=syftbox, email=DO_EMAIL) + return DatasetStorage(config=config, peer_schemas=peer_schemas) + + +def test_reserved_dataset_name_rejected(tmp_path: Path): + storage = _storage(tmp_path) + with pytest.raises(ValueError, match="reserved"): + storage.validate_dataset_name("v2") + # Names that merely resemble the segment are fine. + storage.validate_dataset_name("v2x") + storage.validate_dataset_name("version1") + + +def test_negotiated_protocol_version_for_peer(tmp_path: Path): + protocol0_schema = dataset_registry.schema_for_protocol_version("0") + storage = _storage(tmp_path, peer_schemas={DS_EMAIL: protocol0_schema}) + + # An older peer negotiates down to the version both sides can read. + assert storage.negotiated_protocol_version_for_peer(DS_EMAIL) == "0" + with pytest.raises(MigrationError): + storage.negotiated_protocol_version_for_peer("stranger@test.org") + # Opting out assumes the current protocol. + assert ( + storage.negotiated_protocol_version_for_peer( + "stranger@test.org", raise_on_unknown=False + ) + == DATASET_PROTOCOL_VERSION + ) + + +def test_target_protocol_versions_for_peers(tmp_path: Path): + schema0 = dataset_registry.schema_for_protocol_version("0") + schema1 = dataset_registry.schema_for_protocol_version("1") + storage = _storage( + tmp_path, peer_schemas={"old@test.org": schema0, "new@test.org": schema1} + ) + + # No audience -> widest-compatible (oldest) protocol. + assert storage.target_protocol_versions_for_peers() == {"0"} + # Unknown peer -> also widest-compatible. + assert storage.target_protocol_versions_for_peers(["stranger@test.org"]) == {"0"} + # Mixed audience -> a copy per distinct version. + assert storage.target_protocol_versions_for_peers( + ["old@test.org", "new@test.org"] + ) == {"0", "1"} diff --git a/packages/syft-datasets/tests/migrations/unit/__init__.py b/packages/syft-datasets/tests/migrations/unit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/syft-datasets/tests/migrations/unit/fixtures/Dataset/v1.yaml b/packages/syft-datasets/tests/migrations/unit/fixtures/Dataset/v1.yaml new file mode 100644 index 00000000000..7666a9aabd6 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/fixtures/Dataset/v1.yaml @@ -0,0 +1,15 @@ +canonical_name: Dataset +version: '1' +uid: 00000000-0000-0000-0000-000000000001 +created_at: '2026-07-01T00:00:00Z' +updated_at: '2026-07-01T00:00:00Z' +name: demo +summary: demo dataset +tags: +- demo +location: null +mock_url: syft://do@test.org/public/syft_datasets/demo +private_url: syft://do@test.org/private/syft_datasets/demo +readme_url: syft://do@test.org/public/syft_datasets/demo/readme.md +mock_files_urls: +- syft://do@test.org/public/syft_datasets/demo/mock.csv diff --git a/packages/syft-datasets/tests/migrations/unit/fixtures/PrivateDatasetConfig/v1.yaml b/packages/syft-datasets/tests/migrations/unit/fixtures/PrivateDatasetConfig/v1.yaml new file mode 100644 index 00000000000..d6575f6a9fd --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/fixtures/PrivateDatasetConfig/v1.yaml @@ -0,0 +1,4 @@ +canonical_name: PrivateDatasetConfig +version: '1' +uid: 00000000-0000-0000-0000-000000000001 +data_dir: . diff --git a/packages/syft-datasets/tests/migrations/unit/mocks.py b/packages/syft-datasets/tests/migrations/unit/mocks.py new file mode 100644 index 00000000000..c994191729d --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/mocks.py @@ -0,0 +1,35 @@ +"""Shared mock objects for the syft-dataset migration tests.""" + +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID + +from syft_datasets.models import DatasetV1, PrivateDatasetConfigV1 +from syft_datasets.url import SyftBoxURL + +DO_EMAIL = "do@test.org" +DS_EMAIL = "ds@test.org" +DATASET_NAME = "demo" +DATASET_UID = UUID("00000000-0000-0000-0000-000000000001") + + +def create_mock_dataset() -> DatasetV1: + base = f"syft://{DO_EMAIL}/public/syft_datasets/{DATASET_NAME}" + return DatasetV1( + uid=DATASET_UID, + created_at=datetime(2026, 7, 1, tzinfo=timezone.utc), + updated_at=datetime(2026, 7, 1, tzinfo=timezone.utc), + name=DATASET_NAME, + summary="demo dataset", + tags=["demo"], + mock_url=SyftBoxURL(base), + private_url=SyftBoxURL( + f"syft://{DO_EMAIL}/private/syft_datasets/{DATASET_NAME}" + ), + readme_url=SyftBoxURL(f"{base}/readme.md"), + mock_files_urls=[SyftBoxURL(f"{base}/mock.csv")], + ) + + +def create_mock_private_config() -> PrivateDatasetConfigV1: + return PrivateDatasetConfigV1(uid=DATASET_UID, data_dir=Path("")) diff --git a/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py b/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py new file mode 100644 index 00000000000..82c816dd21e --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py @@ -0,0 +1,100 @@ +"""The hardcoded release artifacts of past syft-dataset releases.""" + +from syft_migration import ( + MigrationService, + ReleasedPackageProtocolInfo, + ReleasedProtocol, +) + +from syft_datasets.migrations import dataset_registry +from syft_datasets.migrations.history import PACKAGE_ARTIFACTS_DIR, PROTOCOLS_DIR +from syft_datasets.models import DatasetV1 + + +def test_0_1_20_artifact_file_loads(): + artifact = ReleasedPackageProtocolInfo.load( + PACKAGE_ARTIFACTS_DIR / "syft-dataset-0.1.20.json" + ) + assert artifact.package_info.package_name == "syft-dataset" + assert artifact.package_info.version == "0.1.20" + assert artifact.package_info.protocol_version == "0" + assert artifact.protocol_schema.protocol_name == "syft-dataset" + assert artifact.protocol_schema.version == "0" + expected_versions = {"Dataset": ["1"], "PrivateDatasetConfig": ["1"]} + assert artifact.protocol_schema.supported_versions == expected_versions + + +def test_all_released_protocols_load(): + protocol_paths = sorted(PROTOCOLS_DIR.glob("*.json")) + assert protocol_paths # at least protocol-0.json exists + + for path in protocol_paths: + released = ReleasedProtocol.load(path) + schema = released.protocol_schema + # The filename encodes the protocol version: protocol-.json. + assert path.name == f"protocol-{schema.version}.json" + assert schema.protocol_name == "syft-dataset" + assert schema.supported_versions + assert set(schema.current_object_schemas) == set(schema.supported_versions) + for canonical_name in schema.supported_versions: + assert schema.current_schema(canonical_name) + + +def test_protocol_0_released_protocol_loads(): + released = ReleasedProtocol.load(PROTOCOLS_DIR / "protocol-0.json") + schema = released.protocol_schema + assert schema.version == "0" + assert set(schema.current_object_schemas) == {"Dataset", "PrivateDatasetConfig"} + assert schema.current_schema("Dataset") == "1" + assert schema.current_schema("PrivateDatasetConfig") == "1" + + +def test_released_object_schemas_unchanged(): + # Released object versions are frozen forever; see the failure message. + drift = dataset_registry.find_schema_drift() + assert drift == [], ( + f"Released object schemas changed: {drift} " + "(canonical_name, object_version, protocol_version).\n" + "A class that shipped in a released protocol was modified in place. " + "Released versions are frozen forever, because peers on old releases " + "still read/write them. To fix:\n" + " 1. Revert your change to the released class (e.g. DatasetV1).\n" + " 2. Create the next version instead: copy the class into " + "models//v.py with version='' and apply your change " + "there.\n" + " 3. Point the current-version alias in models//__init__.py " + "at the new class.\n" + " 4. Register migrations in BOTH directions (v -> v and back) " + "so old peers stay supported; test_upgrade_paths enforces this.\n" + " 5. Add a serialized fixture: tests/migrations/unit/fixtures/" + "/v.yaml.\n" + " 6. Bump DATASET_PROTOCOL_VERSION in syft_datasets/migrations/" + "registry.py — a new object version is a protocol change.\n" + "If NO class was edited and this failure appeared after a pydantic " + "upgrade, model_json_schema() output changed cosmetically: review the " + "diff carefully and regenerate the files under migrations/history/." + ) + + +def test_protocol_bumped_when_changed(): + # Fires when the dataset protocol changes (e.g. a new object version registers) + # without bumping DATASET_PROTOCOL_VERSION. + assert not dataset_registry.protocol_changed_without_bump() + + +def test_historic_schemas_registered_on_import(): + # syft_datasets/__init__ registers every artifact in migrations/history/. + assert dataset_registry.package_version_history["0"].version == "0.1.20" + schema = dataset_registry.schema_for_protocol_version("0") + assert schema.current_schema("Dataset") == "1" + assert schema.current_schema("PrivateDatasetConfig") == "1" + + +def test_downgrade_for_last_released_protocol_version(): + service = MigrationService(registry=dataset_registry) + from .mocks import create_mock_dataset + + dataset = create_mock_dataset() + downgraded = service.downgrade_for_protocol_version(dataset, "0") + assert isinstance(downgraded, DatasetV1) + assert downgraded.name == "demo" diff --git a/packages/syft-datasets/tests/migrations/unit/test_migrations.py b/packages/syft-datasets/tests/migrations/unit/test_migrations.py new file mode 100644 index 00000000000..6201667e87c --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/test_migrations.py @@ -0,0 +1,72 @@ +"""Migrations against the REAL dataset registry, using on-disk fixtures. + +Every registered version of each object has a serialized fixture under +``fixtures//v.yaml``; each fixture loads and upgrades +in memory to the latest version, and the latest version downgrades to every +registered version and round-trips through disk. With only V1 registered these +are no-ops; when a newer version is registered, these tests fail until a fixture +for it is added and then exercise the real migration paths automatically. +""" + +from pathlib import Path + +import yaml +from syft_migration import MigrationService + +from syft_datasets.migrations import dataset_registry + +FIXTURES_DIR = Path(__file__).parent / "fixtures" + + +def _load_fixture(canonical_name: str, version: str) -> dict: + path = FIXTURES_DIR / canonical_name / f"v{version}.yaml" + assert path.exists(), ( + f"Missing fixture {path}; add one for every registered version" + ) + return yaml.safe_load(path.read_text()) + + +def _assert_all_upgrade_to_latest(canonical_name: str): + service = MigrationService(registry=dataset_registry) + latest = dataset_registry.latest_version(canonical_name) + latest_cls = dataset_registry.get_class(canonical_name, latest) + + for version in dataset_registry.versions(canonical_name): + upgraded = service.load( + _load_fixture(canonical_name, version), target_version=latest + ) + assert type(upgraded) is latest_cls + + +def _assert_latest_downgrades_to_all(canonical_name: str, tmp_path: Path): + service = MigrationService(registry=dataset_registry) + latest = dataset_registry.latest_version(canonical_name) + newest = service.load(_load_fixture(canonical_name, latest)) + + for version in dataset_registry.versions(canonical_name): + version_cls = dataset_registry.get_class(canonical_name, version) + downgraded = service.migrate(newest, target_version=version) + assert type(downgraded) is version_cls + + path = tmp_path / f"{canonical_name}_v{version}.yaml" + path.write_text(yaml.safe_dump(downgraded.disk_dict())) + reloaded = service.load( + yaml.safe_load(path.read_text()), target_version=version + ) + assert reloaded == downgraded + + +def test_all_dataset_versions_upgrade_from_disk_to_latest(): + _assert_all_upgrade_to_latest("Dataset") + + +def test_latest_dataset_downgrades_to_all_versions(tmp_path: Path): + _assert_latest_downgrades_to_all("Dataset", tmp_path) + + +def test_all_private_config_versions_upgrade_from_disk_to_latest(): + _assert_all_upgrade_to_latest("PrivateDatasetConfig") + + +def test_latest_private_config_downgrades_to_all_versions(tmp_path: Path): + _assert_latest_downgrades_to_all("PrivateDatasetConfig", tmp_path) diff --git a/packages/syft-datasets/tests/migrations/unit/test_mock_migrations.py b/packages/syft-datasets/tests/migrations/unit/test_mock_migrations.py new file mode 100644 index 00000000000..5a8d7a12392 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/test_mock_migrations.py @@ -0,0 +1,107 @@ +"""Upgrading and downgrading dataset objects across versions. + +Simulates the NEXT release: V2 objects and migrations, in test scope only. +""" + +import yaml +from syft_migration import MigrationRegistry, MigrationService + +from syft_datasets.models import DatasetV1, PrivateDatasetConfigV1 + +from .mocks import create_mock_dataset, create_mock_private_config + + +def _register_dataset_v2(registry: MigrationRegistry) -> type: + class DatasetV2(DatasetV1, registry=registry): + version: str = "2" + license: str = "unknown" # new in v2 + + registry.register_migration( + canonical_name="Dataset", + from_version="1", + to_version="2", + fn=lambda obj: DatasetV2(**obj.model_dump(exclude={"version"})), + ) + registry.register_migration( + canonical_name="Dataset", + from_version="2", + to_version="1", + fn=lambda obj: DatasetV1(**obj.model_dump(exclude={"version", "license"})), + ) + return DatasetV2 + + +def _register_private_config_v2(registry: MigrationRegistry) -> type: + class PrivateDatasetConfigV2(PrivateDatasetConfigV1, registry=registry): + version: str = "2" + checksum: str = "" # new in v2 + + registry.register_migration( + canonical_name="PrivateDatasetConfig", + from_version="1", + to_version="2", + fn=lambda obj: PrivateDatasetConfigV2(**obj.model_dump(exclude={"version"})), + ) + registry.register_migration( + canonical_name="PrivateDatasetConfig", + from_version="2", + to_version="1", + fn=lambda obj: PrivateDatasetConfigV1( + **obj.model_dump(exclude={"version", "checksum"}) + ), + ) + return PrivateDatasetConfigV2 + + +def _version_registry_with_migrations() -> tuple[MigrationRegistry, type, type]: + """A fresh registry holding the current objects plus test-scope V2 versions.""" + registry = MigrationRegistry( + protocol_name="syft-dataset", + package_name="syft-dataset", + package_version="test-next-release", + protocol_version="test-next", + ) + registry.register_object_version(DatasetV1) + registry.register_object_version(PrivateDatasetConfigV1) + dataset_v2 = _register_dataset_v2(registry) + private_config_v2 = _register_private_config_v2(registry) + return registry, dataset_v2, private_config_v2 + + +def test_dataset_upgrades_from_disk(tmp_path): + registry, DatasetV2, _ = _version_registry_with_migrations() + service = MigrationService(registry=registry) + + # An old (v1) dataset file on disk loads and migrates to the next version. + path = tmp_path / "dataset.yaml" + path.write_text(yaml.safe_dump(create_mock_dataset().disk_dict())) + upgraded = service.load(yaml.safe_load(path.read_text()), target_version="2") + assert type(upgraded) is DatasetV2 + assert upgraded.name == "demo" + assert upgraded.license == "unknown" + + +def test_dataset_downgrades(): + registry, DatasetV2, _ = _version_registry_with_migrations() + service = MigrationService(registry=registry) + + v2_only = DatasetV2( + **create_mock_dataset().model_dump(exclude={"version"}), license="mit" + ) + downgraded = service.migrate(v2_only, target_version="1") + assert type(downgraded) is DatasetV1 + assert downgraded.name == "demo" + assert not hasattr(downgraded, "license") + + +def test_private_config_upgrades_and_downgrades(): + registry, _, PrivateDatasetConfigV2 = _version_registry_with_migrations() + service = MigrationService(registry=registry) + + upgraded = service.migrate(create_mock_private_config(), target_version="2") + assert type(upgraded) is PrivateDatasetConfigV2 + assert upgraded.checksum == "" + + downgraded = service.migrate(upgraded, target_version="1") + assert type(downgraded) is PrivateDatasetConfigV1 + assert not hasattr(downgraded, "checksum") diff --git a/packages/syft-datasets/tests/migrations/unit/test_objects_registered.py b/packages/syft-datasets/tests/migrations/unit/test_objects_registered.py new file mode 100644 index 00000000000..bf7a12eb019 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/test_objects_registered.py @@ -0,0 +1,58 @@ +"""Every versioned syft-dataset object is known to the package registry.""" + +import importlib +import pkgutil + +from syft_migration import MigratableObject + +import syft_datasets +from syft_datasets.migrations import dataset_registry +from syft_datasets.models import ( + Dataset, + DatasetV1, + PrivateDatasetConfig, + PrivateDatasetConfigV1, +) + + +def test_versioned_objects_registered_and_aliased(): + # Both objects have at least one version registered in the package registry. + assert dataset_registry.versions("Dataset") + assert dataset_registry.versions("PrivateDatasetConfig") + + # The current-version aliases resolve to the V1 classes. + assert Dataset is DatasetV1 + assert PrivateDatasetConfig is PrivateDatasetConfigV1 + + # The protocol schema covers both objects and resolves a current version. + schema = dataset_registry.compute_protocol_schema() + assert {"Dataset", "PrivateDatasetConfig"} <= set(schema.supported_versions) + assert schema.current_schema(canonical_name="Dataset") + assert schema.current_schema(canonical_name="PrivateDatasetConfig") + + +def _all_subclasses(cls: type) -> set[type]: + subclasses = set(cls.__subclasses__()) + for sub in cls.__subclasses__(): + subclasses |= _all_subclasses(sub) + return subclasses + + +def test_all_migratable_objects_in_package_are_registered(): + # Import every syft_datasets module so all MigratableObject subclasses are defined. + for module_info in pkgutil.walk_packages( + syft_datasets.__path__, prefix="syft_datasets." + ): + importlib.import_module(module_info.name) + + package_objects = [ + cls + for cls in _all_subclasses(MigratableObject) + if cls.__module__.startswith("syft_datasets.") + ] + assert len(package_objects) >= 2 # the scan actually found the dataset objects + + for cls in package_objects: + canonical_name = cls.model_fields["canonical_name"].default + version = cls.model_fields["version"].default + assert dataset_registry.get_class(canonical_name, version) is cls diff --git a/packages/syft-datasets/tests/migrations/unit/test_serialization.py b/packages/syft-datasets/tests/migrations/unit/test_serialization.py new file mode 100644 index 00000000000..cbee4d50032 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/test_serialization.py @@ -0,0 +1,40 @@ +"""Versioned syft-dataset objects serialize with their identity and load back.""" + +import yaml +from syft_migration import MigrationService + +from syft_datasets.migrations import dataset_registry +from syft_datasets.models import DatasetV1, PrivateDatasetConfigV1 + +from .mocks import create_mock_dataset, create_mock_private_config + + +def test_dataset_serialization_roundtrip(): + dataset = create_mock_dataset() + data = yaml.safe_load(yaml.safe_dump(dataset.disk_dict())) + loaded = DatasetV1(**data) + assert loaded == dataset + assert loaded.canonical_name == "Dataset" + assert loaded.version == "1" + + +def test_private_config_serialization_roundtrip(): + config = create_mock_private_config() + data = yaml.safe_load(yaml.safe_dump(config.disk_dict())) + loaded = PrivateDatasetConfigV1(**data) + assert loaded == config + assert loaded.canonical_name == "PrivateDatasetConfig" + assert loaded.version == "1" + + +def test_migration_service_loads_into_versioned_class(): + service = MigrationService(registry=dataset_registry) + + dataset = create_mock_dataset() + loaded = service.load(dataset.disk_dict()) + assert isinstance(loaded, DatasetV1) + assert loaded.name == "demo" + + config = create_mock_private_config() + loaded_config = service.load(config.disk_dict()) + assert isinstance(loaded_config, PrivateDatasetConfigV1) diff --git a/packages/syft-datasets/tests/migrations/unit/test_upgrade_paths.py b/packages/syft-datasets/tests/migrations/unit/test_upgrade_paths.py new file mode 100644 index 00000000000..06baccba6d0 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/test_upgrade_paths.py @@ -0,0 +1,28 @@ +"""Every registered object version can migrate up to latest and down to any lower.""" + +from syft_datasets.migrations import dataset_registry + + +def test_every_version_has_upgrade_path_to_latest(): + assert dataset_registry.objects # sanity: the registry is populated + + for canonical_name, versions in dataset_registry.objects.items(): + for version in versions: + assert dataset_registry.has_upgradeable_path_to_latest( + canonical_name=canonical_name, from_version=version + ), f"No upgrade path for {canonical_name!r} v{version} to latest" + + +def test_every_version_has_downgrade_path_to_all_lower_versions(): + assert dataset_registry.objects # sanity: the registry is populated + + for canonical_name, versions in dataset_registry.objects.items(): + for higher in versions: + for lower in versions: + if lower >= higher: + continue + assert dataset_registry.has_migration_path( + canonical_name=canonical_name, + from_version=higher, + to_version=lower, + ), f"No downgrade path for {canonical_name!r} v{higher} to v{lower}" diff --git a/uv.lock b/uv.lock index 4156afcaf5a..cfb24ccb349 100644 --- a/uv.lock +++ b/uv.lock @@ -4620,10 +4620,11 @@ wheels = [ [[package]] name = "syft-dataset" -version = "0.1.20" +version = "0.1.21" source = { editable = "packages/syft-datasets" } dependencies = [ { name = "pyyaml" }, + { name = "syft-migration" }, { name = "syft-notebook-ui" }, { name = "syft-perms" }, ] @@ -4636,6 +4637,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "pyyaml", specifier = ">=6.0.3" }, + { name = "syft-migration", editable = "packages/syft-migration" }, { name = "syft-notebook-ui", editable = "packages/syft-notebook-ui" }, { name = "syft-perms", editable = "packages/syft-perms" }, ] From 066d617b11928057321eb372a8a06f899f215fff Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Wed, 15 Jul 2026 16:28:14 +0200 Subject: [PATCH 2/2] fix: resolve dataset private dir at its actual on-disk protocol layout The enclave inference example computed private dataset dirs using the default protocol version (v1 segment), but datasets shared with 'any' are written at the widest-compatible protocol (0, no segment). This mismatch broke the log-sink path and the ensure_logs_dataset idempotency check. Resolve the actual on-disk ref via DatasetStorage, falling back to the widest-compatible protocol for datasets not yet synced. --- .../src/enclave_model_api/logs_dataset.py | 14 ++++++++---- .../src/enclave_model_api/paths.py | 22 ++++++++++++++++++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/enclave-model-api-example/src/enclave_model_api/logs_dataset.py b/packages/enclave-model-api-example/src/enclave_model_api/logs_dataset.py index 8a001fc94cd..422c8c29f08 100644 --- a/packages/enclave-model-api-example/src/enclave_model_api/logs_dataset.py +++ b/packages/enclave-model-api-example/src/enclave_model_api/logs_dataset.py @@ -10,7 +10,10 @@ import tempfile from pathlib import Path +from syft_datasets.dataset_ref import DatasetNotFoundError + from enclave_model_api.log_writer import LOG_FILE_NAME +from enclave_model_api.paths import resolve_private_dataset_dir LOGS_DATASET_SUMMARY = ( "Inference request logs (prompt, completion, stats) collected by the " @@ -41,9 +44,12 @@ def ensure_logs_dataset(client, name: str) -> Path: Idempotent so reboots with fresh_state=false keep the existing logs. """ - config = client.datasets.syftbox_config - if (config.get_my_mock_dataset_dir(name) / "dataset.yaml").exists(): - return config.private_dir_for_my_dataset(name) + storage = client.datasets.storage + email = storage.config.email + try: + return storage.private_dataset_dir(storage.find_dataset_ref(email, name)) + except DatasetNotFoundError: + pass mock_dir, private_dir = _seed_dirs() client.create_dataset( @@ -56,4 +62,4 @@ def ensure_logs_dataset(client, name: str) -> Path: sync=False, ) client.sync() - return config.private_dir_for_my_dataset(name) + return resolve_private_dataset_dir(storage, email, name) diff --git a/packages/enclave-model-api-example/src/enclave_model_api/paths.py b/packages/enclave-model-api-example/src/enclave_model_api/paths.py index 7f3eea44856..499609e1d72 100644 --- a/packages/enclave-model-api-example/src/enclave_model_api/paths.py +++ b/packages/enclave-model-api-example/src/enclave_model_api/paths.py @@ -9,6 +9,8 @@ from syft_client.sync.syftbox_manager import get_jupyter_default_syftbox_folder from syft_datasets.config import SyftBoxConfig +from syft_datasets.dataset_ref import DatasetNotFoundError, DatasetRef +from syft_datasets.dataset_storage import DatasetStorage def default_syftbox_folder(email: str) -> Path: @@ -16,12 +18,30 @@ def default_syftbox_folder(email: str) -> Path: return get_jupyter_default_syftbox_folder(email) +def resolve_private_dataset_dir(storage: DatasetStorage, owner: str, name: str) -> Path: + """Private dir at the dataset's actual on-disk protocol layout. + + A dataset may live at protocol 0 (flat) or under a ``v`` segment; the + written layout depends on what the audience can read, not the current + default. Datasets not yet on disk (e.g. weights still syncing) fall back to + the widest-compatible protocol — where a peer running any current release + writes them for us. + """ + try: + ref = storage.find_dataset_ref(owner, name) + except DatasetNotFoundError: + (widest,) = storage.target_protocol_versions_for_peers(None) + ref = DatasetRef(owner=owner, name=name, protocol_version=widest) + return storage.private_dataset_dir(ref) + + def private_dataset_dir( syftbox_folder: Path | str, datasite: str, dataset_name: str ) -> Path: """Private dir of *dataset_name* on *datasite* inside *syftbox_folder*.""" config = SyftBoxConfig(syftbox_folder=Path(syftbox_folder), email=datasite) - return config.private_dir_for_my_dataset(dataset_name) + storage = DatasetStorage(config=config) + return resolve_private_dataset_dir(storage, datasite, dataset_name) def find_checkpoint_dir(weights_dir: Path | str) -> Path | None: