diff --git a/packages/syft-bg/pyproject.toml b/packages/syft-bg/pyproject.toml index 8b6a98fde30..70cc208ecd7 100644 --- a/packages/syft-bg/pyproject.toml +++ b/packages/syft-bg/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "pydantic>=2.0.0", "google-cloud-pubsub>=2.18.0", "syft-job==0.1.39", + "syft-rds", ] [project.scripts] @@ -24,6 +25,7 @@ syft-bg = "syft_bg.cli:main" [tool.uv.sources] "syft-job" = { workspace = true } +"syft-rds" = { workspace = true } [build-system] requires = ["hatchling"] diff --git a/packages/syft-bg/src/syft_bg/approve/handlers/job.py b/packages/syft-bg/src/syft_bg/approve/handlers/job.py index fb661ad4bf1..2819cc36ff7 100644 --- a/packages/syft-bg/src/syft_bg/approve/handlers/job.py +++ b/packages/syft-bg/src/syft_bg/approve/handlers/job.py @@ -18,7 +18,7 @@ ) if TYPE_CHECKING: - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds import SyftRDSClient class StateManager(Protocol): @@ -33,7 +33,7 @@ class JobApprovalHandler: def __init__( self, - client: SyftboxManager, + client: SyftRDSClient, config_path: Optional[Path] = None, state: Optional[StateManager] = None, on_approve: Optional[Callable[[JobInfo], None]] = None, diff --git a/packages/syft-bg/src/syft_bg/approve/handlers/peer.py b/packages/syft-bg/src/syft_bg/approve/handlers/peer.py index d9fda86e026..5fd9dd45165 100644 --- a/packages/syft-bg/src/syft_bg/approve/handlers/peer.py +++ b/packages/syft-bg/src/syft_bg/approve/handlers/peer.py @@ -7,7 +7,7 @@ from syft_bg.approve.config import PeerApprovalConfig if TYPE_CHECKING: - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds import SyftRDSClient class StateManager(Protocol): @@ -22,7 +22,7 @@ class PeerApprovalHandler: def __init__( self, - client: SyftboxManager, + client: SyftRDSClient, config: PeerApprovalConfig, state: Optional[StateManager] = None, on_approve: Optional[Callable[[str], None]] = None, diff --git a/packages/syft-bg/src/syft_bg/approve/monitors/job.py b/packages/syft-bg/src/syft_bg/approve/monitors/job.py index d0fbd8a1ecf..1a42a1476af 100644 --- a/packages/syft-bg/src/syft_bg/approve/monitors/job.py +++ b/packages/syft-bg/src/syft_bg/approve/monitors/job.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from syft_job.job import JobInfo - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds import SyftRDSClient class JobMonitor(Monitor): @@ -18,7 +18,7 @@ class JobMonitor(Monitor): def __init__( self, - client: SyftboxManager, + client: SyftRDSClient, config_path: Optional[Path] = None, state: Optional[StateManager] = None, on_approve: Optional[Callable[[JobInfo], None]] = None, diff --git a/packages/syft-bg/src/syft_bg/approve/monitors/peer.py b/packages/syft-bg/src/syft_bg/approve/monitors/peer.py index 752936268dc..cfd591c91ec 100644 --- a/packages/syft-bg/src/syft_bg/approve/monitors/peer.py +++ b/packages/syft-bg/src/syft_bg/approve/monitors/peer.py @@ -9,7 +9,7 @@ if TYPE_CHECKING: from syft_bg.approve.config import PeerApprovalConfig - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds import SyftRDSClient class PeerMonitor(Monitor): @@ -17,7 +17,7 @@ class PeerMonitor(Monitor): def __init__( self, - client: SyftboxManager, + client: SyftRDSClient, config: PeerApprovalConfig, state: Optional[StateManager] = None, on_approve: Optional[Callable[[str], None]] = None, diff --git a/packages/syft-bg/src/syft_bg/approve/orchestrator.py b/packages/syft-bg/src/syft_bg/approve/orchestrator.py index 20db094644d..cd1ece02802 100644 --- a/packages/syft-bg/src/syft_bg/approve/orchestrator.py +++ b/packages/syft-bg/src/syft_bg/approve/orchestrator.py @@ -12,7 +12,7 @@ from syft_bg.common.state import JsonStateManager if TYPE_CHECKING: - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds import SyftRDSClient class ApprovalOrchestrator(BaseOrchestrator): @@ -20,7 +20,7 @@ class ApprovalOrchestrator(BaseOrchestrator): def __init__( self, - client: SyftboxManager, + client: SyftRDSClient, config: AutoApproveConfig, config_path: Optional[Path] = None, ): @@ -40,10 +40,10 @@ def setup(self) -> None: @classmethod def from_client( cls, - client: SyftboxManager, + client: SyftRDSClient, interval: int = 5, ) -> ApprovalOrchestrator: - """Create orchestrator from a SyftboxManager client.""" + """Create orchestrator from a SyftRDSClient.""" if not client.has_do_role: raise ValueError( "ApprovalOrchestrator should only run on Data Owner (DO) side." @@ -67,31 +67,35 @@ def from_config( if not config.syftbox_root: raise ValueError("Config missing 'syftbox_root' field") - # Wait for sync to seed the cache before building SyftboxManager — - # SyftboxManager.from_config triggers _load_file_hashes_from_disk, - # which races sync's identical replay if both run cold-start. + # Wait for sync to seed the cache before building the RDS client — + # its nested SyftboxManager.from_config triggers + # _load_file_hashes_from_disk, which races sync's identical replay if + # both run cold-start. cls._wait_for_sync_ready(label="Approve") - from syft_client.sync.environments.environment import Environment - from syft_client.sync.syftbox_manager import SyftboxManager - from syft_client.sync.utils.syftbox_utils import check_env + from syft_rds import ( + Environment, + SyftRDSClient, + SyftRDSClientConfig, + check_env, + ) - env = check_env() - if env == Environment.COLAB: - client = SyftboxManager.for_colab( + if check_env() == Environment.COLAB: + rds_config = SyftRDSClientConfig.for_colab( email=config.do_email, has_do_role=True, skip_peer_on_patch_version_diff=config.skip_peer_on_patch_version_diff, force_ignore_peer_version=config.force_ignore_peer_version, ) else: - client = SyftboxManager.for_jupyter( + rds_config = SyftRDSClientConfig.for_jupyter( email=config.do_email, has_do_role=True, token_path=config.drive_token_path, skip_peer_on_patch_version_diff=config.skip_peer_on_patch_version_diff, force_ignore_peer_version=config.force_ignore_peer_version, ) + client = SyftRDSClient.from_config(rds_config) return cls(client=client, config=config) diff --git a/packages/syft-bg/src/syft_bg/cli/init/flow.py b/packages/syft-bg/src/syft_bg/cli/init/flow.py index 26f65792c03..253c49d048c 100644 --- a/packages/syft-bg/src/syft_bg/cli/init/flow.py +++ b/packages/syft-bg/src/syft_bg/cli/init/flow.py @@ -100,7 +100,7 @@ def _resolve_common_settings(config: UserPassedConfig, result: SyftBgConfig) -> "Data Owner email address", default=default_email or None ) - from syft_client.sync.syftbox_manager import get_jupyter_default_syftbox_folder + from syft_rds import get_jupyter_default_syftbox_folder default_syftbox = result.syftbox_root or str( get_jupyter_default_syftbox_folder(result.do_email) diff --git a/packages/syft-bg/src/syft_bg/common/syft_bg_config.py b/packages/syft-bg/src/syft_bg/common/syft_bg_config.py index 6647bf67a91..a0c2e1e2aa4 100644 --- a/packages/syft-bg/src/syft_bg/common/syft_bg_config.py +++ b/packages/syft-bg/src/syft_bg/common/syft_bg_config.py @@ -38,9 +38,7 @@ def _set_default_syftbox_root(cls, data: dict) -> dict: if not isinstance(data, dict): return data if not data.get("syftbox_root") and data.get("do_email"): - from syft_client.sync.syftbox_manager import ( - get_jupyter_default_syftbox_folder, - ) + from syft_rds import get_jupyter_default_syftbox_folder data = dict(data) data["syftbox_root"] = str( diff --git a/packages/syft-bg/src/syft_bg/notify/handlers/job.py b/packages/syft-bg/src/syft_bg/notify/handlers/job.py index 16182327573..a7358766b2b 100644 --- a/packages/syft-bg/src/syft_bg/notify/handlers/job.py +++ b/packages/syft-bg/src/syft_bg/notify/handlers/job.py @@ -5,7 +5,7 @@ from syft_bg.common.state import JsonStateManager from syft_bg.notify.gmail.sender import GmailSender -from syft_client.sync.utils.path_filters import is_normal_syncable_path +from syft_rds import is_normal_syncable_path if TYPE_CHECKING: from syft_job.client import JobClient diff --git a/packages/syft-bg/src/syft_bg/sync/orchestrator.py b/packages/syft-bg/src/syft_bg/sync/orchestrator.py index b2258c4d1e5..e4f4195ae03 100644 --- a/packages/syft-bg/src/syft_bg/sync/orchestrator.py +++ b/packages/syft-bg/src/syft_bg/sync/orchestrator.py @@ -13,7 +13,7 @@ from syft_bg.sync.snapshot import PeerVersionInfo, SyncSnapshot if TYPE_CHECKING: - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds import SyftRDSClient def sync_ready_path() -> Path: @@ -24,7 +24,7 @@ def sync_ready_path() -> Path: class SyncOrchestrator(BaseOrchestrator): def __init__( self, - client: SyftboxManager, + client: SyftRDSClient, state: JsonStateManager, config: SyncConfig, ): @@ -42,26 +42,29 @@ def from_config(cls, config: SyncConfig) -> SyncOrchestrator: if not config.syftbox_root: raise ValueError("SyncConfig missing 'syftbox_root'") - from syft_client.sync.environments.environment import Environment - from syft_client.sync.syftbox_manager import SyftboxManager - from syft_client.sync.utils.syftbox_utils import check_env + from syft_rds import ( + Environment, + SyftRDSClient, + SyftRDSClientConfig, + check_env, + ) - env = check_env() - if env == Environment.COLAB: - client = SyftboxManager.for_colab( + if check_env() == Environment.COLAB: + rds_config = SyftRDSClientConfig.for_colab( email=config.do_email, has_do_role=True, skip_peer_on_patch_version_diff=config.skip_peer_on_patch_version_diff, force_ignore_peer_version=config.force_ignore_peer_version, ) else: - client = SyftboxManager.for_jupyter( + rds_config = SyftRDSClientConfig.for_jupyter( email=config.do_email, has_do_role=True, token_path=config.drive_token_path, skip_peer_on_patch_version_diff=config.skip_peer_on_patch_version_diff, force_ignore_peer_version=config.force_ignore_peer_version, ) + client = SyftRDSClient.from_config(rds_config) state = JsonStateManager(state_file=config.sync_state_path) diff --git a/packages/syft-enclave/pyproject.toml b/packages/syft-enclave/pyproject.toml index 55a93237862..087eab59ad1 100644 --- a/packages/syft-enclave/pyproject.toml +++ b/packages/syft-enclave/pyproject.toml @@ -8,6 +8,7 @@ requires-python = ">=3.10" dependencies = [ "syft-client", + "syft-rds", "pydantic-settings>=2.11.0", "requests>=2.32.0", "google-auth[pyjwt]>=2.22.0", @@ -19,6 +20,7 @@ build-backend = "hatchling.build" [tool.uv.sources] "syft-client" = { workspace = true } +"syft-rds" = { workspace = true } [tool.hatch.build.targets.wheel] packages = ["src/syft_enclaves"] diff --git a/packages/syft-enclave/src/syft_enclaves/client.py b/packages/syft-enclave/src/syft_enclaves/client.py index b0064c87798..64528453411 100644 --- a/packages/syft-enclave/src/syft_enclaves/client.py +++ b/packages/syft-enclave/src/syft_enclaves/client.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import Optional -from syft_client.sync.syftbox_manager import SyftboxManager, SyftboxManagerConfig +from syft_rds import SyftRDSClient, SyftRDSClientConfig from syft_client.sync.version.peer_manager import CompatAction from syft_client.sync.peers.peer import Peer from syft_client.sync.peers.peer_list import PeerList @@ -35,31 +35,34 @@ class SyftEnclaveClient: def __init__( self, - manager: SyftboxManager, + rds: SyftRDSClient, data_owners: list[str] | None = None, ): - self._manager = manager + # The Remote Data Science product client. It OWNS the job + dataset + # surface (job_client / job_runner / dataset_manager) and composes the + # generic sync engine (a SyftboxManager) as ``rds.sync_engine``. + self._rds = rds # Data owners whose approval gates every job run on this enclave. # Fixed at launch/deploy time; stored in memory. self.data_owners = list(data_owners or []) @property def email(self) -> str: - return self._manager.email + return self._rds.email @property def syftbox_folder(self) -> Path: - return self._manager.syftbox_folder + return self._rds.syftbox_folder @property def peers(self) -> PeerList: - return self._manager.peers + return self._rds.peers def add_peer(self, peer_email: str, force: bool = False, verbose: bool = True): - self._manager.add_peer(peer_email, force=force, verbose=verbose) + self._rds.add_peer(peer_email, force=force, verbose=verbose) def load_peers(self): - self._manager.load_peers() + self._rds.load_peers() def approve_peer_request( self, @@ -67,12 +70,12 @@ def approve_peer_request( verbose: bool = True, peer_must_exist: bool = True, ): - self._manager.approve_peer_request( + self._rds.approve_peer_request( email_or_peer, verbose=verbose, peer_must_exist=peer_must_exist ) def reject_peer_request(self, email_or_peer: str | Peer): - self._manager.reject_peer_request(email_or_peer) + self._rds.reject_peer_request(email_or_peer) def attest_peer(self, peer_email: str): """Verify an enclave peer's attestation by re-reading SYFT_version.json @@ -80,10 +83,8 @@ def attest_peer(self, peer_email: str): raises AttestationError only when verification of an existing token fails.""" from syft_enclaves.attestation import verify_attestation_token - version_info = ( - self._manager.peer_manager.connection_router.read_peer_version_file( - peer_email - ) + version_info = self._rds.peer_manager.connection_router.read_peer_version_file( + peer_email ) if version_info is None: print( @@ -99,29 +100,29 @@ def attest_peer(self, peer_email: str): return verify_attestation_token(version_info.attestation_token) def sync(self): - self._manager.sync() + self._rds.sync() def delete_syftbox( self, verbose: bool = True, broadcast_delete_events: bool = True ): """Delete all SyftBox state (Drive files + local caches/folder).""" - self._manager.delete_syftbox( + self._rds.delete_syftbox( verbose=verbose, broadcast_delete_events=broadcast_delete_events ) def create_dataset(self, *args, **kwargs): - return self._manager.create_dataset(*args, **kwargs) + return self._rds.create_dataset(*args, **kwargs) def share_private_dataset(self, tag: str, enclave_email: str): - self._manager.share_private_dataset(tag, enclave_email) + self._rds.share_private_dataset(tag, enclave_email) @property def datasets(self) -> SyftDatasetManager: - return self._manager.dataset_manager + return self._rds.dataset_manager @property def jobs(self) -> JobsList: - jobs_list = self._manager.job_client.jobs + jobs_list = self._rds.job_client.jobs wrapped = [ EnclaveJobInfo.from_job_info(j) if j.job_headers.get("job_type") == "enclave" @@ -149,14 +150,14 @@ def submit_python_job( otherwise surfaces much later as a stuck, never-distributed job). """ if not force_submission: - result = self._manager.peer_manager.get_peer_compatibility_status( + result = self._rds.peer_manager.get_peer_compatibility_status( enclave_email, action=CompatAction.SUBMIT, ignore_peer_version=ignore_peer_version, ) result.raise_on_skip(operation="submit job") result.maybe_warn() - job_dir = self._manager.job_client.submit_python_job( + job_dir = self._rds.job_client.submit_python_job( enclave_email, code_path, job_name, @@ -164,7 +165,7 @@ def submit_python_job( share_results_with_do=share_results_with_do, **kwargs, ) - self._manager.push_job_files(job_dir) + self._rds.sync_engine.push_job_files(job_dir) def run_jobs(self) -> None: """Run approved enclave jobs.""" @@ -178,7 +179,7 @@ def run_jobs(self) -> None: state.status = JobStatus.APPROVED state.save(job.job_review_path / "state.yaml") - self._manager.process_approved_jobs( + self._rds.process_approved_jobs( force_execution=True, share_outputs_with_submitter=True, share_logs_with_submitter=True, @@ -206,14 +207,14 @@ def distribute_results(self) -> None: results_shared_marker.write_text("shared") - self._manager.sync() + self._rds.sync() def _read_state_file(self, job: JobInfo) -> dict[Path, bytes]: """Read the job state.yaml as a {path_in_datasite: bytes} dict.""" state_file = job.job_review_path / "state.yaml" if not state_file.exists(): return {} - datasite_dir = self._manager.syftbox_folder / self._manager.email + datasite_dir = self._rds.syftbox_folder / self._rds.email state_rel = state_file.relative_to(datasite_dir) return {state_rel: state_file.read_bytes()} @@ -226,24 +227,23 @@ def _forward_results_to_recipients(self, job: JobInfo, recipients: list[str]): files_by_datasite_path.update(self._read_state_file(job)) if not files_by_datasite_path: return - events_message = ( - self._manager.datasite_owner_syncer.event_cache.create_events_for_files( - files_by_datasite_path - ) + syncer = self._rds.sync_engine.datasite_owner_syncer + events_message = syncer.event_cache.create_events_for_files( + files_by_datasite_path ) - self._manager.datasite_owner_syncer.queue_event_for_syftbox( + syncer.queue_event_for_syftbox( recipients=recipients, file_change_events_message=events_message, ) - self._manager.datasite_owner_syncer.process_syftbox_events_queue() + syncer.process_syftbox_events_queue() def approve_job(self, job: JobInfo) -> None: """Approve an enclave job and push the approval state file to the enclave.""" job.approve() file_name = enclave_approval_file_name(self.email) approval_file = job.job_review_path / file_name - relative_path = approval_file.relative_to(self._manager.syftbox_folder) - self._manager.datasite_watcher_syncer.on_file_change( + relative_path = approval_file.relative_to(self._rds.syftbox_folder) + self._rds.sync_engine.datasite_watcher_syncer.on_file_change( relative_path, process_now=True ) @@ -255,10 +255,8 @@ def receive_jobs(self): 3. Creates JobState with PartyApprovalStatus per DO 4. Sets permissions and marks as distributed """ - self._manager.job_client.scan_inbox() - inbox_dir = self._manager.job_client.config.get_all_submissions_dir( - self._manager.email - ) + self._rds.job_client.scan_inbox() + inbox_dir = self._rds.job_client.config.get_all_submissions_dir(self._rds.email) if not inbox_dir.exists(): return @@ -280,8 +278,8 @@ def _try_distribute_job(self, ds_email: str, job_dir: Path): if config.job_type != "enclave" or not config.datasets: return - review_dir = self._manager.job_client.config.get_review_job_dir( - self._manager.email, ds_email, job_dir.name + review_dir = self._rds.job_client.config.get_review_job_dir( + self._rds.email, ds_email, job_dir.name ) distributed_marker = review_dir / "distributed" if distributed_marker.exists(): @@ -306,39 +304,37 @@ def _try_distribute_job(self, ds_email: str, job_dir: Path): def _forward_job_to_dos(self, job_dir: Path, do_emails: list[str]): """Forward job files to DOs via the event-based outbox mechanism.""" files_by_datasite_path = self._get_files_in_dir(job_dir) - events_message = ( - self._manager.datasite_owner_syncer.event_cache.create_events_for_files( - files_by_datasite_path - ) + syncer = self._rds.sync_engine.datasite_owner_syncer + events_message = syncer.event_cache.create_events_for_files( + files_by_datasite_path ) - self._manager.datasite_owner_syncer.queue_event_for_syftbox( + syncer.queue_event_for_syftbox( recipients=do_emails, file_change_events_message=events_message, ) - self._manager.datasite_owner_syncer.process_syftbox_events_queue() + syncer.process_syftbox_events_queue() def _forward_approval_files_to_dos(self, review_dir: Path, do_emails: list[str]): """Forward each DO's approval state file to them individually.""" - datasite_dir = self._manager.syftbox_folder / self._manager.email + datasite_dir = self._rds.syftbox_folder / self._rds.email + syncer = self._rds.sync_engine.datasite_owner_syncer for do_email in do_emails: file_name = enclave_approval_file_name(do_email) approval_file = review_dir / file_name path_in_datasite = approval_file.relative_to(datasite_dir) files_by_datasite_path = {path_in_datasite: approval_file.read_bytes()} - events_message = ( - self._manager.datasite_owner_syncer.event_cache.create_events_for_files( - files_by_datasite_path - ) + events_message = syncer.event_cache.create_events_for_files( + files_by_datasite_path ) - self._manager.datasite_owner_syncer.queue_event_for_syftbox( + syncer.queue_event_for_syftbox( recipients=[do_email], file_change_events_message=events_message, ) - self._manager.datasite_owner_syncer.process_syftbox_events_queue() + syncer.process_syftbox_events_queue() def _get_files_in_dir(self, directory: Path) -> dict[Path, bytes]: """Read all files under directory, keyed by path relative to the datasite root.""" - datasite_dir = self._manager.syftbox_folder / self._manager.email + datasite_dir = self._rds.syftbox_folder / self._rds.email files_by_datasite_path = {} for f in directory.rglob("*"): if not f.is_file(): @@ -376,14 +372,14 @@ def _set_job_permissions( ): """Grant inbox read to everyone who needs to see the job (referenced + approving DOs), and approval-file write to the approving DOs.""" - datasite = self._manager.syftbox_folder / self._manager.email + datasite = self._rds.syftbox_folder / self._rds.email ctx = SyftPermContext(datasite=datasite) inbox_rel = job_dir.relative_to(datasite) ds_email = job_dir.parent.name job_name = job_dir.name - review_dir = self._manager.job_client.config.get_review_job_dir( - self._manager.email, ds_email, job_name + review_dir = self._rds.job_client.config.get_review_job_dir( + self._rds.email, ds_email, job_name ) review_rel = review_dir.relative_to(datasite) @@ -397,25 +393,32 @@ def _set_job_permissions( @classmethod def from_config( cls, - config: SyftboxManagerConfig, + config: SyftRDSClientConfig, data_owners: list[str] | None = None, ) -> "SyftEnclaveClient": - """Build a SyftEnclaveClient from a manager config with a wrapped job_client. + """Build a SyftEnclaveClient from an RDS config with a wrapped job_client. + + The enclave client composes a ``SyftRDSClient`` (the RDS product client), + which OWNS the job + dataset surface and holds the generic sync engine at + ``rds.sync_engine``. We wrap the RDS-owned job_client with + ``EnclaveJobClient`` (settable on the RDS client) and install the private + dataset immutability filter on the nested sync engine's watcher cache. - Encryption rides along on ``config`` via + Encryption rides along on ``config.sync`` via ``peer_manager_config.use_encryption``; ``SyftboxManager.from_config`` - loads/generates this datasite's keys from its per-datasite key file, so no - extra step is needed. + (invoked inside ``SyftRDSClient.from_config``) loads/generates this + datasite's keys from its per-datasite key file, so no extra step is needed. """ - manager = SyftboxManager.from_config(config) - manager.job_client = EnclaveJobClient(manager.job_client) + rds = SyftRDSClient.from_config(config) + rds.job_client = EnclaveJobClient(rds.job_client) - if manager.datasite_watcher_syncer: - manager.datasite_watcher_syncer.datasite_watcher_cache.pre_write_filter = ( - make_private_dataset_immutability_filter(manager.syftbox_folder) + sync_engine = rds.sync_engine + if sync_engine.datasite_watcher_syncer: + sync_engine.datasite_watcher_syncer.datasite_watcher_cache.pre_write_filter = make_private_dataset_immutability_filter( + sync_engine.syftbox_folder ) - return cls(manager, data_owners=data_owners) + return cls(rds, data_owners=data_owners) @classmethod def for_enclave( @@ -432,7 +435,7 @@ def for_enclave( data_owners: Emails whose approval gates every job on this enclave. encryption: Enable end-to-end drive encryption. """ - config = SyftboxManagerConfig.for_jupyter( + config = SyftRDSClientConfig.for_jupyter( email=email, has_ds_role=True, has_do_role=True, @@ -479,7 +482,12 @@ def quad_with_mock_drive_service_connection( clients = create_clients(configs) enclave, do1, do2, ds = clients enclave.data_owners = [do1.email, do2.email] - managers = tuple(c._manager for c in clients) + # The setup helpers manipulate the generic sync engine directly + # (connections, file-watcher callbacks, version files, encryption, + # peering). Each client's sync engine lives at ``rds.sync_engine``; the + # RDS-owned job/dataset managers react via the peer-lifecycle callbacks + # registered on that same engine during SyftRDSClient.__init__. + managers = tuple(c._rds.sync_engine for c in clients) setup_connections(managers) setup_callbacks(managers) write_versions(managers) diff --git a/packages/syft-enclave/src/syft_enclaves/login.py b/packages/syft-enclave/src/syft_enclaves/login.py index 1018dd35505..6c2847bc8fc 100644 --- a/packages/syft-enclave/src/syft_enclaves/login.py +++ b/packages/syft-enclave/src/syft_enclaves/login.py @@ -5,8 +5,8 @@ from syft_client.sync.environments.environment import Environment from syft_client.sync.login import _init_client_login, _resolve_login_params from syft_client.sync.login_utils import handle_potential_version_mismatches_on_login -from syft_client.sync.syftbox_manager import SyftboxManagerConfig from syft_client.sync.utils.syftbox_utils import check_env +from syft_rds import SyftRDSClientConfig from syft_enclaves.client import SyftEnclaveClient @@ -38,7 +38,7 @@ def _login( handle_potential_version_mismatches_on_login(email, token_path) if env == Environment.COLAB: - config = SyftboxManagerConfig.for_colab( + config = SyftRDSClientConfig.for_colab( email=email, has_do_role=has_do_role, has_ds_role=has_ds_role, @@ -47,7 +47,7 @@ def _login( skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, ) else: - config = SyftboxManagerConfig.for_jupyter( + config = SyftRDSClientConfig.for_jupyter( email=email, has_do_role=has_do_role, has_ds_role=has_ds_role, @@ -62,8 +62,9 @@ def _login( client = SyftEnclaveClient.from_config(config) # Reuses syft-client's login init: verifies the token authenticates as - # `email`, writes the local version, then syncs / loads peers. - _init_client_login(client._manager, sync=sync, load_peers=load_peers) + # `email`, writes the local version, then syncs / loads peers. Operates on + # the generic sync engine nested inside the RDS client. + _init_client_login(client._rds.sync_engine, sync=sync, load_peers=load_peers) return client diff --git a/packages/syft-enclave/src/syft_enclaves/runner.py b/packages/syft-enclave/src/syft_enclaves/runner.py index ffbe33ed713..96a071c7f9a 100644 --- a/packages/syft-enclave/src/syft_enclaves/runner.py +++ b/packages/syft-enclave/src/syft_enclaves/runner.py @@ -112,7 +112,7 @@ def _on_initializing(self) -> None: "fresh_state=true — wiping ALL SyftBox state " "(local folder + Google Drive files) before init" ) - self.client._manager.delete_syftbox() + self.client.delete_syftbox() logger.info("State wipe complete — enclave starts with a clean slate") def _on_attesting(self) -> None: @@ -133,8 +133,9 @@ def _publish_attestation(self) -> None: """Fetch attestation JWT from the TEE and write it into the version file.""" eat_nonce = build_eat_nonce() token = fetch_attestation_token(eat_nonce=eat_nonce) - self.client._manager.peer_manager.get_own_version().attestation_token = token - self.client._manager.peer_manager.write_own_version() + peer_manager = self.client._rds.peer_manager + peer_manager.get_own_version().attestation_token = token + peer_manager.write_own_version() logger.info("Attestation token published to SYFT_version.json") def _on_peering(self) -> None: diff --git a/packages/syft-enclave/src/syft_enclaves/utils.py b/packages/syft-enclave/src/syft_enclaves/utils.py index 393df737263..8ccabeef540 100644 --- a/packages/syft-enclave/src/syft_enclaves/utils.py +++ b/packages/syft-enclave/src/syft_enclaves/utils.py @@ -1,4 +1,4 @@ -from syft_client.sync.syftbox_manager import SyftboxManagerConfig +from syft_rds import SyftRDSClientConfig from syft_client.sync.connections.drive.mock_drive_service import ( MockDriveBackingStore, MockDriveService, @@ -9,25 +9,25 @@ def create_configs( enclave_email, do1_email, do2_email, ds_email, use_in_memory_cache ) -> tuple: - enclave_config = SyftboxManagerConfig._base_config_for_testing( + enclave_config = SyftRDSClientConfig._base_config_for_testing( email=enclave_email, has_do_role=True, has_ds_role=True, use_in_memory_cache=use_in_memory_cache, ) - do1_config = SyftboxManagerConfig._base_config_for_testing( + do1_config = SyftRDSClientConfig._base_config_for_testing( email=do1_email, has_do_role=True, has_ds_role=True, use_in_memory_cache=use_in_memory_cache, ) - do2_config = SyftboxManagerConfig._base_config_for_testing( + do2_config = SyftRDSClientConfig._base_config_for_testing( email=do2_email, has_do_role=True, has_ds_role=True, use_in_memory_cache=use_in_memory_cache, ) - ds_config = SyftboxManagerConfig._base_config_for_testing( + ds_config = SyftRDSClientConfig._base_config_for_testing( email=ds_email, has_ds_role=True, use_in_memory_cache=use_in_memory_cache, diff --git a/packages/syft-enclave/tests/test_enclave_client.py b/packages/syft-enclave/tests/test_enclave_client.py index 4ad5c7743ec..909630906df 100644 --- a/packages/syft-enclave/tests/test_enclave_client.py +++ b/packages/syft-enclave/tests/test_enclave_client.py @@ -15,21 +15,21 @@ def test_quad_initialization(): assert all(isinstance(c, SyftEnclaveClient) for c in clients) # Correct roles - assert enclave._manager.has_do_role is True - assert enclave._manager.has_ds_role is True + assert enclave._rds.has_do_role is True + assert enclave._rds.has_ds_role is True - assert do1._manager.has_do_role is True - assert do1._manager.has_ds_role is True + assert do1._rds.has_do_role is True + assert do1._rds.has_ds_role is True - assert do2._manager.has_do_role is True - assert do2._manager.has_ds_role is True + assert do2._rds.has_do_role is True + assert do2._rds.has_ds_role is True - assert ds._manager.has_do_role is False - assert ds._manager.has_ds_role is True + assert ds._rds.has_do_role is False + assert ds._rds.has_ds_role is True - # Helper to get approved peer emails for a manager + # Helper to get approved peer emails for a client def approved_emails(client): - return {p.email for p in client._manager.peer_manager.approved_peers} + return {p.email for p in client._rds.peer_manager.approved_peers} # Enclave (DO-only): approved DS, DO1, DO2 assert approved_emails(enclave) == {ds.email, do1.email, do2.email} diff --git a/packages/syft-enclave/tests/test_enclave_datasets.py b/packages/syft-enclave/tests/test_enclave_datasets.py index 77ef5c992ca..7fdae2d83ff 100644 --- a/packages/syft-enclave/tests/test_enclave_datasets.py +++ b/packages/syft-enclave/tests/test_enclave_datasets.py @@ -38,7 +38,7 @@ def test_share_private_dataset_with_enclave(): # DO1 + DS sync -> DS can see mock data do1.sync() - ds._manager.sync() + ds._rds.sync() ds_datasets = ds.datasets.get_all() assert len(ds_datasets) == 1 assert ds_datasets[0].name == "testdataset" @@ -51,11 +51,7 @@ def test_share_private_dataset_with_enclave(): assert mock_content == "Hello, world!" non_existing_ds_private_dir = ( - ds._manager.syftbox_folder - / do1.email - / "private" - / "syft_datasets" - / "testdataset" + ds._rds.syftbox_folder / do1.email / "private" / "syft_datasets" / "testdataset" ) assert not non_existing_ds_private_dir.exists() @@ -63,12 +59,12 @@ def test_share_private_dataset_with_enclave(): do1.share_private_dataset("testdataset", enclave.email) # Enclave syncs (as DS) and pulls the private data events from DO1's outbox - enclave._manager.sync() + enclave._rds.sync() # Enclave can see the dataset via mock data (shared with DS and enclave shares peers) # But more importantly, enclave can access private files via shared_private_dir enclave_private_dir = ( - enclave._manager.syftbox_folder + enclave._rds.syftbox_folder / do1.email / "private" / "syft_datasets" diff --git a/packages/syft-enclave/tests/test_encryption.py b/packages/syft-enclave/tests/test_encryption.py index 056adab7816..226d78cf039 100644 --- a/packages/syft-enclave/tests/test_encryption.py +++ b/packages/syft-enclave/tests/test_encryption.py @@ -34,21 +34,21 @@ def test_encrypted_quad_keys_and_bundles(): ) for client in (enclave, do1, do2, ds): - store = client._manager._peer_store + store = client._rds.sync_engine._peer_store assert store is not None assert store.use_encryption is True assert store.has_my_keys() is True # The enclave peers with all three; it should hold each of their bundles # after wiring, so it can encrypt to and verify from them. - enclave_store = enclave._manager._peer_store + enclave_store = enclave._rds.sync_engine._peer_store for peer_email in (do1.email, do2.email, ds.email): assert enclave_store.has_peer_bundle(peer_email), ( f"enclave missing encryption bundle for {peer_email}" ) # And the DS holds the enclave's bundle (the link we care most about). - assert ds._manager._peer_store.has_peer_bundle(enclave.email) + assert ds._rds.sync_engine._peer_store.has_peer_bundle(enclave.email) def test_encrypted_message_to_enclave_roundtrips(): diff --git a/packages/syft-enclave/tests/test_immutability.py b/packages/syft-enclave/tests/test_immutability.py index 15ea205ada1..7057f965511 100644 --- a/packages/syft-enclave/tests/test_immutability.py +++ b/packages/syft-enclave/tests/test_immutability.py @@ -122,10 +122,10 @@ def test_enclave_blocks_reshare_of_private_dataset(): # First share + sync — enclave receives private data do1.share_private_dataset("testdataset", enclave.email) - enclave._manager.sync() + enclave._rds.sync() enclave_private_dir = ( - enclave._manager.syftbox_folder + enclave._rds.syftbox_folder / do1.email / "private" / "syft_datasets" @@ -137,7 +137,7 @@ def test_enclave_blocks_reshare_of_private_dataset(): # DO modifies private data locally and re-shares private_path.write_text("TAMPERED DATA") - do1._manager.dataset_manager.delete("testdataset", require_confirmation=False) + do1._rds.dataset_manager.delete("testdataset", require_confirmation=False) do1.create_dataset( name="testdataset", mock_path=mock_path, @@ -148,7 +148,7 @@ def test_enclave_blocks_reshare_of_private_dataset(): sync=False, ) do1.share_private_dataset("testdataset", enclave.email) - enclave._manager.sync() + enclave._rds.sync() # Enclave still has original content — overwrite was blocked assert (enclave_private_dir / "private.txt").read_bytes() == original_content diff --git a/packages/syft-enclave/tests/test_runner.py b/packages/syft-enclave/tests/test_runner.py index c51a9c1c48d..c9ce52edca8 100644 --- a/packages/syft-enclave/tests/test_runner.py +++ b/packages/syft-enclave/tests/test_runner.py @@ -26,7 +26,7 @@ def test_fresh_state_true_invokes_delete_syftbox(tmp_path, monkeypatch): runner = EnclaveRunner(client=client, fresh_state=True) runner.init() - client._manager.delete_syftbox.assert_called_once_with() + client.delete_syftbox.assert_called_once_with() def test_fresh_state_false_skips_delete_syftbox(tmp_path, monkeypatch): @@ -39,7 +39,7 @@ def test_fresh_state_false_skips_delete_syftbox(tmp_path, monkeypatch): runner = EnclaveRunner(client=client, fresh_state=False) runner.init() - client._manager.delete_syftbox.assert_not_called() + client.delete_syftbox.assert_not_called() def test_fresh_state_default_is_true(tmp_path, monkeypatch): @@ -52,7 +52,7 @@ def test_fresh_state_default_is_true(tmp_path, monkeypatch): runner = EnclaveRunner(client=client) # no fresh_state arg assert runner.fresh_state is True runner.init() - client._manager.delete_syftbox.assert_called_once_with() + client.delete_syftbox.assert_called_once_with() def test_fresh_state_uses_default_kwargs_on_delete(tmp_path, monkeypatch): @@ -66,6 +66,6 @@ def test_fresh_state_uses_default_kwargs_on_delete(tmp_path, monkeypatch): # Must be called with no positional or keyword args — let the method's # own defaults handle broadcast_delete_events and verbose. - call = client._manager.delete_syftbox.call_args + call = client.delete_syftbox.call_args assert call.args == () assert call.kwargs == {} diff --git a/packages/syft-rds/pyproject.toml b/packages/syft-rds/pyproject.toml new file mode 100644 index 00000000000..57e82968466 --- /dev/null +++ b/packages/syft-rds/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "syft-rds" +version = "0.1.0" +description = "Remote Data Science product: datasets + jobs composed on top of the syft-client sync engine" +authors = [{ name = "OpenMined", email = "info@openmined.org" }] +license = { text = "Apache-2.0" } +requires-python = ">=3.10" + +dependencies = [ + "syft-client", + "syft-job", + "syft-dataset", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.uv.sources] +"syft-client" = { workspace = true } +"syft-job" = { workspace = true } +"syft-dataset" = { workspace = true } + +[tool.hatch.build.targets.wheel] +packages = ["src/syft_rds"] diff --git a/packages/syft-rds/src/syft_rds/__init__.py b/packages/syft-rds/src/syft_rds/__init__.py new file mode 100644 index 00000000000..bd434a50625 --- /dev/null +++ b/packages/syft-rds/src/syft_rds/__init__.py @@ -0,0 +1,27 @@ +"""syft-rds: Remote Data Science product composed on top of syft-client.""" + +from syft_rds.client import SyftRDSClient +from syft_rds.config import SyftRDSClientConfig +from syft_rds.job_auto_approval import auto_approve_and_run_jobs, job_matches_criteria +from syft_rds.login import login_do, login_ds + +# Generic sync-stack helpers re-exported so consumers that compose the RDS product +# (e.g. syft-bg) have a single integration surface and need not import from +# syft_client.sync internals directly. +from syft_client.sync.syftbox_manager import get_jupyter_default_syftbox_folder +from syft_client.sync.utils.path_filters import is_normal_syncable_path +from syft_client.sync.environments.environment import Environment +from syft_client.sync.utils.syftbox_utils import check_env + +__all__ = [ + "SyftRDSClient", + "SyftRDSClientConfig", + "login_do", + "login_ds", + "auto_approve_and_run_jobs", + "job_matches_criteria", + "get_jupyter_default_syftbox_folder", + "is_normal_syncable_path", + "Environment", + "check_env", +] diff --git a/packages/syft-rds/src/syft_rds/client.py b/packages/syft-rds/src/syft_rds/client.py new file mode 100644 index 00000000000..6d152f0e127 --- /dev/null +++ b/packages/syft-rds/src/syft_rds/client.py @@ -0,0 +1,702 @@ +"""The Remote Data Science client. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from syft_client.sync.syftbox_manager import SyftboxManager +from syft_datasets.dataset_manager import ( + DATASET_COLLECTION_PREFIX, + PRIVATE_DATASET_COLLECTION_PREFIX, +) +from syft_client.sync.utils.pre_submit_scan import run_pre_submit_check +from syft_client.sync.version.peer_manager import CompatAction +from syft_job.client import JobClient +from syft_job.job_runner import SyftJobRunner +from syft_datasets.dataset_manager import SyftDatasetManager +from syft_job import SyftJobConfig +from syft_datasets.config import SyftBoxConfig +from syft_rds.config import DATASET_COLLECTION_SPECS + +if TYPE_CHECKING: + from syft_rds.config import SyftRDSClientConfig + +logger = logging.getLogger(__name__) + + +class SyftRDSClient: + def __init__(self, sync_engine, job_client, job_runner, dataset_manager): + # The nested generic sync engine (composition). + self._sync = sync_engine + # RDS-owned domain managers. + self._job_client = job_client + self._job_runner = job_runner + self._dataset_manager = dataset_manager + # React to sync-engine peer lifecycle events with RDS-owned logic. + self._sync.on("peer_approved", self._on_peer_approved) + self._sync.on("peers_loaded", self._on_peers_loaded) + # React to collection restores so RDS can run dataset-specific post-processing + # (the private dataset's data_dir fixup) without the sync core knowing datasets. + if self._sync.datasite_owner_syncer is not None: + self._sync.datasite_owner_syncer.on( + "collection_restored", self._on_collection_restored + ) + + @classmethod + def from_config(cls, config: "SyftRDSClientConfig") -> "SyftRDSClient": + + sync_engine = SyftboxManager.from_config(config.sync) + job_client = JobClient.from_config(config.job) + job_runner = ( + SyftJobRunner.from_config(config.job) if config.sync.has_do_role else None + ) + dataset_manager = SyftDatasetManager.from_config(config.dataset) + return cls(sync_engine, job_client, job_runner, dataset_manager) + + @classmethod + def _build_rds_pair_from_managers(cls, ds_mgr, do_mgr): + """Wrap a paired ``(ds, do)`` ``SyftboxManager`` tuple into RDS clients. + + Peers were already approved/loaded during pairing (before our callbacks + were registered), so we replay the DO-side setup so its owned + ``JobClient`` has the DS job folders + install sources. + """ + + def _build(mgr) -> "SyftRDSClient": + job_cfg = SyftJobConfig( + syftbox_folder=Path(mgr.syftbox_folder), + current_user_email=mgr.email, + has_do_role=mgr.has_do_role, + ) + job_client = JobClient.from_config(job_cfg) + job_runner = ( + SyftJobRunner.from_config(job_cfg) if mgr.has_do_role else None + ) + dataset_manager = SyftDatasetManager.from_config( + SyftBoxConfig(syftbox_folder=mgr.syftbox_folder, email=mgr.email) + ) + return cls(mgr, job_client, job_runner, dataset_manager) + + ds_rds = _build(ds_mgr) + do_rds = _build(do_mgr) + do_rds._on_peers_loaded() + for peer in do_mgr.peer_manager.approved_peers: + do_rds._on_peer_approved(peer.email) + ds_rds._on_peers_loaded() + return ds_rds, do_rds + + @classmethod + def pair_with_mock_drive_service_connection(cls, **kwargs): + """(ds, do) pair of self-contained RDS clients sharing one mock Drive. + """ + ds_mgr, do_mgr = SyftboxManager.pair_with_mock_drive_service_connection( + collection_specs=DATASET_COLLECTION_SPECS, **kwargs + ) + return cls._build_rds_pair_from_managers(ds_mgr, do_mgr) + + @classmethod + def _pair_with_google_drive_testing_connection(cls, **kwargs): + """(ds, do) pair of self-contained RDS clients sharing a REAL Google Drive. + """ + ds_mgr, do_mgr = SyftboxManager._pair_with_google_drive_testing_connection( + collection_specs=DATASET_COLLECTION_SPECS, **kwargs + ) + return cls._build_rds_pair_from_managers(ds_mgr, do_mgr) + + @property + def sync_engine(self) -> SyftboxManager: + return self._sync + + # RDS-owned domain managers, exposed so consumers (e.g. syft-enclave, which + # wraps the job client) can read/replace them. + @property + def job_client(self) -> Any: + return self._job_client + + @job_client.setter + def job_client(self, value: Any) -> None: + self._job_client = value + + @property + def job_runner(self) -> Any: + return self._job_runner + + @job_runner.setter + def job_runner(self, value: Any) -> None: + self._job_runner = value + + @property + def dataset_manager(self) -> Any: + return self._dataset_manager + + # ------------------------------------------------------------------ # + # delegated identity + sync surface (owned by the generic core) + # ------------------------------------------------------------------ # + @property + def email(self) -> str: + return self._sync.email + + @property + def syftbox_folder(self) -> Path: + return self._sync.syftbox_folder + + @property + def has_do_role(self) -> bool: + return self._sync.has_do_role + + @property + def has_ds_role(self) -> bool: + return self._sync.has_ds_role + + @property + def peer_manager(self) -> Any: + return self._sync.peer_manager + + @property + def peers(self) -> Any: + """Combined peer list (approved + requests). Delegates to the sync engine. + + Auto-syncs first when ``PRE_SYNC`` is enabled (the engine's behavior). + """ + return self._sync.peers + + def sync(self, *args: Any, **kwargs: Any) -> Any: + return self._sync.sync(*args, **kwargs) + + def add_peer(self, *args: Any, **kwargs: Any) -> Any: + return self._sync.add_peer(*args, **kwargs) + + def approve_peer_request(self, *args: Any, **kwargs: Any) -> Any: + return self._sync.approve_peer_request(*args, **kwargs) + + def reject_peer_request(self, *args: Any, **kwargs: Any) -> Any: + return self._sync.reject_peer_request(*args, **kwargs) + + def load_peers(self, *args: Any, **kwargs: Any) -> Any: + return self._sync.load_peers(*args, **kwargs) + + def delete_syftbox(self, *args: Any, **kwargs: Any) -> Any: + return self._sync.delete_syftbox(*args, **kwargs) + + # ------------------------------------------------------------------ # + # peer lifecycle callbacks (RDS-owned reactions to sync events) + # ------------------------------------------------------------------ # + def _on_peer_approved(self, peer_email: str) -> None: + """A peer was approved: set up their DS job folder and share datasets.""" + if self.has_do_role: + self._job_client.setup_ds_job_folder_as_do(peer_email) + self._share_any_datasets_with_peer(peer_email) + + def _on_collection_restored( + self, prefix: str, tag: str, local_dir: Any + ) -> None: + """A collection was restored from the backend by the owner-syncer. + + For the PRIVATE dataset collection, rewrite ``private_metadata.yaml``'s + machine-specific ``data_dir`` to the current local path, so job execution + can locate the real data after a restore onto a new machine/path. This is + dataset domain knowledge, so it lives here (rds) rather than the sync core. + """ + from syft_datasets.dataset_manager import PRIVATE_DATASET_COLLECTION_PREFIX + + if prefix != PRIVATE_DATASET_COLLECTION_PREFIX: + return + + import yaml + + metadata_path = Path(local_dir) / "private_metadata.yaml" + if not metadata_path.exists(): + return + data = yaml.safe_load(metadata_path.read_text()) + expected_dir = str(local_dir) + if data.get("data_dir") != expected_dir: + data["data_dir"] = expected_dir + metadata_path.write_text(yaml.safe_dump(data, indent=2, sort_keys=False)) + + def _on_peers_loaded(self, *args: Any, **kwargs: Any) -> None: + """Peers were (re)loaded: copy each peer's advertised install source + into our owned job client so submitted run.sh references the DO's path.""" + for peer in self._sync.peer_manager.peer_store.syncable_peers: + if peer.version: + self._job_client.peer_install_sources[peer.email] = ( + peer.version.syft_client_install_source + ) + + def _share_any_datasets_with_peer(self, peer_email: str) -> None: + """Share all datasets tagged 'any' with a specific peer. + + Google Drive "anyone with link" files are not discoverable via search, + so explicit user sharing is added. Reads the cache populated during + ``pull_initial_state()`` in the nested DatasiteOwnerSyncer. + """ + for tag, content_hash in self._sync.datasite_owner_syncer._any_shared_collections: + try: + self._sync._connection_router.owner_share_collection( + DATASET_COLLECTION_PREFIX, tag, content_hash, [peer_email] + ) + except Exception: + # Ignore errors (e.g., already shared) + pass + + # ------------------------------------------------------------------ # + # job product surface (RDS-owned) + # ------------------------------------------------------------------ # + def submit_bash_job( + self, + user: str, + script: str, + job_name: str = "", + sync=True, + force_submission: bool = False, + ignore_peer_version: bool = False, + ): + # Check version compatibility before submission (uses cached versions) + if not force_submission: + result = self._sync.peer_manager.get_peer_compatibility_status( + user, + action=CompatAction.SUBMIT, + ignore_peer_version=ignore_peer_version, + ) + result.raise_on_skip(operation="submit job") + result.maybe_warn() + job_dir = self._job_client.submit_bash_job(user, script, job_name=job_name) + self._sync.push_job_files(job_dir) + + def submit_python_job( + self, + user: str, + code_path: str, + job_name: str | None = "", + dependencies: list[str] | None = None, + entrypoint: str | None = None, + sync=True, + force_submission: bool = False, + ignore_peer_version: bool = False, + ): + peer_emails = {p.email for p in self._sync.peer_manager.syncable_peers} + if user not in peer_emails: + print(f"⚠️ {user} is not in your peer list.") + print(f" Add them first with: client.add_peer('{user}')") + return + + if not force_submission: + if not run_pre_submit_check(Path(code_path)): + print("Submission aborted.") + return + + print(f"📤 Submitting '{code_path}' to {user}...") + if job_name: + print(f" Job name : {job_name}") + if dependencies: + print(f" Dependencies : {', '.join(dependencies)}") + + # Check version compatibility before submission (uses cached versions) + if not force_submission: + result = self._sync.peer_manager.get_peer_compatibility_status( + user, + action=CompatAction.SUBMIT, + ignore_peer_version=ignore_peer_version, + ) + result.raise_on_skip(operation="submit job") + result.maybe_warn() + job_dir = self._job_client.submit_python_job( + user, + code_path, + job_name=job_name, + dependencies=dependencies, + entrypoint=entrypoint, + ) + self._sync.push_job_files(job_dir) + + print("\n✅ Job submitted successfully!") + print(" Status : inbox (waiting for DO to review)") + print(f"\n⏳ Next step: wait for {user} to approve and run it.") + print(" Check progress with: client.jobs") + + @property + def jobs(self) -> Any: + """List of jobs. Auto-syncs first unless PRE_SYNC=false.""" + if os.environ.get("PRE_SYNC", "true").lower() == "true": + self._sync.sync() + return self._job_client.jobs + + def process_approved_jobs( + self, + stream_output: bool = True, + timeout: int | None = None, + force_execution: bool = False, + share_outputs_with_submitter: bool = False, + share_logs_with_submitter: bool = False, + ignore_peer_version: bool = False, + ) -> None: + """Process approved jobs (DO only). Auto-syncs after unless PRE_SYNC=false.""" + if not self.has_do_role: + raise ValueError("Only dataset owners can process approved jobs") + if self._job_runner is None: + raise ValueError("Job runner is not configured for this client") + + skip_job_names = [] + + if not force_execution: + approved_jobs = [ + job for job in self._job_client.jobs if job.status == "approved" + ] + for job in approved_jobs: + result = self._sync.peer_manager.get_peer_compatibility_status( + job.submitted_by, + action=CompatAction.EXECUTE, + ignore_peer_version=ignore_peer_version, + ) + result.maybe_warn() + if result.should_skip: + skip_job_names.append(job.name) + + self._job_runner.process_approved_jobs( + stream_output=stream_output, + timeout=timeout, + skip_job_names=skip_job_names if skip_job_names else None, + share_outputs_with_submitter=share_outputs_with_submitter, + share_logs_with_submitter=share_logs_with_submitter, + ) + + if os.environ.get("PRE_SYNC", "true").lower() == "true": + self._sync.sync() + + # ------------------------------------------------------------------ # + # dataset product surface (RDS-owned) + # ------------------------------------------------------------------ # + def create_dataset( + self, + name: str, + mock_path: str | Path, + private_path: str | Path, + summary: str | None = None, + readme_path: Path | None = None, + location: str | None = None, + tags: list[str] | None = None, + users: list[str] | str | None = None, + upload_private: bool = False, + sync=True, + ): + if self._dataset_manager is None: + raise ValueError("Dataset manager is not set") + + # Only DO can create datasets + if not self.has_do_role: + raise ValueError("Only dataset owners can create datasets") + + # Convert None to empty list + if users is None: + users = [] + + dataset_name = None + created_local = False + mock_folder_id = None + private_folder_id = None + + try: + # Create dataset locally + dataset = self._dataset_manager.create( + name=name, + mock_path=mock_path, + private_path=private_path, + summary=summary, + readme_path=readme_path, + location=location, + tags=tags, + users=users, + ) + created_local = True + dataset_name = dataset.name + + # Upload mock data to collection folder + mock_folder_id = self._upload_dataset_to_collection(dataset, users) + + # Upload private data to a separate owner-only collection + if upload_private: + private_folder_id = self._upload_private_dataset_to_collection(dataset) + + if sync: + self.sync() + + return dataset + + except Exception: + logger.error( + "Failed to create dataset%s, cleaning up", + f" '{dataset_name}'" if dataset_name else "", + ) + self._cleanup_failed_dataset_creation( + dataset_name, created_local, mock_folder_id, private_folder_id + ) + raise + + def _cleanup_failed_dataset_creation( + self, + dataset_name: str | None, + created_local: bool, + mock_folder_id: str | None, + private_folder_id: str | None, + ) -> None: + """Best-effort cleanup after a failed create_dataset, in reverse order.""" + if private_folder_id is not None: + try: + self._sync._connection_router.delete_file_by_id(private_folder_id) + except Exception: + logger.warning( + "Cleanup: failed to delete private GDrive folder %s", + private_folder_id, + ) + + if mock_folder_id is not None: + try: + self._sync._connection_router.delete_file_by_id(mock_folder_id) + except Exception: + logger.warning( + "Cleanup: failed to delete mock GDrive folder %s", + mock_folder_id, + ) + + if created_local and dataset_name is not None: + try: + self._dataset_manager.delete(dataset_name, require_confirmation=False) + except Exception: + logger.warning( + "Cleanup: failed to delete local dataset '%s'", + dataset_name, + ) + + def _upload_dataset_to_collection(self, dataset, users: list[str] | str) -> str: + """Upload dataset files to collection folder. Returns the folder ID.""" + from syft_client.sync.connections.drive.gdrive_transport import ( + CollectionFolder, + ) + + collection_tag = dataset.name + + # Prepare files to upload + files = {} + for mock_file in dataset.mock_files: + if mock_file.exists(): + files[mock_file.name] = mock_file.read_bytes() + + metadata_path = dataset.mock_dir / "dataset.yaml" + if metadata_path.exists(): + files["dataset.yaml"] = metadata_path.read_bytes() + + if dataset.readme_path and dataset.readme_path.exists(): + files[dataset.readme_path.name] = dataset.readme_path.read_bytes() + + # Compute content hash + content_hash = CollectionFolder.compute_hash(files) + + # Create collection folder with hash in name + folder_id = self._sync._connection_router.owner_create_collection_folder( + DATASET_COLLECTION_PREFIX, + tag=collection_tag, + content_hash=content_hash, + owner_email=self.email, + ) + + # Upload files + self._sync._connection_router.owner_upload_collection_files( + DATASET_COLLECTION_PREFIX, collection_tag, content_hash, files + ) + + # Share with users + if users == "any": + self._sync._connection_router.owner_tag_collection_as_any( + DATASET_COLLECTION_PREFIX, collection_tag, content_hash + ) + self._sync.datasite_owner_syncer._any_shared_collections.append( + (collection_tag, content_hash) + ) + # Share with all already-approved peers + peer_emails = [p.email for p in self._sync.peer_manager.approved_peers] + if peer_emails: + self._sync._connection_router.owner_share_collection( + DATASET_COLLECTION_PREFIX, collection_tag, content_hash, peer_emails + ) + else: + self._sync._connection_router.owner_share_collection( + DATASET_COLLECTION_PREFIX, collection_tag, content_hash, users + ) + + return folder_id + + def _upload_private_dataset_to_collection(self, dataset) -> str | None: + """Upload private dataset files to a separate owner-only collection folder. + Returns the folder ID, or None if no files to upload.""" + from syft_client.sync.connections.drive.gdrive_transport import ( + CollectionFolder, + ) + + collection_tag = dataset.name + + # Collect all files in private dir (data, metadata, permissions) + files = {} + for f in dataset.private_dir.iterdir(): + if f.is_file(): + files[f.name] = f.read_bytes() + + if not files: + return None + + content_hash = CollectionFolder.compute_hash(files) + + # Create private collection folder (no sharing) + folder_id = self._sync._connection_router.owner_create_collection_folder( + PRIVATE_DATASET_COLLECTION_PREFIX, + tag=collection_tag, + content_hash=content_hash, + owner_email=self.email, + ) + + # Upload files + self._sync._connection_router.owner_upload_collection_files( + PRIVATE_DATASET_COLLECTION_PREFIX, collection_tag, content_hash, files + ) + + return folder_id + + def delete_dataset( + self, + name: str, + datasite: str | None = None, + require_confirmation: bool = True, + sync=True, + ): + if self._dataset_manager is None: + raise ValueError("Dataset manager is not set") + self._dataset_manager.delete( + name=name, + datasite=datasite, + require_confirmation=require_confirmation, + ) + # Delete collection folders from Google Drive so DS peers + # pick up the deletion on their next sync. + try: + self._sync._connection_router.owner_delete_collection( + DATASET_COLLECTION_PREFIX, name + ) + except Exception: + logger.warning("Failed to delete dataset collection '%s' from Drive", name) + try: + self._sync._connection_router.owner_delete_collection( + PRIVATE_DATASET_COLLECTION_PREFIX, name + ) + except Exception: + logger.warning( + "Failed to delete private dataset collection '%s' from Drive", + name, + ) + if sync: + self.sync() + + def share_dataset(self, tag: str, users: list[str] | str, sync=True): + """ + Share an existing dataset with additional users. + + Args: + tag: Dataset name + users: List of email addresses or "any" + sync: Whether to sync after sharing + """ + from syft_client.sync.connections.drive.gdrive_transport import ( + CollectionFolder, + ) + + if self._dataset_manager is None: + raise ValueError("Dataset manager is not set") + + if not self.has_do_role: + raise ValueError("Only dataset owners can share datasets") + + # Verify dataset exists + dataset = self._dataset_manager.get(name=tag, datasite=self.email) + if dataset is None: + raise ValueError(f"Dataset {tag} not found") + + # Compute current content hash from local files + files = {} + for mock_file in dataset.mock_files: + if mock_file.exists(): + files[mock_file.name] = mock_file.read_bytes() + metadata_path = dataset.mock_dir / "dataset.yaml" + if metadata_path.exists(): + files["dataset.yaml"] = metadata_path.read_bytes() + if dataset.readme_path and dataset.readme_path.exists(): + files[dataset.readme_path.name] = dataset.readme_path.read_bytes() + + content_hash = CollectionFolder.compute_hash(files) + + # Share collection + if users == "any": + self._sync._connection_router.owner_tag_collection_as_any( + DATASET_COLLECTION_PREFIX, tag, content_hash + ) + self._sync.datasite_owner_syncer._any_shared_collections.append( + (tag, content_hash) + ) + peer_emails = [p.email for p in self._sync.peer_manager.approved_peers] + if peer_emails: + self._sync._connection_router.owner_share_collection( + DATASET_COLLECTION_PREFIX, tag, content_hash, peer_emails + ) + else: + if isinstance(users, str): + users = [users] + self._sync._connection_router.owner_share_collection( + DATASET_COLLECTION_PREFIX, tag, content_hash, users + ) + + if sync: + self.sync() + + def share_private_dataset(self, tag: str, enclave_email: str): + """Share private dataset files with an enclave via outbox events.""" + if not self.has_do_role: + raise ValueError("Only data owners can share private datasets") + + with self._sync._sync_file_lock(): + files = self._dataset_manager.get_private_dataset_files(tag) + events_message = ( + self._sync.datasite_owner_syncer.event_cache.create_events_for_files( + files + ) + ) + self._sync.datasite_owner_syncer.queue_event_for_syftbox( + recipients=[enclave_email], + file_change_events_message=events_message, + ) + self._sync.datasite_owner_syncer.process_syftbox_events_queue() + + @property + def datasets(self) -> Any: + """The dataset manager. Auto-syncs first unless PRE_SYNC=false.""" + if self._dataset_manager is None: + raise ValueError("Dataset manager is not set") + + if os.environ.get("PRE_SYNC", "true").lower() == "true": + self._sync.sync() + + return self._dataset_manager + + def _resolve_dataset_owners_for_name(self, dataset_name: str) -> list: + """Resolve which datasite(s) own a dataset of the given name. + + Used by syft_client.utils.resolve_dataset_files_path when a client is + passed. Owned here (drives the RDS dataset manager). + """ + return [ + dataset.owner + for dataset in self._dataset_manager.get_all() + if dataset.name == dataset_name + ] + + def __repr__(self) -> str: + return f"SyftRDSClient(email={self._sync.email!r})" diff --git a/packages/syft-rds/src/syft_rds/config.py b/packages/syft-rds/src/syft_rds/config.py new file mode 100644 index 00000000000..97f191f911f --- /dev/null +++ b/packages/syft-rds/src/syft_rds/config.py @@ -0,0 +1,187 @@ +"""Configuration for the Remote Data Science product. + +``SyftRDSClientConfig`` COMPOSES the three sub-configs the RDS layer owns: + +* ``sync`` – the (domain-free) ``SyftboxManagerConfig`` sync engine config, +* ``job`` – the ``SyftJobConfig`` for the RDS-owned ``JobClient``/``SyftJobRunner``, +* ``dataset`` – the ``SyftBoxConfig`` for the RDS-owned ``SyftDatasetManager``. + +All three are derived from the SAME primitives (email, syftbox folder, role) +so their paths always line up. The RDS layer is also the place where the +dataset collection sync specs are *supplied* into the generic sync core; the +core itself stays domain-free. +""" + +from __future__ import annotations + +from pathlib import Path + +from pydantic import BaseModel + +from syft_client.sync.syftbox_manager import ( + SyftboxManagerConfig, + get_jupyter_default_syftbox_folder, + get_colab_default_syftbox_folder, +) +from syft_client.sync.sync.collection_spec import CollectionSyncSpec +from syft_job import SyftJobConfig +from syft_datasets.config import SyftBoxConfig +from syft_datasets.dataset_manager import ( + DATASET_COLLECTION_PREFIX, + PRIVATE_DATASET_COLLECTION_PREFIX, +) + +# The RDS layer OWNS the dataset collection values: both the on-wire prefixes and +# the local subpaths live here, never in the domain-free syft-client sync core. +COLLECTION_SUBPATH = Path("public/syft_datasets") +PRIVATE_COLLECTION_SUBPATH = Path("private/syft_datasets") + +# The RDS layer OWNS the dataset collection specs (syft-client core stays domain-free). +# Two specs, distinguished purely by the two generic behavioral flags: +# * public (mock) – mirror + shareable → peers' watchers pull it. +# * private (real) – restore-only + owner-only → the owner restores it for itself; +# peer-facing watchers skip it; it is never shared. +DATASET_COLLECTION_SPECS = [ + CollectionSyncSpec.public(DATASET_COLLECTION_PREFIX, COLLECTION_SUBPATH), + CollectionSyncSpec.private( + PRIVATE_DATASET_COLLECTION_PREFIX, PRIVATE_COLLECTION_SUBPATH + ), +] + + +class SyftRDSClientConfig(BaseModel): + sync: SyftboxManagerConfig + job: SyftJobConfig + dataset: SyftBoxConfig + + # ------------------------------------------------------------------ # + # private helper: build all three sub-configs from shared primitives + # ------------------------------------------------------------------ # + @staticmethod + def _compose( + *, + email: str, + folder: Path, + has_do_role: bool, + sync: SyftboxManagerConfig, + ) -> "SyftRDSClientConfig": + """Build the composed config from ONE set of primitives. + + ``sync`` is built by the caller (it differs per environment); ``job`` + and ``dataset`` are derived here from the SAME email/folder/role so all + paths line up with the sync engine. + """ + job = SyftJobConfig( + syftbox_folder=folder, + current_user_email=email, + has_do_role=has_do_role, + ) + dataset = SyftBoxConfig(syftbox_folder=folder, email=email) + return SyftRDSClientConfig(sync=sync, job=job, dataset=dataset) + + @classmethod + def for_jupyter( + cls, + email, + has_do_role=False, + has_ds_role=False, + token_path=None, + **kw, + ) -> "SyftRDSClientConfig": + folder = get_jupyter_default_syftbox_folder(email) + sync = SyftboxManagerConfig.for_jupyter( + email=email, + has_do_role=has_do_role, + has_ds_role=has_ds_role, + token_path=token_path, + collection_specs=DATASET_COLLECTION_SPECS, + **kw, + ) + return cls._compose( + email=email, folder=folder, has_do_role=has_do_role, sync=sync + ) + + @classmethod + def for_colab( + cls, + email, + has_do_role=False, + has_ds_role=False, + **kw, + ) -> "SyftRDSClientConfig": + folder = get_colab_default_syftbox_folder(email) + sync = SyftboxManagerConfig.for_colab( + email=email, + has_do_role=has_do_role, + has_ds_role=has_ds_role, + collection_specs=DATASET_COLLECTION_SPECS, + **kw, + ) + return cls._compose( + email=email, folder=folder, has_do_role=has_do_role, sync=sync + ) + + @classmethod + def _base_config_for_testing( + cls, + email=None, + syftbox_folder=None, + has_do_role=False, + has_ds_role=False, + **kw, + ) -> "SyftRDSClientConfig": + """Build a composed config over ``SyftboxManagerConfig._base_config_for_testing``. + + The sync sub-config is the base testing config (in-/out-of-memory caches, + mock connections wired in later); ``job`` and ``dataset`` are derived from + the SAME email + folder that the sync config resolved, so paths align. + """ + sync = SyftboxManagerConfig._base_config_for_testing( + email=email, + syftbox_folder=syftbox_folder, + has_do_role=has_do_role, + has_ds_role=has_ds_role, + collection_specs=DATASET_COLLECTION_SPECS, + **kw, + ) + # _base_config_for_testing may have generated a random email/folder. + return cls._compose( + email=sync.email, + folder=sync.syftbox_folder, + has_do_role=sync.has_do_role, + sync=sync, + ) + + @classmethod + def for_google_drive_testing_connection( + cls, + email, + token_path, + syftbox_folder=None, + has_do_role=False, + has_ds_role=False, + **kw, + ) -> "SyftRDSClientConfig": + """Build a composed config over ``SyftboxManagerConfig.for_google_drive_testing_connection``. + + Same shape as :meth:`_base_config_for_testing`, but the sync sub-config is + wired to a REAL Google Drive connection (via ``token_path``) instead of the + in-memory mock — for integration tests that exercise the actual transport. + ``job`` and ``dataset`` are derived from the SAME email + folder the sync + config resolved, so paths align. + """ + sync = SyftboxManagerConfig.for_google_drive_testing_connection( + email=email, + token_path=token_path, + syftbox_folder=syftbox_folder, + has_do_role=has_do_role, + has_ds_role=has_ds_role, + collection_specs=DATASET_COLLECTION_SPECS, + **kw, + ) + return cls._compose( + email=sync.email, + folder=sync.syftbox_folder, + has_do_role=sync.has_do_role, + sync=sync, + ) diff --git a/syft_client/job_auto_approval.py b/packages/syft-rds/src/syft_rds/job_auto_approval.py similarity index 95% rename from syft_client/job_auto_approval.py rename to packages/syft-rds/src/syft_rds/job_auto_approval.py index 711199f916a..a8d491018f5 100644 --- a/syft_client/job_auto_approval.py +++ b/packages/syft-rds/src/syft_rds/job_auto_approval.py @@ -10,7 +10,7 @@ from syft_job.job import JobInfo if TYPE_CHECKING: - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds.client import SyftRDSClient def _get_non_empty_lines(content: str) -> list[str]: @@ -143,7 +143,7 @@ def job_matches_criteria( def auto_approve_and_run_jobs( - client: SyftboxManager, + client: SyftRDSClient, *, required_file_contents: Dict[str, str], required_file_paths: List[str], @@ -164,7 +164,7 @@ def auto_approve_and_run_jobs( 5. (Optional) Were submitted by an approved peer Args: - client: SyftboxManager instance + client: SyftRDSClient instance required_file_contents: Dict mapping filename to expected content. Content is compared after stripping trailing whitespace. Example: {"main.py": "print('hello')"} @@ -180,8 +180,8 @@ def auto_approve_and_run_jobs( List of JobInfo objects that were approved. Example: - >>> from syft_client.sync.syftbox_manager import SyftboxManager - >>> client = SyftboxManager.for_jupyter(email="me@example.com", ...) + >>> from syft_rds import login_do + >>> client = login_do(email="me@example.com", ...) >>> approved = auto_approve_and_run_jobs( ... client, ... required_file_contents={"main.py": EXPECTED_SCRIPT}, @@ -193,7 +193,7 @@ def auto_approve_and_run_jobs( approved_peers = None if peers_only: client.load_peers() - approved_peers = [p.email for p in client._approved_peers] + approved_peers = [p.email for p in client.peer_manager.approved_peers] approved_jobs = [] jobs = client.jobs diff --git a/packages/syft-rds/src/syft_rds/login.py b/packages/syft-rds/src/syft_rds/login.py new file mode 100644 index 00000000000..7850d4bf178 --- /dev/null +++ b/packages/syft-rds/src/syft_rds/login.py @@ -0,0 +1,102 @@ +"""Entry points for the Remote Data Science product. + + from syft_rds import login_do + rds_client = login_do(email, token_path) + rds_client.datasets + rds_client.jobs + +These build a composed ``SyftRDSClientConfig`` (which owns the sync + job + +dataset sub-configs), construct a self-contained ``SyftRDSClient`` from it +""" + +from __future__ import annotations + +from pathlib import Path + +from syft_client.sync.environments.environment import Environment +from syft_client.sync.login import _init_client_login, _resolve_login_params +from syft_client.sync.login_utils import handle_potential_version_mismatches_on_login +from syft_client.sync.utils.syftbox_utils import check_env + +from syft_rds.client import SyftRDSClient +from syft_rds.config import SyftRDSClientConfig + + +def _login( + *, + email: str | None, + token_path: str | Path | None, + sync: bool, + load_peers: bool, + skip_peer_on_patch_version_diff: bool | None, + has_do_role: bool, + has_ds_role: bool, +) -> SyftRDSClient: + """Shared RDS login. + + Mirrors the pre-split ``syft_client`` login flow: detect the environment, + resolve login params + """ + env = check_env() + email, token_path = _resolve_login_params(email, token_path) + handle_potential_version_mismatches_on_login(email, token_path) + + if env == Environment.COLAB: + config = SyftRDSClientConfig.for_colab( + email=email, + has_do_role=has_do_role, + has_ds_role=has_ds_role, + skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + ) + else: + config = SyftRDSClientConfig.for_jupyter( + email=email, + has_do_role=has_do_role, + has_ds_role=has_ds_role, + token_path=Path(token_path) if token_path is not None else None, + skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + ) + + client = SyftRDSClient.from_config(config) + _init_client_login(client.sync_engine, sync=sync, load_peers=load_peers) + return client + + +def login_do( + email: str | None = None, + token_path: str | Path | None = None, + *, + sync: bool = True, + load_peers: bool = True, + skip_peer_on_patch_version_diff: bool | None = None, +) -> SyftRDSClient: + """Log in as a Data Owner and return a self-contained RDS client.""" + return _login( + email=email, + token_path=token_path, + sync=sync, + load_peers=load_peers, + skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + has_do_role=True, + has_ds_role=False, + ) + + +def login_ds( + email: str | None = None, + token_path: str | Path | None = None, + *, + sync: bool = True, + load_peers: bool = True, + skip_peer_on_patch_version_diff: bool | None = None, +) -> SyftRDSClient: + """Log in as a Data Scientist and return a self-contained RDS client.""" + return _login( + email=email, + token_path=token_path, + sync=sync, + load_peers=load_peers, + skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + has_do_role=False, + has_ds_role=True, + ) diff --git a/packages/syft-rds/tests/conftest.py b/packages/syft-rds/tests/conftest.py new file mode 100644 index 00000000000..ee302387819 --- /dev/null +++ b/packages/syft-rds/tests/conftest.py @@ -0,0 +1,23 @@ +"""Pytest configuration for the syft-rds test suite. + +Mirrors the top-level ``tests/conftest.py`` so RDS product tests behave the same +whether run here or as part of the full monorepo suite. +""" + +import os + +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def disable_pre_sync_for_tests(): + """Disable PRE_SYNC by default for all tests (explicit sync control).""" + original_value = os.environ.get("PRE_SYNC") + os.environ["PRE_SYNC"] = "false" + + yield + + if original_value is not None: + os.environ["PRE_SYNC"] = original_value + else: + os.environ.pop("PRE_SYNC", None) diff --git a/packages/syft-rds/tests/dataset_test_utils.py b/packages/syft-rds/tests/dataset_test_utils.py new file mode 100644 index 00000000000..94bc1a89ed0 --- /dev/null +++ b/packages/syft-rds/tests/dataset_test_utils.py @@ -0,0 +1,20 @@ +"""Shared helpers for the syft-rds product tests. + +Uniquely named (not ``utils.py``) to avoid a pytest module-name collision with +``tests/unit/utils.py`` when the full monorepo suite is collected together. +""" + +import random +from pathlib import Path + + +def create_tmp_dataset_files(): + tmp_dir = Path("/tmp/syft-datasets-testing") / str(random.randint(1, 1000000)) + tmp_dir.mkdir(parents=True, exist_ok=True) + mock_path = tmp_dir / "mock.txt" + private_path = tmp_dir / "private.txt" + readme_path = tmp_dir / "readme.md" + mock_path.write_text("Hello, world!") + private_path.write_text("Hello, world private!") + readme_path.write_text("Hello, world!") + return mock_path, private_path, readme_path diff --git a/packages/syft-rds/tests/test_client.py b/packages/syft-rds/tests/test_client.py new file mode 100644 index 00000000000..304c0b82589 --- /dev/null +++ b/packages/syft-rds/tests/test_client.py @@ -0,0 +1,19 @@ +"""Tests for the self-contained SyftRDSClient product.""" + +from syft_rds import SyftRDSClient + + +def test_rds_layer_supplies_collection_specs(): + """The composed DS sync engine's watcher cache received the dataset spec + from the RDS layer (the RDS -> generic-engine spec-injection seam).""" + from syft_datasets.dataset_manager import DATASET_COLLECTION_PREFIX + + ds, do = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False + ) + + watcher_cache = ds.sync_engine.datasite_watcher_syncer.datasite_watcher_cache + specs = watcher_cache.collection_specs + + assert len(specs) > 0 + assert any(spec.prefix == DATASET_COLLECTION_PREFIX for spec in specs) diff --git a/tests/unit/test_create_dataset_cleanup.py b/packages/syft-rds/tests/test_create_dataset_cleanup.py similarity index 92% rename from tests/unit/test_create_dataset_cleanup.py rename to packages/syft-rds/tests/test_create_dataset_cleanup.py index ac938dee596..68621ae914b 100644 --- a/tests/unit/test_create_dataset_cleanup.py +++ b/packages/syft-rds/tests/test_create_dataset_cleanup.py @@ -6,14 +6,16 @@ import pytest from syft_client.sync.syftbox_manager import SyftboxManager -from tests.unit.utils import create_tmp_dataset_files +from syft_rds import SyftRDSClient +from syft_datasets.dataset_manager import DATASET_COLLECTION_PREFIX +from dataset_test_utils import create_tmp_dataset_files class TestCreateDatasetCleanup: """Tests that create_dataset cleans up partial state on failure.""" def _make_do_manager(self): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) return do_manager @@ -93,7 +95,9 @@ def test_cleanup_on_private_upload_failure(self): ) # Mock GDrive folder should have been cleaned up too - collections = do_manager._connection_router.owner_list_dataset_collections() + collections = do_manager.sync_engine._connection_router.owner_list_collections( + DATASET_COLLECTION_PREFIX + ) assert "testdataset" not in collections def test_cleanup_on_sync_failure(self): @@ -145,9 +149,7 @@ def test_cleanup_logs_warning_on_failure(self, caplog): side_effect=OSError("delete broken"), ), ): - with caplog.at_level( - logging.WARNING, logger="syft_client.sync.syftbox_manager" - ): + with caplog.at_level(logging.WARNING, logger="syft_rds.client"): with pytest.raises(RuntimeError, match="upload failed"): do_manager.create_dataset(**self._dataset_kwargs()) diff --git a/tests/unit/test_dataset_upload_private.py b/packages/syft-rds/tests/test_dataset_upload_private.py similarity index 76% rename from tests/unit/test_dataset_upload_private.py rename to packages/syft-rds/tests/test_dataset_upload_private.py index c877f9d824d..659de3034cc 100644 --- a/tests/unit/test_dataset_upload_private.py +++ b/packages/syft-rds/tests/test_dataset_upload_private.py @@ -1,14 +1,17 @@ from syft_client.sync.connections.drive.gdrive_transport import GDriveConnection -from syft_client.sync.syftbox_manager import SyftboxManager -from syft_datasets.dataset_manager import PRIVATE_DATASET_COLLECTION_PREFIX -from tests.unit.utils import create_tmp_dataset_files +from syft_rds import SyftRDSClient +from syft_datasets.dataset_manager import ( + DATASET_COLLECTION_PREFIX, + PRIVATE_DATASET_COLLECTION_PREFIX, +) +from dataset_test_utils import create_tmp_dataset_files class TestDatasetUploadPrivate: def test_ds_cannot_see_private_data(self): """When DO creates a dataset with upload_private=True, DS should see mock data but NOT private data.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) @@ -36,14 +39,14 @@ def test_ds_cannot_see_private_data(self): assert len(dataset.mock_files) > 0 # DS should NOT see any private collections via the connection - private_collections = ( - ds_manager._connection_router.owner_list_private_dataset_collections() + private_collections = ds_manager.sync_engine._connection_router.owner_list_all_collections_with_permissions( + PRIVATE_DATASET_COLLECTION_PREFIX ) assert len(private_collections) == 0 # Verify private collection folder exists on GDrive (visible to DO) - do_private_collections = ( - do_manager._connection_router.owner_list_private_dataset_collections() + do_private_collections = do_manager.sync_engine._connection_router.owner_list_all_collections_with_permissions( + PRIVATE_DATASET_COLLECTION_PREFIX ) assert len(do_private_collections) == 1 assert do_private_collections[0].tag == "testdataset" @@ -51,7 +54,7 @@ def test_ds_cannot_see_private_data(self): def test_do_cold_start_restores_private_data(self): """When DO creates a dataset with upload_private=True, then loses local data, a fresh sync should restore the private files.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) @@ -79,7 +82,7 @@ def test_do_cold_start_restores_private_data(self): shutil.rmtree(private_dir) assert not private_dir.exists() - do_manager.datasite_owner_syncer.initial_sync_done = False + do_manager.sync_engine.datasite_owner_syncer.initial_sync_done = False # Sync again — should restore private data from GDrive do_manager.sync() @@ -91,7 +94,7 @@ def test_do_cold_start_restores_private_data(self): def test_default_behavior_no_private_upload(self): """When upload_private is not set, no private collection should be created.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) @@ -107,20 +110,22 @@ def test_default_behavior_no_private_upload(self): ) # No private collection should exist - private_collections = ( - do_manager._connection_router.owner_list_private_dataset_collections() + private_collections = do_manager.sync_engine._connection_router.owner_list_all_collections_with_permissions( + PRIVATE_DATASET_COLLECTION_PREFIX ) assert len(private_collections) == 0 # Mock collection should still exist mock_collections = ( - do_manager._connection_router.owner_list_dataset_collections() + do_manager.sync_engine._connection_router.owner_list_collections( + DATASET_COLLECTION_PREFIX + ) ) assert "testdataset" in mock_collections def test_ds_cannot_find_private_folders_via_gdrive_query(self): """DS searching GDrive directly for private collection prefix should find nothing.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) @@ -137,7 +142,9 @@ def test_ds_cannot_find_private_folders_via_gdrive_query(self): ) # Get the DS's raw GDrive connection and search for private folders - ds_connection: GDriveConnection = ds_manager._connection_router.connections[0] + ds_connection: GDriveConnection = ( + ds_manager.sync_engine._connection_router.connections[0] + ) results = ( ds_connection.drive_service.files() .list( @@ -152,7 +159,9 @@ def test_ds_cannot_find_private_folders_via_gdrive_query(self): assert len(results.get("files", [])) == 0 # DO should find it with the same query - do_connection: GDriveConnection = do_manager._connection_router.connections[0] + do_connection: GDriveConnection = ( + do_manager.sync_engine._connection_router.connections[0] + ) do_results = ( do_connection.drive_service.files() .list( diff --git a/tests/unit/test_datasets_jobs_repr.py b/packages/syft-rds/tests/test_datasets_jobs_repr.py similarity index 92% rename from tests/unit/test_datasets_jobs_repr.py rename to packages/syft-rds/tests/test_datasets_jobs_repr.py index e64025bf25f..d53e3053deb 100644 --- a/tests/unit/test_datasets_jobs_repr.py +++ b/packages/syft-rds/tests/test_datasets_jobs_repr.py @@ -2,14 +2,14 @@ import pytest -from syft_client.sync.syftbox_manager import SyftboxManager +from syft_rds import SyftRDSClient from syft_job.job import JobInfo, JobsList -from tests.unit.utils import create_tmp_dataset_files +from dataset_test_utils import create_tmp_dataset_files def _create_manager_with_dataset(): """Create a pair of managers and a dataset, return (ds_manager, do_manager, dataset).""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -76,7 +76,7 @@ def test_dataset_manager_repr_html(): def test_dataset_manager_repr_html_with_tags(): - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -92,7 +92,7 @@ def test_dataset_manager_repr_html_with_tags(): def test_dataset_manager_repr_html_empty(): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -116,7 +116,7 @@ def test_dataset_manager_get_missing_lists_available(): def test_dataset_manager_get_missing_no_datasets(): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) diff --git a/tests/unit/test_job_auto_approval.py b/packages/syft-rds/tests/test_job_auto_approval.py similarity index 91% rename from tests/unit/test_job_auto_approval.py rename to packages/syft-rds/tests/test_job_auto_approval.py index 342cd8ee754..b83e5444bef 100644 --- a/tests/unit/test_job_auto_approval.py +++ b/packages/syft-rds/tests/test_job_auto_approval.py @@ -10,8 +10,8 @@ from syft_bg.api import auto_approve_job from syft_bg.approve.config import AutoApproveConfig from syft_bg.common.config import get_default_paths -from syft_client.job_auto_approval import auto_approve_and_run_jobs -from syft_client.sync.syftbox_manager import SyftboxManager +from syft_rds import SyftRDSClient +from syft_rds.job_auto_approval import auto_approve_and_run_jobs @contextmanager @@ -41,7 +41,7 @@ def test_auto_approve_and_run_jobs(): - The exact Python script content (newline agnostic) - Exactly the required files (no more, no less) """ - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -134,7 +134,7 @@ def _create_project_dir(script_content="print('hello')\n", data_content='{"k": " def test_auto_approve_job_default_all_content_matched(): """Default behavior: all files are content-matched.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -156,7 +156,7 @@ def test_auto_approve_job_default_all_content_matched(): def test_auto_approve_job_file_paths_only(): """file_paths specified: those are name-only, rest are content-matched.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -175,7 +175,7 @@ def test_auto_approve_job_file_paths_only(): def test_auto_approve_job_contents_only(): """contents specified: only those files are content-matched, rest ignored.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -194,7 +194,7 @@ def test_auto_approve_job_contents_only(): def test_auto_approve_job_both_contents_and_file_paths(): """Both specified: contents are content-matched, file_paths are name-only.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -213,7 +213,7 @@ def test_auto_approve_job_both_contents_and_file_paths(): def test_auto_approve_job_overlap_error(): """Overlap between contents and file_paths should fail.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -227,7 +227,7 @@ def test_auto_approve_job_overlap_error(): def test_auto_approve_job_file_not_found_error(): """Referencing a non-existent file should fail.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -241,7 +241,7 @@ def test_auto_approve_job_file_not_found_error(): def test_auto_approve_job_nested_directory(): """Files in subdirectories are stored with relative paths.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -272,7 +272,7 @@ def test_auto_approve_job_nested_directory(): def test_auto_approve_job_default_no_special_treatment_for_non_params_json(): """Default behavior with non-params.json files: all are content-matched.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -294,7 +294,7 @@ def test_auto_approve_job_default_no_special_treatment_for_non_params_json(): def test_auto_approve_job_default_params_json_is_name_only(): """Default behavior: params.json is automatically name-only.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) diff --git a/tests/unit/test_load_dataset_code.py b/packages/syft-rds/tests/test_load_dataset_code.py similarity index 89% rename from tests/unit/test_load_dataset_code.py rename to packages/syft-rds/tests/test_load_dataset_code.py index d8efa55c271..ec2c1cc1192 100644 --- a/tests/unit/test_load_dataset_code.py +++ b/packages/syft-rds/tests/test_load_dataset_code.py @@ -5,7 +5,7 @@ import pytest import syft_client as sc -from syft_client.sync.syftbox_manager import SyftboxManager +from syft_rds import SyftRDSClient def _create_tmp_code_folder() -> Path: @@ -49,7 +49,7 @@ def _create_local_dataset(do_manager, name: str) -> None: def test_load_dataset_code_top_level_and_nested(): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) _create_local_dataset(do_manager, "my_code_dataset") @@ -68,7 +68,7 @@ def test_load_dataset_code_top_level_and_nested(): def test_load_dataset_code_explicit_module_name(): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) _create_local_dataset(do_manager, "my_code_dataset") @@ -82,7 +82,7 @@ def test_load_dataset_code_explicit_module_name(): def test_load_dataset_code_invalid_path_raises(): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) with pytest.raises(ValueError, match="Invalid path"): @@ -90,7 +90,7 @@ def test_load_dataset_code_invalid_path_raises(): def test_load_dataset_code_missing_file_raises(): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) _create_local_dataset(do_manager, "my_code_dataset") diff --git a/pyproject.toml b/pyproject.toml index e39fc4db933..1067d97c465 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,31 +30,14 @@ dependencies = [ "google-auth[pyjwt]>=2.22.0", "google-auth-oauthlib>=1.0.0", "rich>=13.0.0", - "pandas", "pydantic-settings>=2.11.0", - "pyyaml>=6.0", - "jinja2>=3.1.0", - "click>=8.0.0", "portalocker>=2.8.0", - "python-daemon>=3.0.0", - "syft-bg==0.3.11", - "syft-job==0.1.39", # TODO: move to optional-dependencies after adding lazy imports - "syft-dataset==0.1.20", # TODO: move to optional-dependencies after adding lazy imports "syft-permissions==0.1.14", "syft-perms==0.1.14", "syft-crypto-python>=0.1.2b2", ] -[project.optional-dependencies] -datasets = [ - "syft-dataset==0.1.20", -] -job = [ - "syft-job==0.1.39", -] -all = [ - "syft-client[datasets,job]", -] + [tool.uv] default-groups = ["dev", "test"] diff --git a/scripts/auto_approve_jobs.py b/scripts/auto_approve_jobs.py index 67839f00603..531535c402b 100644 --- a/scripts/auto_approve_jobs.py +++ b/scripts/auto_approve_jobs.py @@ -9,8 +9,8 @@ import time from pathlib import Path -from syft_client.job_auto_approval import auto_approve_and_run_jobs -from syft_client.sync.syftbox_manager import SyftboxManager +from syft_rds import login_do +from syft_rds.job_auto_approval import auto_approve_and_run_jobs # Configuration - edit these values EMAIL = "your-email@example.com" @@ -37,9 +37,8 @@ def main(): - client = SyftboxManager.for_jupyter( + client = login_do( email=EMAIL, - has_do_role=True, token_path=TOKEN_PATH, ) diff --git a/syft_client/sync/callback_mixin.py b/syft_client/sync/callback_mixin.py index 0f979cb6b24..30319f7ae1c 100644 --- a/syft_client/sync/callback_mixin.py +++ b/syft_client/sync/callback_mixin.py @@ -1,6 +1,9 @@ +import logging from pydantic import BaseModel from typing import Dict, List, Callable +logger = logging.getLogger(__name__) + class BaseModelCallbackMixin(BaseModel): callbacks: Dict[str, List[Callable]] = {} @@ -9,3 +12,24 @@ def add_callback(self, on: str, callback: Callable): if on not in self.callbacks: self.callbacks[on] = [] self.callbacks[on].append(callback) + + def on(self, event: str, callback: Callable) -> None: + """Readable alias for registering a lifecycle callback.""" + self.add_callback(event, callback) + + def _emit(self, event: str, *args, **kwargs) -> None: + """Fire every callback registered for `event`, ISOLATED from each other + and from the emitting core. + + No-op when nothing is registered, so emitting is always safe. + """ + for callback in self.callbacks.get(event, []): + try: + callback(*args, **kwargs) + except Exception as e: + logger.exception( + "callback for event %r failed (%r) exception: %r", + event, + callback, + e, + ) diff --git a/syft_client/sync/connections/base_connection.py b/syft_client/sync/connections/base_connection.py index a1c5afc881f..b49ad55cbd1 100644 --- a/syft_client/sync/connections/base_connection.py +++ b/syft_client/sync/connections/base_connection.py @@ -32,58 +32,55 @@ def watcher_send_proposed_file_changes_message( def from_config(cls, config: ConnectionConfig): return config.connection_type.from_config(config) - def owner_create_dataset_collection_folder( - self, tag: str, content_hash: str, owner_email: str + # ========================================================================= + # GENERIC PREFIX-PARAMETERIZED COLLECTION METHODS + # ========================================================================= + + def owner_create_collection_folder( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> str: raise NotImplementedError() - def owner_tag_dataset_collection_as_any(self, tag: str, content_hash: str) -> None: + def owner_tag_collection_as_any( + self, prefix: str, tag: str, content_hash: str + ) -> None: raise NotImplementedError() - def owner_share_dataset_collection( - self, tag: str, content_hash: str, users: list[str] + def owner_share_collection( + self, prefix: str, tag: str, content_hash: str, users: list[str] ) -> None: raise NotImplementedError() - def owner_upload_dataset_files( - self, tag: str, content_hash: str, files: dict[str, bytes] + def owner_upload_collection_files( + self, prefix: str, tag: str, content_hash: str, files: dict[str, bytes] ) -> None: raise NotImplementedError() - def owner_list_dataset_collections(self) -> list[str]: + def owner_list_collections(self, prefix: str) -> list[str]: raise NotImplementedError() - def owner_list_all_dataset_collections_with_permissions( - self, + def owner_list_all_collections_with_permissions( + self, prefix: str ) -> list[FileCollection]: - """Returns list of FileCollection objects for DO's dataset collections.""" - raise NotImplementedError() - - def watcher_list_dataset_collections(self) -> list[dict]: - """Returns list of dicts with keys: owner_email, tag, content_hash""" raise NotImplementedError() - def watcher_download_dataset_collection( - self, tag: str, content_hash: str, owner_email: str - ) -> dict[str, bytes]: + def owner_delete_collection(self, prefix: str, tag: str) -> None: raise NotImplementedError() - def owner_create_private_dataset_collection_folder( - self, tag: str, content_hash: str, owner_email: str - ) -> str: + def watcher_list_collections(self, prefix: str) -> list[dict]: raise NotImplementedError() - def owner_upload_private_dataset_files( - self, tag: str, content_hash: str, files: dict[str, bytes] - ) -> None: + def watcher_download_collection( + self, prefix: str, tag: str, content_hash: str, owner_email: str + ) -> dict[str, bytes]: raise NotImplementedError() - def owner_list_private_dataset_collections(self) -> list[FileCollection]: + def watcher_get_collection_file_metadatas( + self, prefix: str, tag: str, content_hash: str, owner_email: str + ) -> list[dict]: raise NotImplementedError() - def owner_get_private_collection_file_metadatas( - self, tag: str, content_hash: str, owner_email: str - ) -> list[dict]: + def watcher_download_collection_file(self, file_id: str) -> bytes: raise NotImplementedError() # ========================================================================= diff --git a/syft_client/sync/connections/collection_prefixes.py b/syft_client/sync/connections/collection_prefixes.py new file mode 100644 index 00000000000..ab49d0af787 --- /dev/null +++ b/syft_client/sync/connections/collection_prefixes.py @@ -0,0 +1,5 @@ +"""Canonical wire-level collection folder-name prefixes. +""" + +DATASET_COLLECTION_PREFIX = "syft_datasetcollection" +PRIVATE_DATASET_COLLECTION_PREFIX = "syft_privatecollection" diff --git a/syft_client/sync/connections/connection_router.py b/syft_client/sync/connections/connection_router.py index e165ed86e21..b2c4a7b83d6 100644 --- a/syft_client/sync/connections/connection_router.py +++ b/syft_client/sync/connections/connection_router.py @@ -258,71 +258,70 @@ def read_peer_encryption_bundle(self, peer_email: str) -> str | None: return connection.read_peer_encryption_bundle(peer_email) # ========================================================================= - # DATASET METHODS (with encryption) + # GENERIC PREFIX-PARAMETERIZED COLLECTION METHODS (with encryption) # ========================================================================= - def owner_create_dataset_collection_folder( - self, tag: str, content_hash: str, owner_email: str + def owner_create_collection_folder( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> str: connection = self.connection_for_send_message() - return connection.owner_create_dataset_collection_folder( - tag, content_hash, owner_email + return connection.owner_create_collection_folder( + prefix, tag, content_hash, owner_email ) - def owner_tag_dataset_collection_as_any(self, tag: str, content_hash: str) -> None: + def owner_tag_collection_as_any( + self, prefix: str, tag: str, content_hash: str + ) -> None: connection = self.connection_for_send_message() - connection.owner_tag_dataset_collection_as_any(tag, content_hash) + connection.owner_tag_collection_as_any(prefix, tag, content_hash) - def owner_share_dataset_collection( - self, tag: str, content_hash: str, users: list[str] + def owner_share_collection( + self, prefix: str, tag: str, content_hash: str, users: list[str] ) -> None: connection = self.connection_for_send_message() - connection.owner_share_dataset_collection(tag, content_hash, users) + connection.owner_share_collection(prefix, tag, content_hash, users) - def owner_upload_dataset_files( + def owner_upload_collection_files( self, + prefix: str, tag: str, content_hash: str, files: dict[str, bytes], recipient_email: str | None = None, ) -> None: - """Upload dataset files, encrypting each file if encryption is enabled.""" + """Upload collection files, encrypting each file if encryption is enabled.""" if recipient_email and self.peer_store: files = { name: self.peer_store.encrypt_if_needed(recipient_email, data) for name, data in files.items() } connection = self.connection_for_send_message() - connection.owner_upload_dataset_files(tag, content_hash, files) + connection.owner_upload_collection_files(prefix, tag, content_hash, files) - def owner_list_dataset_collections(self) -> list[str]: + def owner_list_collections(self, prefix: str) -> list[str]: connection = self.connection_for_send_message() - return connection.owner_list_dataset_collections() + return connection.owner_list_collections(prefix) - def owner_list_all_dataset_collections_with_permissions( - self, + def owner_list_all_collections_with_permissions( + self, prefix: str ) -> list[FileCollection]: connection = self.connection_for_send_message() - return connection.owner_list_all_dataset_collections_with_permissions() + return connection.owner_list_all_collections_with_permissions(prefix) - def owner_delete_dataset_collection(self, tag: str) -> None: + def owner_delete_collection(self, prefix: str, tag: str) -> None: connection = self.connection_for_send_message() - connection.owner_delete_dataset_collection(tag) + connection.owner_delete_collection(prefix, tag) - def owner_delete_private_dataset_collection(self, tag: str) -> None: - connection = self.connection_for_send_message() - connection.owner_delete_private_dataset_collection(tag) - - def watcher_list_dataset_collections(self) -> list[dict]: + def watcher_list_collections(self, prefix: str) -> list[dict]: connection = self.connection_for_receive_message() - return connection.watcher_list_dataset_collections() + return connection.watcher_list_collections(prefix) - def watcher_download_dataset_collection( - self, tag: str, content_hash: str, owner_email: str + def watcher_download_collection( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> dict[str, bytes]: connection = self.connection_for_datasite_watcher() - files = connection.watcher_download_dataset_collection( - tag, content_hash, owner_email + files = connection.watcher_download_collection( + prefix, tag, content_hash, owner_email ) if self.peer_store and owner_email: files = { @@ -331,32 +330,6 @@ def watcher_download_dataset_collection( } return files - def owner_create_private_dataset_collection_folder( - self, tag: str, content_hash: str, owner_email: str - ) -> str: - connection = self.connection_for_send_message() - return connection.owner_create_private_dataset_collection_folder( - tag, content_hash, owner_email - ) - - def owner_upload_private_dataset_files( - self, tag: str, content_hash: str, files: dict[str, bytes] - ) -> None: - connection = self.connection_for_send_message() - connection.owner_upload_private_dataset_files(tag, content_hash, files) - - def owner_list_private_dataset_collections(self) -> list[FileCollection]: - connection = self.connection_for_send_message() - return connection.owner_list_private_dataset_collections() - - def owner_get_private_collection_file_metadatas( - self, tag: str, content_hash: str, owner_email: str - ) -> List[dict]: - connection = self.connection_for_datasite_watcher() - return connection.owner_get_private_collection_file_metadatas( - tag, content_hash, owner_email - ) - def connection_for_version_read( self, create_new: bool = False ) -> SyftboxPlatformConnection: @@ -387,17 +360,17 @@ def share_version_file_with_peer(self, peer_email: str) -> None: connection = self.connection_for_own_syftbox() connection.share_version_file_with_peer(peer_email) - def watcher_get_dataset_collection_file_metadatas( - self, tag: str, content_hash: str, owner_email: str + def watcher_get_collection_file_metadatas( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> List[dict]: connection = self.connection_for_datasite_watcher() - return connection.watcher_get_dataset_collection_file_metadatas( - tag, content_hash, owner_email + return connection.watcher_get_collection_file_metadatas( + prefix, tag, content_hash, owner_email ) - def watcher_download_dataset_file(self, file_id: str, owner_email: str) -> bytes: + def watcher_download_collection_file(self, file_id: str, owner_email: str) -> bytes: connection = self.connection_for_datasite_watcher() - data = connection.watcher_download_dataset_file(file_id) + data = connection.watcher_download_collection_file(file_id) data = self.peer_store.decrypt_dataset_if_needed(owner_email, data) return data diff --git a/syft_client/sync/connections/drive/gdrive_transport.py b/syft_client/sync/connections/drive/gdrive_transport.py index 3cf50f2dd7c..a0fbfcb9f4f 100644 --- a/syft_client/sync/connections/drive/gdrive_transport.py +++ b/syft_client/sync/connections/drive/gdrive_transport.py @@ -29,10 +29,6 @@ FileCollection, SyftboxPlatformConnection, ) -from syft_datasets.dataset_manager import ( - DATASET_COLLECTION_PREFIX, - PRIVATE_DATASET_COLLECTION_PREFIX, -) from syft_client.sync.events.file_change_event import ( FileChangeEventsMessageFileName, FileChangeEventsMessage, @@ -66,6 +62,11 @@ SYFTBOX_FOLDER = "SyftBox" GOOGLE_FOLDER_MIME_TYPE = "application/vnd.google-apps.folder" SCOPES = ["https://www.googleapis.com/auth/drive"] + +from syft_client.sync.connections.collection_prefixes import ( + DATASET_COLLECTION_PREFIX, + PRIVATE_DATASET_COLLECTION_PREFIX, +) logging.getLogger("google_auth_httplib2").setLevel(logging.ERROR) @@ -143,58 +144,42 @@ def as_string(self) -> str: return f"{SYFT_CLIENT_VERSION}#{self.email}" -class DatasetCollectionFolder(BaseModel): - """Represents a dataset collection folder with format: {prefix}_{tag}_{hash}""" +class CollectionFolder(BaseModel): + """Naming value object for collection folders (domain-agnostic). - tag: str - content_hash: str - - def as_string(self) -> str: - return f"{DATASET_COLLECTION_PREFIX}_{self.tag}_{self.content_hash}" - - @classmethod - def from_name(cls, name: str) -> "DatasetCollectionFolder": - """Parse folder name like 'syft_datasetcollection_mytag_abc123'""" - parts = name.split("_") - if len(parts) < 3: - raise ValueError(f"Invalid dataset collection folder name: {name}") - # prefix is parts[0:2] joined = "syft_datasetcollection" - # tag is parts[2:-1] joined (in case tag has underscores) - # hash is parts[-1] - tag = "_".join(parts[2:-1]) - content_hash = parts[-1] - return cls(tag=tag, content_hash=content_hash) - - @staticmethod - def compute_hash(files: dict[str, bytes]) -> str: - """Compute a hash from file contents.""" - from syft_client.sync.file_utils import compute_file_hashes - - return compute_file_hashes(files) - - -class PrivateDatasetCollectionFolder(BaseModel): - """Represents a private dataset collection folder with format: {prefix}_{tag}_{hash}""" + Collections (datasets, private datasets, or any future collection type) are + named ``{prefix}_{tag}_{content_hash}`` on the wire. This is the single + source of truth for building and parsing that name; it also computes the + content hash from the folder's file contents. + """ + prefix: str tag: str content_hash: str - def as_string(self) -> str: - return f"{PRIVATE_DATASET_COLLECTION_PREFIX}_{self.tag}_{self.content_hash}" + @property + def folder_name(self) -> str: + """The on-wire folder name. Doubles as the cache key (same string).""" + return f"{self.prefix}_{self.tag}_{self.content_hash}" @classmethod - def from_name(cls, name: str) -> "PrivateDatasetCollectionFolder": - """Parse folder name like 'syft_privatecollection_mytag_abc123'""" - parts = name.split("_") - if len(parts) < 3: - raise ValueError(f"Invalid private collection folder name: {name}") - tag = "_".join(parts[2:-1]) - content_hash = parts[-1] - return cls(tag=tag, content_hash=content_hash) + def from_name(cls, prefix: str, name: str) -> "CollectionFolder": + """Parse '{prefix}_{tag}_{hash}' into a CollectionFolder. + + Raises ValueError if the name does not carry the prefix, so callers can + skip non-matching folders. + """ + marker = f"{prefix}_" + if not name.startswith(marker): + raise ValueError(f"Invalid collection folder name: {name}") + tag, separator, content_hash = name[len(prefix) + 1 :].rpartition("_") + if not separator or not tag or not content_hash: + raise ValueError(f"Invalid collection folder name: {name}") + return cls(prefix=prefix, tag=tag, content_hash=content_hash) @staticmethod def compute_hash(files: dict[str, bytes]) -> str: - """Compute a hash from file contents.""" + """Compute a content hash from file contents.""" from syft_client.sync.file_utils import compute_file_hashes return compute_file_hashes(files) @@ -286,7 +271,7 @@ class Config: personal_syftbox_event_id_cache: Dict[str, str] = {} # tag -> dataset collection folder id - dataset_collection_folder_id_cache: Dict[str, str] = {} + collection_folder_id_cache: Dict[str, str] = {} # Rolling state caches for single-API-call optimization _rolling_state_folder_id: str | None = None @@ -404,8 +389,8 @@ def _copy_caches_to(self, other: "GDriveConnection") -> None: other.personal_syftbox_event_id_cache = dict( self.personal_syftbox_event_id_cache ) - other.dataset_collection_folder_id_cache = dict( - self.dataset_collection_folder_id_cache + other.collection_folder_id_cache = dict( + self.collection_folder_id_cache ) @property @@ -1174,7 +1159,7 @@ def reset_caches(self): self.own_datasite_outbox_cache.clear() self.archive_folder_id_cache.clear() self.personal_syftbox_event_id_cache.clear() - self.dataset_collection_folder_id_cache.clear() + self.collection_folder_id_cache.clear() self._rolling_state_folder_id = None self._rolling_state_file_id = None self._encryption_bundles_folder_id = None @@ -1493,34 +1478,44 @@ def get_inbox_proposed_event_id_from_name( items = results.get("files", []) return items[0]["id"] if items else None - def owner_create_dataset_collection_folder( - self, tag: str, content_hash: str, owner_email: str + # ========================================================================= + # GENERIC COLLECTION METHODS (parameterized on `prefix`) + # + # Folder names are built and parsed via CollectionFolder, the single source + # of truth for the f"{prefix}_{tag}_{content_hash}" scheme. On-wire behavior + # is byte-identical to the previous inline construction. + # ========================================================================= + + def owner_create_collection_folder( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> str: - """Create /SyftBox/{DATASET_COLLECTION_PREFIX}_{tag}_{hash} folder.""" - folder_obj = DatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() - cache_key = f"{tag}_{content_hash}" + """Create /SyftBox/{prefix}_{tag}_{hash} folder.""" + folder_name = CollectionFolder( + prefix=prefix, tag=tag, content_hash=content_hash + ).folder_name # Check cache - if cache_key in self.dataset_collection_folder_id_cache: - return self.dataset_collection_folder_id_cache[cache_key] + if folder_name in self.collection_folder_id_cache: + return self.collection_folder_id_cache[folder_name] syftbox_folder_id = self.get_syftbox_folder_id() # Check if exists folder_id = self._find_folder_by_name(folder_name, parent_id=syftbox_folder_id) if folder_id: - self.dataset_collection_folder_id_cache[cache_key] = folder_id + self.collection_folder_id_cache[folder_name] = folder_id return folder_id # Create new folder folder_id = self.create_folder(folder_name, syftbox_folder_id) - self.dataset_collection_folder_id_cache[cache_key] = folder_id + self.collection_folder_id_cache[folder_name] = folder_id return folder_id - def owner_tag_dataset_collection_as_any(self, tag: str, content_hash: str) -> None: - """Mark dataset collection as shared with 'any' via appProperties.""" - folder_id = self._get_dataset_collection_folder_id(tag, content_hash) + def owner_tag_collection_as_any( + self, prefix: str, tag: str, content_hash: str + ) -> None: + """Mark collection as shared with 'any' via appProperties.""" + folder_id = self._get_collection_folder_id(prefix, tag, content_hash) execute_with_retries( self.drive_service.files().update( fileId=folder_id, @@ -1528,13 +1523,13 @@ def owner_tag_dataset_collection_as_any(self, tag: str, content_hash: str) -> No ) ) - def owner_share_dataset_collection( - self, tag: str, content_hash: str, users: list[str] + def owner_share_collection( + self, prefix: str, tag: str, content_hash: str, users: list[str] ) -> None: - """Share dataset collection folder with specific users via batch API.""" + """Share collection folder with specific users via batch API.""" if not users: return - folder_id = self._get_dataset_collection_folder_id(tag, content_hash) + folder_id = self._get_collection_folder_id(prefix, tag, content_hash) self._batch_add_permissions(folder_id, users) def _batch_add_permissions(self, file_id: str, users: list[str]) -> None: @@ -1565,11 +1560,11 @@ def callback(request_id, response, exception): ) batch_execute_with_retries(batch) - def owner_upload_dataset_files( - self, tag: str, content_hash: str, files: dict[str, bytes] + def owner_upload_collection_files( + self, prefix: str, tag: str, content_hash: str, files: dict[str, bytes] ) -> None: - """Upload dataset files to collection folder.""" - folder_id = self._get_dataset_collection_folder_id(tag, content_hash) + """Upload files to a collection folder.""" + folder_id = self._get_collection_folder_id(prefix, tag, content_hash) for file_path, content in files.items(): file_payload, _ = self.create_file_payload(content) @@ -1582,11 +1577,11 @@ def owner_upload_dataset_files( ) ) - def owner_list_dataset_collections(self) -> list[str]: - """List collections created by DO (owned by me).""" + def owner_list_collections(self, prefix: str) -> list[str]: + """List collections created by DO (owned by me). Returns list of tags.""" syftbox_folder_id = self.get_syftbox_folder_id() query = ( - f"name contains '{DATASET_COLLECTION_PREFIX}_' and '{syftbox_folder_id}' in parents " + f"name contains '{prefix}_' and '{syftbox_folder_id}' in parents " f"and 'me' in owners and trashed=false and mimeType='{GOOGLE_FOLDER_MIME_TYPE}'" ) results = execute_with_retries( @@ -1597,19 +1592,20 @@ def owner_list_dataset_collections(self) -> list[str]: result = [] for folder in folders: try: - folder_obj = DatasetCollectionFolder.from_name(folder["name"]) - result.append(folder_obj.tag) + cf = CollectionFolder.from_name(prefix, folder["name"]) + result.append(cf.tag) except ValueError: continue return result - def owner_list_all_dataset_collections_with_permissions( + def owner_list_all_collections_with_permissions( self, + prefix: str, ) -> list[FileCollection]: - """List all DO's dataset collections with permissions info.""" + """List all DO's collections with permissions info.""" syftbox_folder_id = self.get_syftbox_folder_id() query = ( - f"name contains '{DATASET_COLLECTION_PREFIX}_' and '{syftbox_folder_id}' in parents " + f"name contains '{prefix}_' and '{syftbox_folder_id}' in parents " f"and 'me' in owners and trashed=false and mimeType='{GOOGLE_FOLDER_MIME_TYPE}'" ) results = execute_with_retries( @@ -1622,7 +1618,7 @@ def owner_list_all_dataset_collections_with_permissions( for folder in results.get("files", []): folder_id = folder["id"] try: - folder_obj = DatasetCollectionFolder.from_name(folder["name"]) + cf = CollectionFolder.from_name(prefix, folder["name"]) has_anyone = ( folder.get("appProperties", {}).get("syft_shared_with_any") == "true" @@ -1630,8 +1626,8 @@ def owner_list_all_dataset_collections_with_permissions( collections.append( FileCollection( folder_id=folder_id, - tag=folder_obj.tag, - content_hash=folder_obj.content_hash, + tag=cf.tag, + content_hash=cf.content_hash, has_any_permission=has_anyone, ) ) @@ -1640,22 +1636,24 @@ def owner_list_all_dataset_collections_with_permissions( return collections - def owner_delete_dataset_collection(self, tag: str) -> None: - """Delete all public dataset collection folders matching the given tag.""" - collections = self.owner_list_all_dataset_collections_with_permissions() + def owner_delete_collection(self, prefix: str, tag: str) -> None: + """Delete all collection folders (for prefix) matching the given tag.""" + collections = self.owner_list_all_collections_with_permissions(prefix) for c in collections: if c.tag == tag: self.delete_file_by_id(c.folder_id) - cache_key = f"{c.tag}_{c.content_hash}" - self.dataset_collection_folder_id_cache.pop(cache_key, None) + folder_name = CollectionFolder( + prefix=prefix, tag=c.tag, content_hash=c.content_hash + ).folder_name + self.collection_folder_id_cache.pop(folder_name, None) - def watcher_list_dataset_collections(self) -> list[dict]: + def watcher_list_collections(self, prefix: str) -> list[dict]: """List collections shared with DS (not owned by me). Returns list of dicts with keys: owner_email, tag, content_hash """ query = ( - f"name contains '{DATASET_COLLECTION_PREFIX}_' and not 'me' in owners " + f"name contains '{prefix}_' and not 'me' in owners " f"and trashed=false and mimeType='{GOOGLE_FOLDER_MIME_TYPE}'" ) results = execute_with_retries( @@ -1666,15 +1664,15 @@ def watcher_list_dataset_collections(self) -> list[dict]: result = [] for folder in folders: try: - folder_obj = DatasetCollectionFolder.from_name(folder["name"]) + cf = CollectionFolder.from_name(prefix, folder["name"]) owner_email = folder.get("owners", [{}])[0].get( "emailAddress", "unknown" ) result.append( { "owner_email": owner_email, - "tag": folder_obj.tag, - "content_hash": folder_obj.content_hash, + "tag": cf.tag, + "content_hash": cf.content_hash, } ) except ValueError: @@ -1682,12 +1680,13 @@ def watcher_list_dataset_collections(self) -> list[dict]: continue return result - def watcher_download_dataset_collection( - self, tag: str, content_hash: str, owner_email: str + def watcher_download_collection( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> dict[str, bytes]: - """Download all files from a dataset collection.""" - folder_obj = DatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() + """Download all files from a collection.""" + folder_name = CollectionFolder( + prefix=prefix, tag=tag, content_hash=content_hash + ).folder_name # Try to find folder by name (could be owned by someone else) folder_id = self._find_folder_by_name(folder_name, owner_email=owner_email) @@ -1703,12 +1702,13 @@ def watcher_download_dataset_collection( return files - def watcher_get_dataset_collection_file_metadatas( - self, tag: str, content_hash: str, owner_email: str + def watcher_get_collection_file_metadatas( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> List[Dict]: - """Get file metadata from a dataset collection without downloading.""" - folder_obj = DatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() + """Get file metadata from a collection without downloading.""" + folder_name = CollectionFolder( + prefix=prefix, tag=tag, content_hash=content_hash + ).folder_name folder_id = self._find_folder_by_name(folder_name, owner_email=owner_email) if not folder_id: @@ -1717,142 +1717,29 @@ def watcher_get_dataset_collection_file_metadatas( file_metadatas = self.get_file_metadatas_from_folder(folder_id) return [{"file_id": f["id"], "file_name": f["name"]} for f in file_metadatas] - def watcher_download_dataset_file(self, file_id: str) -> bytes: - """Download a single file from a dataset collection.""" + def watcher_download_collection_file(self, file_id: str) -> bytes: + """Download a single file from a collection.""" return self.download_file(file_id) - def _get_dataset_collection_folder_id(self, tag: str, content_hash: str) -> str: - """Get folder ID for dataset collection, with caching.""" - cache_key = f"{tag}_{content_hash}" - if cache_key in self.dataset_collection_folder_id_cache: - return self.dataset_collection_folder_id_cache[cache_key] - - folder_obj = DatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() - syftbox_folder_id = self.get_syftbox_folder_id() - folder_id = self._find_folder_by_name(folder_name, parent_id=syftbox_folder_id) - - if not folder_id: - raise ValueError( - f"Collection folder {tag} with hash {content_hash} not found" - ) - - self.dataset_collection_folder_id_cache[cache_key] = folder_id - return folder_id - - # ========================================================================= - # PRIVATE DATASET COLLECTION METHODS - # ========================================================================= - - def owner_create_private_dataset_collection_folder( - self, tag: str, content_hash: str, owner_email: str + def _get_collection_folder_id( + self, prefix: str, tag: str, content_hash: str ) -> str: - """Create /SyftBox/{PRIVATE_DATASET_COLLECTION_PREFIX}_{tag}_{hash} folder. - - No sharing is applied — only the owner can access this folder. - """ - folder_obj = PrivateDatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() - cache_key = f"private_{tag}_{content_hash}" - - if cache_key in self.dataset_collection_folder_id_cache: - return self.dataset_collection_folder_id_cache[cache_key] + """Get folder ID for a collection, with caching.""" + folder_name = CollectionFolder( + prefix=prefix, tag=tag, content_hash=content_hash + ).folder_name + if folder_name in self.collection_folder_id_cache: + return self.collection_folder_id_cache[folder_name] syftbox_folder_id = self.get_syftbox_folder_id() folder_id = self._find_folder_by_name(folder_name, parent_id=syftbox_folder_id) - if folder_id: - self.dataset_collection_folder_id_cache[cache_key] = folder_id - return folder_id - - folder_id = self.create_folder(folder_name, syftbox_folder_id) - self.dataset_collection_folder_id_cache[cache_key] = folder_id - return folder_id - - def owner_upload_private_dataset_files( - self, tag: str, content_hash: str, files: dict[str, bytes] - ) -> None: - """Upload files to a private dataset collection folder.""" - folder_id = self._get_private_collection_folder_id(tag, content_hash) - for file_path, content in files.items(): - file_payload, _ = self.create_file_payload(content) - file_name = Path(file_path).name - file_metadata = {"name": file_name, "parents": [folder_id]} - execute_with_retries( - self.drive_service.files().create( - body=file_metadata, media_body=file_payload, fields="id" - ) - ) - - def owner_list_private_dataset_collections(self) -> list[FileCollection]: - """List private collections owned by DO.""" - syftbox_folder_id = self.get_syftbox_folder_id() - query = ( - f"name contains '{PRIVATE_DATASET_COLLECTION_PREFIX}_' " - f"and '{syftbox_folder_id}' in parents " - f"and 'me' in owners and trashed=false " - f"and mimeType='{GOOGLE_FOLDER_MIME_TYPE}'" - ) - results = execute_with_retries( - self.drive_service.files().list(q=query, fields="files(id,name)") - ) - - collections = [] - for folder in results.get("files", []): - try: - folder_obj = PrivateDatasetCollectionFolder.from_name(folder["name"]) - collections.append( - FileCollection( - folder_id=folder["id"], - tag=folder_obj.tag, - content_hash=folder_obj.content_hash, - ) - ) - except ValueError: - continue - return collections - - def owner_delete_private_dataset_collection(self, tag: str) -> None: - """Delete all private dataset collection folders matching the given tag.""" - collections = self.owner_list_private_dataset_collections() - for c in collections: - if c.tag == tag: - self.delete_file_by_id(c.folder_id) - cache_key = f"private_{c.tag}_{c.content_hash}" - self.dataset_collection_folder_id_cache.pop(cache_key, None) - - def owner_get_private_collection_file_metadatas( - self, tag: str, content_hash: str, owner_email: str - ) -> List[Dict]: - """Get file metadata from a private dataset collection without downloading.""" - folder_obj = PrivateDatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() - folder_id = self._find_folder_by_name(folder_name, owner_email=owner_email) if not folder_id: raise ValueError( - f"Private collection {tag} with hash {content_hash} not found" - ) - - file_metadatas = self.get_file_metadatas_from_folder(folder_id) - return [{"file_id": f["id"], "file_name": f["name"]} for f in file_metadatas] - - def _get_private_collection_folder_id(self, tag: str, content_hash: str) -> str: - """Get folder ID for private dataset collection, with caching.""" - cache_key = f"private_{tag}_{content_hash}" - if cache_key in self.dataset_collection_folder_id_cache: - return self.dataset_collection_folder_id_cache[cache_key] - - folder_obj = PrivateDatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() - syftbox_folder_id = self.get_syftbox_folder_id() - folder_id = self._find_folder_by_name(folder_name, parent_id=syftbox_folder_id) - - if not folder_id: - raise ValueError( - f"Private collection folder {tag} with hash {content_hash} not found" + f"Collection folder {tag} with hash {content_hash} not found" ) - self.dataset_collection_folder_id_cache[cache_key] = folder_id + self.collection_folder_id_cache[folder_name] = folder_id return folder_id def _get_version_file_id(self) -> Optional[str]: diff --git a/syft_client/sync/syftbox_manager.py b/syft_client/sync/syftbox_manager.py index 4e9837d8a79..e7a930bff65 100644 --- a/syft_client/sync/syftbox_manager.py +++ b/syft_client/sync/syftbox_manager.py @@ -2,6 +2,7 @@ import fcntl from syft_client.sync.peers.peer_store import PeerStore from syft_client.sync.utils.path_filters import is_normal_syncable_path +from syft_client.sync.callback_mixin import BaseModelCallbackMixin import logging import shutil from contextlib import contextmanager @@ -10,15 +11,10 @@ from concurrent.futures import ThreadPoolExecutor import time from pydantic import ConfigDict -from syft_job.client import BaseJobClient, JobClient -from syft_job.job import JobsList -from syft_job.job_runner import SyftJobRunner -from syft_job import SyftJobConfig -from syft_datasets.config import SyftBoxConfig -from syft_datasets.dataset_manager import SyftDatasetManager from syft_client.sync.platforms.base_platform import BasePlatform from pydantic import BaseModel, PrivateAttr from typing import List, Optional, cast +from syft_client.sync.sync.collection_spec import CollectionSyncSpec from syft_client.sync.sync.caches.datasite_watcher_cache import ( DataSiteWatcherCacheConfig, ) @@ -34,7 +30,6 @@ FileChangeEvent, FileChangeEventsMessage, ) -from syft_client.sync.utils.pre_submit_scan import run_pre_submit_check from syft_client.sync.utils.syftbox_utils import ( random_email, random_syftbox_folder_for_testing, @@ -56,7 +51,6 @@ DatasiteWatcherSyncerConfig, ) from syft_client.sync.version.peer_manager import ( - CompatAction, PeerManager, PeerManagerConfig, ) @@ -68,7 +62,6 @@ COLAB_DEFAULT_SYFTBOX_FOLDER = Path("/") JUPYTER_DEFAULT_SYFTBOX_FOLDER = Path.home() / "SyftBox" -COLLECTION_SUBPATH = Path("public/syft_datasets") # ANSI codes for highlighting important warnings in terminals / notebooks. _ANSI_RED = "\033[1;91m" @@ -95,8 +88,6 @@ class SyftboxManagerConfig(BaseModel): peer_manager_config: PeerManagerConfig datasite_watcher_syncer_config: DatasiteWatcherSyncerConfig - dataset_manager_config: SyftBoxConfig - job_client_config: SyftJobConfig @classmethod def for_colab( @@ -110,18 +101,29 @@ def for_colab( bool ] = None, # None: value is determined by the role force_ignore_peer_version: bool = False, + collection_specs: list["CollectionSyncSpec"] | None = None, ): if not has_ds_role and not has_do_role: raise ValueError("At least one of has_ds_role or has_do_role must be True") + # No collection specs by default: the generic sync engine knows nothing + # about custom collections (eg: datasets or jobs). + if collection_specs is None: + collection_specs = [] + syftbox_folder = get_colab_default_syftbox_folder(email) use_in_memory_cache = False - collections_folder = syftbox_folder / email / COLLECTION_SUBPATH + collections_folder = ( + syftbox_folder / email / collection_specs[0].local_subpath + if collection_specs + else None + ) connection_configs = [GdriveConnectionConfig(email=email, token_path=None)] datasite_owner_syncer_config = DatasiteOwnerSyncerConfig( email=email, syftbox_folder=syftbox_folder, collections_folder=collections_folder, + collection_specs=collection_specs, connection_configs=connection_configs, cache_config=DataSiteOwnerEventCacheConfig( email=email, @@ -138,19 +140,10 @@ def for_colab( email=email, use_in_memory_cache=use_in_memory_cache, syftbox_folder=syftbox_folder, - collection_subpath=COLLECTION_SUBPATH, + collection_specs=collection_specs, connection_configs=connection_configs, ), ) - job_client_config = SyftJobConfig( - syftbox_folder=syftbox_folder, - current_user_email=email, - has_do_role=has_do_role, - ) - dataset_manager_config = SyftBoxConfig( - syftbox_folder=syftbox_folder, - email=email, - ) peer_manager_config = PeerManagerConfig( syftbox_folder=syftbox_folder, email=email, @@ -171,8 +164,6 @@ def for_colab( use_in_memory_cache=False, datasite_owner_syncer_config=datasite_owner_syncer_config, datasite_watcher_syncer_config=datasite_watcher_syncer_config, - dataset_manager_config=dataset_manager_config, - job_client_config=job_client_config, peer_manager_config=peer_manager_config, ) @@ -189,12 +180,23 @@ def for_jupyter( bool ] = None, # None: value is determined by the role force_ignore_peer_version: bool = False, + collection_specs: list["CollectionSyncSpec"] | None = None, ): if not has_ds_role and not has_do_role: raise ValueError("At least one of has_ds_role or has_do_role must be True") + # No collection specs by default: the generic sync engine knows nothing + # about datasets. syft-rds registers its dataset spec at initialization + # (see syft_rds.config.DATASET_COLLECTION_SPECS). + if collection_specs is None: + collection_specs = [] + syftbox_folder = get_jupyter_default_syftbox_folder(email) - collections_folder = syftbox_folder / email / COLLECTION_SUBPATH + collections_folder = ( + syftbox_folder / email / collection_specs[0].local_subpath + if collection_specs + else None + ) connection_configs = [ GdriveConnectionConfig(email=email, token_path=token_path) @@ -203,6 +205,7 @@ def for_jupyter( email=email, syftbox_folder=syftbox_folder, collections_folder=collections_folder, + collection_specs=collection_specs, connection_configs=connection_configs, cache_config=DataSiteOwnerEventCacheConfig( email=email, @@ -220,19 +223,10 @@ def for_jupyter( email=email, use_in_memory_cache=False, syftbox_folder=syftbox_folder, - collection_subpath=COLLECTION_SUBPATH, + collection_specs=collection_specs, connection_configs=connection_configs, ), ) - dataset_manager_config = SyftBoxConfig( - syftbox_folder=syftbox_folder, - email=email, - ) - job_client_config = SyftJobConfig( - syftbox_folder=syftbox_folder, - current_user_email=email, - has_do_role=has_do_role, - ) peer_manager_config = PeerManagerConfig( syftbox_folder=syftbox_folder, email=email, @@ -252,8 +246,6 @@ def for_jupyter( use_in_memory_cache=False, datasite_owner_syncer_config=datasite_owner_syncer_config, datasite_watcher_syncer_config=datasite_watcher_syncer_config, - dataset_manager_config=dataset_manager_config, - job_client_config=job_client_config, peer_manager_config=peer_manager_config, ) @@ -267,15 +259,27 @@ def _base_config_for_testing( has_do_role: bool = False, use_in_memory_cache: bool = True, check_versions: bool = False, + collection_specs: list["CollectionSyncSpec"] | None = None, ): + # No collection specs by default: the generic sync engine knows nothing + # about datasets. syft-rds registers its dataset spec at initialization + # (see syft_rds.config.DATASET_COLLECTION_SPECS). + if collection_specs is None: + collection_specs = [] + syftbox_folder = syftbox_folder or random_syftbox_folder_for_testing() email = email or random_email() - collections_folder = syftbox_folder / email / COLLECTION_SUBPATH + collections_folder = ( + syftbox_folder / email / collection_specs[0].local_subpath + if collection_specs + else None + ) datasite_owner_syncer_config = DatasiteOwnerSyncerConfig( email=email, syftbox_folder=syftbox_folder, collections_folder=collections_folder, + collection_specs=collection_specs, write_files=write_files, cache_config=DataSiteOwnerEventCacheConfig( email=email, @@ -291,19 +295,10 @@ def _base_config_for_testing( email=email, use_in_memory_cache=use_in_memory_cache, syftbox_folder=syftbox_folder, - collection_subpath=COLLECTION_SUBPATH, + collection_specs=collection_specs, ), ) - dataset_manager_config = SyftBoxConfig( - syftbox_folder=syftbox_folder, - email=email, - ) - job_client_config = SyftJobConfig( - syftbox_folder=Path(syftbox_folder), - current_user_email=email, - has_do_role=has_do_role, - ) peer_manager_config = PeerManagerConfig( syftbox_folder=syftbox_folder, email=email, @@ -323,8 +318,6 @@ def _base_config_for_testing( use_in_memory_cache=use_in_memory_cache, datasite_owner_syncer_config=datasite_owner_syncer_config, datasite_watcher_syncer_config=datasite_watcher_syncer_config, - dataset_manager_config=dataset_manager_config, - job_client_config=job_client_config, peer_manager_config=peer_manager_config, ) @@ -339,10 +332,21 @@ def for_google_drive_testing_connection( has_do_role: bool = False, use_in_memory_cache: bool = True, check_versions: bool = False, + collection_specs: list["CollectionSyncSpec"] | None = None, ): + # No collection specs by default: the generic sync engine knows nothing + # about datasets. syft-rds registers its dataset spec at initialization + # (see syft_rds.config.DATASET_COLLECTION_SPECS). + if collection_specs is None: + collection_specs = [] + syftbox_folder = syftbox_folder or random_syftbox_folder_for_testing() email = email or random_email() - collections_folder = Path(syftbox_folder) / email / COLLECTION_SUBPATH + collections_folder = ( + Path(syftbox_folder) / email / collection_specs[0].local_subpath + if collection_specs + else None + ) connection_configs = [ GdriveConnectionConfig(email=email, token_path=token_path) ] @@ -350,6 +354,7 @@ def for_google_drive_testing_connection( email=email, syftbox_folder=syftbox_folder, collections_folder=collections_folder, + collection_specs=collection_specs, connection_configs=connection_configs, cache_config=DataSiteOwnerEventCacheConfig( email=email, @@ -366,20 +371,11 @@ def for_google_drive_testing_connection( email=email, use_in_memory_cache=use_in_memory_cache, syftbox_folder=syftbox_folder, - collection_subpath=COLLECTION_SUBPATH, + collection_specs=collection_specs, connection_configs=connection_configs, ), ) - dataset_manager_config = SyftBoxConfig( - syftbox_folder=syftbox_folder, - email=email, - ) - job_client_config = SyftJobConfig( - syftbox_folder=syftbox_folder, - current_user_email=email, - has_do_role=has_do_role, - ) peer_manager_config = PeerManagerConfig( syftbox_folder=syftbox_folder, email=email, @@ -397,13 +393,11 @@ def for_google_drive_testing_connection( has_ds_role=has_ds_role, has_do_role=has_do_role, use_in_memory_cache=False, - dataset_manager_config=dataset_manager_config, - job_client_config=job_client_config, peer_manager_config=peer_manager_config, ) -class SyftboxManager(BaseModel): +class SyftboxManager(BaseModelCallbackMixin): # needed for peers model_config = ConfigDict(arbitrary_types_allowed=True) @@ -415,9 +409,6 @@ class SyftboxManager(BaseModel): datasite_owner_syncer: DatasiteOwnerSyncer | None = None job_file_change_handler: JobFileChangeHandler | None = None - dataset_manager: SyftDatasetManager | None = None - job_client: BaseJobClient | None = None - job_runner: SyftJobRunner | None = None peer_manager: PeerManager | None = None has_do_role: bool = False has_ds_role: bool = False @@ -435,22 +426,13 @@ class SyftboxManager(BaseModel): "config", "has_do_role", "has_ds_role", - "dataset_manager", "peer_manager", "peers", - "jobs", - "datasets", "add_peer", "load_peers", "approve_peer_request", "reject_peer_request", "sync", - "create_dataset", - "delete_dataset", - "share_dataset", - "submit_bash_job", - "submit_python_job", - "process_approved_jobs", "create_checkpoint", "should_create_checkpoint", "try_create_checkpoint", @@ -509,10 +491,6 @@ def from_config(cls, config: SyftboxManagerConfig): datasite_owner_syncer = None job_file_change_handler = None datasite_watcher_syncer = None - job_runner = None - - dataset_manager = SyftDatasetManager.from_config(config.dataset_manager_config) - job_client = JobClient.from_config(config.job_client_config) if config.has_do_role: datasite_owner_syncer = DatasiteOwnerSyncer.from_config( @@ -520,7 +498,6 @@ def from_config(cls, config: SyftboxManagerConfig): ) job_file_change_handler = JobFileChangeHandler() - job_runner = SyftJobRunner.from_config(config.job_client_config) if config.has_ds_role: datasite_watcher_syncer = DatasiteWatcherSyncer.from_config( @@ -538,9 +515,6 @@ def from_config(cls, config: SyftboxManagerConfig): datasite_owner_syncer=datasite_owner_syncer, job_file_change_handler=job_file_change_handler, datasite_watcher_syncer=datasite_watcher_syncer, - dataset_manager=dataset_manager, - job_client=job_client, - job_runner=job_runner, peer_manager=peer_manager, has_do_role=config.has_do_role, has_ds_role=config.has_ds_role, @@ -641,6 +615,7 @@ def _pair_with_google_drive_testing_connection( use_in_memory_cache: bool = True, clear_caches: bool = True, check_versions: bool = False, + collection_specs: list["CollectionSyncSpec"] | None = None, ): receiver_config = SyftboxManagerConfig.for_google_drive_testing_connection( email=do_email, @@ -650,6 +625,7 @@ def _pair_with_google_drive_testing_connection( has_ds_role=False, has_do_role=True, check_versions=check_versions, + collection_specs=collection_specs, ) receiver_manager = cls.from_config(receiver_config) @@ -662,6 +638,7 @@ def _pair_with_google_drive_testing_connection( has_ds_role=True, has_do_role=False, check_versions=check_versions, + collection_specs=collection_specs, ) sender_manager = cls.from_config(sender_config) @@ -722,6 +699,7 @@ def pair_with_mock_drive_service_connection( use_in_memory_cache: bool = True, check_versions: bool = False, encryption: bool = False, + collection_specs: list["CollectionSyncSpec"] | None = None, ): """Create a pair of managers using mock Google Drive services for testing. @@ -750,6 +728,7 @@ def pair_with_mock_drive_service_connection( has_do_role=True, use_in_memory_cache=use_in_memory_cache, check_versions=check_versions, + collection_specs=collection_specs, ) ds_config = SyftboxManagerConfig._base_config_for_testing( @@ -759,6 +738,7 @@ def pair_with_mock_drive_service_connection( has_do_role=False, use_in_memory_cache=use_in_memory_cache, check_versions=check_versions, + collection_specs=collection_specs, ) # Create managers from configs @@ -822,84 +802,12 @@ def add_peer( ): """Add a peer. Delegates to PeerManager.""" self.peer_manager.add_peer(peer_email, force=force, verbose=verbose) - self._sync_peer_install_sources_to_job_client() + self._emit_peers_loaded() if self.has_do_role: self._post_approve_peer_do(peer_email) if sync: self.sync() - def submit_bash_job( - self, - user: str, - script: str, - job_name: str = "", - sync=True, - force_submission: bool = False, - ignore_peer_version: bool = False, - ): - # Check version compatibility before submission (uses cached versions) - if not force_submission: - result = self.peer_manager.get_peer_compatibility_status( - user, - action=CompatAction.SUBMIT, - ignore_peer_version=ignore_peer_version, - ) - result.raise_on_skip(operation="submit job") - result.maybe_warn() - job_dir = self.job_client.submit_bash_job(user, script, job_name=job_name) - self.push_job_files(job_dir) - - def submit_python_job( - self, - user: str, - code_path: str, - job_name: str | None = "", - dependencies: list[str] | None = None, - entrypoint: str | None = None, - sync=True, - force_submission: bool = False, - ignore_peer_version: bool = False, - ): - peer_emails = {p.email for p in self.peer_manager.syncable_peers} - if user not in peer_emails: - print(f"⚠️ {user} is not in your peer list.") - print(f" Add them first with: client.add_peer('{user}')") - return - - if not force_submission: - if not run_pre_submit_check(Path(code_path)): - print("Submission aborted.") - return - - print(f"📤 Submitting '{code_path}' to {user}...") - if job_name: - print(f" Job name : {job_name}") - if dependencies: - print(f" Dependencies : {', '.join(dependencies)}") - - # Check version compatibility before submission (uses cached versions) - if not force_submission: - result = self.peer_manager.get_peer_compatibility_status( - user, - action=CompatAction.SUBMIT, - ignore_peer_version=ignore_peer_version, - ) - result.raise_on_skip(operation="submit job") - result.maybe_warn() - job_dir = self.job_client.submit_python_job( - user, - code_path, - job_name=job_name, - dependencies=dependencies, - entrypoint=entrypoint, - ) - self.push_job_files(job_dir) - - print("\n✅ Job submitted successfully!") - print(" Status : inbox (waiting for DO to review)") - print(f"\n⏳ Next step: wait for {user} to approve and run it.") - print(" Check progress with: client.jobs") - def push_job_files(self, job_dir: Path): file_paths = [Path(p) for p in job_dir.rglob("*") if p.is_file()] relative_file_paths = [p.relative_to(self.syftbox_folder) for p in file_paths] @@ -995,7 +903,7 @@ def load_peers(self, force_download: bool = False): instead of using the cached copy. """ cast(PeerManager, self.peer_manager).load_peers(force_download=force_download) - self._sync_peer_install_sources_to_job_client() + self._emit_peers_loaded() def _check_peer_request_exists(self, email: str) -> bool: """Check if a peer request exists. Delegates to PeerManager.""" @@ -1011,133 +919,22 @@ def approve_peer_request( self.peer_manager.approve_peer_request( email_or_peer, verbose=verbose, peer_must_exist=peer_must_exist ) - self._sync_peer_install_sources_to_job_client() self._post_approve_peer_do(email_or_peer) def _post_approve_peer_do(self, email_or_peer: str | Peer): - """Shared post-approval logic: job folder setup and dataset sharing.""" peer_email = ( email_or_peer if isinstance(email_or_peer, str) else email_or_peer.email ) + self._emit_peers_loaded() + self._emit("peer_approved", peer_email) - if self.has_do_role: - self.job_client.setup_ds_job_folder_as_do(peer_email) - self._share_any_datasets_with_peer(peer_email) - - def _sync_peer_install_sources_to_job_client(self) -> None: - """Copy each peer's advertised syft-client install source into job_client. - - Called after peer version exchanges so that, when the DS submits a job - to a DO, the run.sh references the DO's local install path rather than - the DS's local detection. - """ - if not self.job_client: - return - for peer in self.peer_manager.peer_store.syncable_peers: - source = peer.version.syft_client_install_source if peer.version else None - if source: - self.job_client.peer_install_sources[peer.email] = source - - def _ensure_local_peer_permissions(self) -> None: - """Recreate local permission files for all approved peers. - - After an upgrade that deletes local state, permission files - (syft.pub.yaml) are lost. This restores them so that incoming - proposals from approved peers are not silently rejected. - """ - if not self.has_do_role: - return - for peer in self.peer_manager.approved_peers: - self.job_client.setup_ds_job_folder_as_do(peer.email) - - def _share_any_datasets_with_peer(self, peer_email: str): - """Share all datasets that have 'any' permission with a specific peer. - - This is needed because Google Drive "anyone with link" files are not - discoverable via search. By adding explicit user sharing, the peer - can discover these datasets. - - Uses cache populated during pull_initial_state() in DatasiteOwnerSyncer. - """ - for tag, content_hash in self.datasite_owner_syncer._any_shared_datasets: - try: - self._connection_router.owner_share_dataset_collection( - tag, content_hash, [peer_email] - ) - except Exception: - # Ignore errors (e.g., already shared) - pass + def _emit_peers_loaded(self) -> None: + self._emit("peers_loaded") def reject_peer_request(self, email_or_peer: str | Peer): """Reject a pending peer request. Delegates to PeerManager.""" self.peer_manager.reject_peer_request(email_or_peer) - @property - def jobs(self) -> JobsList: - """ - Get the list of jobs. Automatically calls sync() before returning jobs - if PRE_SYNC environment variable is set to "true" (case-insensitive). - - PRE_SYNC defaults to "true", so auto-sync is enabled by default. - To disable auto-sync, set: PRE_SYNC=false - """ - if os.environ.get("PRE_SYNC", "true").lower() == "true": - self.sync() - return self.job_client.jobs - - def process_approved_jobs( - self, - stream_output: bool = True, - timeout: int | None = None, - force_execution: bool = False, - share_outputs_with_submitter: bool = False, - share_logs_with_submitter: bool = False, - ignore_peer_version: bool = False, - ) -> None: - """ - Process approved jobs. Automatically calls sync() after processing - - Args: - stream_output: If True (default), stream output in real-time. - If False, capture output at end. - timeout: Timeout in seconds per job. Defaults to 300 (5 minutes). - Can also be set via SYFT_DEFAULT_JOB_TIMEOUT_SECONDS env var. - force_execution: If True, process all approved jobs regardless of - version compatibility. If False (default), skip jobs - from peers with incompatible or unknown versions. - share_outputs_with_submitter: If True, grant read access on outputs to submitter. - share_logs_with_submitter: If True, grant read access on logs to submitter. - - PRE_SYNC defaults to "true", so auto-sync is enabled by default. - To disable auto-sync, set: PRE_SYNC=false - """ - skip_job_names = [] - - if not force_execution: - approved_jobs = [ - job for job in self.job_client.jobs if job.status == "approved" - ] - for job in approved_jobs: - result = self.peer_manager.get_peer_compatibility_status( - job.submitted_by, - action=CompatAction.EXECUTE, - ignore_peer_version=ignore_peer_version, - ) - result.maybe_warn() - if result.should_skip: - skip_job_names.append(job.name) - - self.job_runner.process_approved_jobs( - stream_output=stream_output, - timeout=timeout, - skip_job_names=skip_job_names if skip_job_names else None, - share_outputs_with_submitter=share_outputs_with_submitter, - share_logs_with_submitter=share_logs_with_submitter, - ) - - if os.environ.get("PRE_SYNC", "true").lower() == "true": - self.sync() - def _add_connection(self, connection: SyftboxPlatformConnection): if not ( isinstance(connection, GDriveConnection) @@ -1166,317 +963,6 @@ def _send_file_change(self, path: str | Path, content: str): def _get_all_accepted_events_do(self) -> List[FileChangeEvent]: return self.datasite_owner_syncer.connection_router.owner_get_all_accepted_events_messages() - def create_dataset( - self, - name: str, - mock_path: str | Path, - private_path: str | Path, - summary: str | None = None, - readme_path: Path | None = None, - location: str | None = None, - tags: list[str] | None = None, - users: list[str] | str | None = None, - upload_private: bool = False, - sync=True, - ): - if self.dataset_manager is None: - raise ValueError("Dataset manager is not set") - - # Only DO can create datasets - if not self.has_do_role: - raise ValueError("Only dataset owners can create datasets") - - # Convert None to empty list - if users is None: - users = [] - - dataset_name = None - created_local = False - mock_folder_id = None - private_folder_id = None - - try: - # Create dataset locally - dataset = self.dataset_manager.create( - name=name, - mock_path=mock_path, - private_path=private_path, - summary=summary, - readme_path=readme_path, - location=location, - tags=tags, - users=users, - ) - created_local = True - dataset_name = dataset.name - - # Upload mock data to collection folder - mock_folder_id = self._upload_dataset_to_collection(dataset, users) - - # Upload private data to a separate owner-only collection - if upload_private: - private_folder_id = self._upload_private_dataset_to_collection(dataset) - - if sync: - self.sync() - - return dataset - - except Exception: - logger.error( - "Failed to create dataset%s, cleaning up", - f" '{dataset_name}'" if dataset_name else "", - ) - self._cleanup_failed_dataset_creation( - dataset_name, created_local, mock_folder_id, private_folder_id - ) - raise - - def _cleanup_failed_dataset_creation( - self, - dataset_name: str | None, - created_local: bool, - mock_folder_id: str | None, - private_folder_id: str | None, - ) -> None: - """Best-effort cleanup after a failed create_dataset, in reverse order.""" - if private_folder_id is not None: - try: - self._connection_router.delete_file_by_id(private_folder_id) - except Exception: - logger.warning( - "Cleanup: failed to delete private GDrive folder %s", - private_folder_id, - ) - - if mock_folder_id is not None: - try: - self._connection_router.delete_file_by_id(mock_folder_id) - except Exception: - logger.warning( - "Cleanup: failed to delete mock GDrive folder %s", - mock_folder_id, - ) - - if created_local and dataset_name is not None: - try: - self.dataset_manager.delete(dataset_name, require_confirmation=False) - except Exception: - logger.warning( - "Cleanup: failed to delete local dataset '%s'", - dataset_name, - ) - - def _upload_dataset_to_collection(self, dataset, users: list[str] | str) -> str: - """Upload dataset files to collection folder. Returns the folder ID.""" - from syft_client.sync.connections.drive.gdrive_transport import ( - DatasetCollectionFolder, - ) - - collection_tag = dataset.name - - # Prepare files to upload - files = {} - for mock_file in dataset.mock_files: - if mock_file.exists(): - files[mock_file.name] = mock_file.read_bytes() - - metadata_path = dataset.mock_dir / "dataset.yaml" - if metadata_path.exists(): - files["dataset.yaml"] = metadata_path.read_bytes() - - if dataset.readme_path and dataset.readme_path.exists(): - files[dataset.readme_path.name] = dataset.readme_path.read_bytes() - - # Compute content hash - content_hash = DatasetCollectionFolder.compute_hash(files) - - # Create collection folder with hash in name - folder_id = self._connection_router.owner_create_dataset_collection_folder( - tag=collection_tag, content_hash=content_hash, owner_email=self.email - ) - - # Upload files - self._connection_router.owner_upload_dataset_files( - collection_tag, content_hash, files - ) - - # Share with users - if users == "any": - self._connection_router.owner_tag_dataset_collection_as_any( - collection_tag, content_hash - ) - self.datasite_owner_syncer._any_shared_datasets.append( - (collection_tag, content_hash) - ) - # Share with all already-approved peers - peer_emails = [p.email for p in self.peer_manager.approved_peers] - if peer_emails: - self._connection_router.owner_share_dataset_collection( - collection_tag, content_hash, peer_emails - ) - else: - self._connection_router.owner_share_dataset_collection( - collection_tag, content_hash, users - ) - - return folder_id - - def _upload_private_dataset_to_collection(self, dataset) -> str | None: - """Upload private dataset files to a separate owner-only collection folder. - Returns the folder ID, or None if no files to upload.""" - from syft_client.sync.connections.drive.gdrive_transport import ( - PrivateDatasetCollectionFolder, - ) - - collection_tag = dataset.name - - # Collect all files in private dir (data, metadata, permissions) - files = {} - for f in dataset.private_dir.iterdir(): - if f.is_file(): - files[f.name] = f.read_bytes() - - if not files: - return None - - content_hash = PrivateDatasetCollectionFolder.compute_hash(files) - - # Create private collection folder (no sharing) - folder_id = ( - self._connection_router.owner_create_private_dataset_collection_folder( - tag=collection_tag, content_hash=content_hash, owner_email=self.email - ) - ) - - # Upload files - self._connection_router.owner_upload_private_dataset_files( - collection_tag, content_hash, files - ) - - return folder_id - - def delete_dataset( - self, - name: str, - datasite: str | None = None, - require_confirmation: bool = True, - sync=True, - ): - if self.dataset_manager is None: - raise ValueError("Dataset manager is not set") - self.dataset_manager.delete( - name=name, - datasite=datasite, - require_confirmation=require_confirmation, - ) - # Delete collection folders from Google Drive so DS peers - # pick up the deletion on their next sync. - try: - self._connection_router.owner_delete_dataset_collection(name) - except Exception: - logger.warning("Failed to delete dataset collection '%s' from Drive", name) - try: - self._connection_router.owner_delete_private_dataset_collection(name) - except Exception: - logger.warning( - "Failed to delete private dataset collection '%s' from Drive", - name, - ) - if sync: - self.sync() - - def share_dataset(self, tag: str, users: list[str] | str, sync=True): - """ - Share an existing dataset with additional users. - - Args: - tag: Dataset name - users: List of email addresses or "any" - sync: Whether to sync after sharing - """ - from syft_client.sync.connections.drive.gdrive_transport import ( - DatasetCollectionFolder, - ) - - if self.dataset_manager is None: - raise ValueError("Dataset manager is not set") - - if not self.has_do_role: - raise ValueError("Only dataset owners can share datasets") - - # Verify dataset exists - dataset = self.dataset_manager.get(name=tag, datasite=self.email) - if dataset is None: - raise ValueError(f"Dataset {tag} not found") - - # Compute current content hash from local files - files = {} - for mock_file in dataset.mock_files: - if mock_file.exists(): - files[mock_file.name] = mock_file.read_bytes() - metadata_path = dataset.mock_dir / "dataset.yaml" - if metadata_path.exists(): - files["dataset.yaml"] = metadata_path.read_bytes() - if dataset.readme_path and dataset.readme_path.exists(): - files[dataset.readme_path.name] = dataset.readme_path.read_bytes() - - content_hash = DatasetCollectionFolder.compute_hash(files) - - # Share collection - if users == "any": - self._connection_router.owner_tag_dataset_collection_as_any( - tag, content_hash - ) - self.datasite_owner_syncer._any_shared_datasets.append((tag, content_hash)) - peer_emails = [p.email for p in self.peer_manager.approved_peers] - if peer_emails: - self._connection_router.owner_share_dataset_collection( - tag, content_hash, peer_emails - ) - else: - if isinstance(users, str): - users = [users] - self._connection_router.owner_share_dataset_collection( - tag, content_hash, users - ) - - if sync: - self.sync() - - def share_private_dataset(self, tag: str, enclave_email: str): - """Share private dataset files with an enclave via outbox events.""" - if not self.has_do_role: - raise ValueError("Only data owners can share private datasets") - - with self._sync_file_lock(): - files = self.dataset_manager.get_private_dataset_files(tag) - events_message = ( - self.datasite_owner_syncer.event_cache.create_events_for_files(files) - ) - self.datasite_owner_syncer.queue_event_for_syftbox( - recipients=[enclave_email], - file_change_events_message=events_message, - ) - self.datasite_owner_syncer.process_syftbox_events_queue() - - @property - def datasets(self) -> SyftDatasetManager: - """ - Get the dataset manager. Automatically calls sync() before returning datasets - if PRE_SYNC environment variable is set to "true" (case-insensitive). - - PRE_SYNC defaults to "true", so auto-sync is enabled by default. - To disable auto-sync, set: PRE_SYNC=false - """ - if self.dataset_manager is None: - raise ValueError("Dataset manager is not set") - - if os.environ.get("PRE_SYNC", "true").lower() == "true": - self.sync() - - return self.dataset_manager - @property def _connection_router(self) -> ConnectionRouter: # for DOs we have a syncer, for DSs we have a watcher syncer @@ -1707,13 +1193,6 @@ def _get_all_peer_platforms(self) -> List[BasePlatform]: def _resolve_path(self, path: str | Path) -> Path: return resolve_path(path, syftbox_folder=self.syftbox_folder) - def _resolve_dataset_owners_for_name(self, dataset_name: str) -> str | None: - matches = [] - for dataset in self.dataset_manager.get_all(): - if dataset.name == dataset_name: - matches.append(dataset.owner) - return matches - def _copy(self): from copy import deepcopy @@ -1730,19 +1209,3 @@ def _copy(self): new_do_connection = GDriveConnection.from_service(self.email, drive_service) new_manager._add_connection(new_do_connection) return new_manager - - # def resolve_dataset_path( - # self, dataset_name: str, owner_email: str | None = None - # ) -> Path: - # if owner_email is None: - # owner_emails = self._resolve_dataset_owners_for_name(dataset_name) - # if len(owner_emails) == 1: - # owner_email = owner_emails[0] - # else: - # raise ValueError( - # f"Dataset {dataset_name} has 0 or multiple owners: {owner_emails}, please specify the owner_email" - # ) - - # return resolve_dataset_path( - # dataset_name, syftbox_folder=self.syftbox_folder, owner_email=owner_email - # ) diff --git a/syft_client/sync/sync/caches/datasite_owner_cache.py b/syft_client/sync/sync/caches/datasite_owner_cache.py index eb54cd55962..c29f5754ff1 100644 --- a/syft_client/sync/sync/caches/datasite_owner_cache.py +++ b/syft_client/sync/sync/caches/datasite_owner_cache.py @@ -90,10 +90,6 @@ def from_config(cls, config: DataSiteOwnerEventCacheConfig): raise ValueError("syftbox_folder is required for non-in-memory cache") if config.email is None: raise ValueError("email is required for non-in-memory cache") - if config.collections_folder is None: - raise ValueError( - "collections_folder is required for non-in-memory cache" - ) syftbox_folder_name = Path(config.syftbox_folder).name my_datasite_folder = config.syftbox_folder / config.email syftbox_parent = Path(config.syftbox_folder).parent @@ -168,8 +164,14 @@ def latest_cached_timestamp(self) -> float | None: return None return max(m.timestamp for m in cached_messages) - def collections_relative_path(self) -> Path: - """Return the collections folder path relative to the datasite root.""" + def collections_relative_path(self) -> Path | None: + """Return the collections folder path relative to the datasite root. + + Returns None when no collections folder is configured (no collection + specs registered); + """ + if self.collections_folder is None: + return None return self.collections_folder.relative_to(self.syftbox_folder / self.email) def get_syncable_paths(self) -> dict[Path, bytes]: diff --git a/syft_client/sync/sync/caches/datasite_watcher_cache.py b/syft_client/sync/sync/caches/datasite_watcher_cache.py index 5263b0fcf36..22afe5d953a 100644 --- a/syft_client/sync/sync/caches/datasite_watcher_cache.py +++ b/syft_client/sync/sync/caches/datasite_watcher_cache.py @@ -14,6 +14,7 @@ CacheFileConnection, InMemoryCacheFileConnection, ) +from syft_client.sync.sync.collection_spec import CollectionSyncSpec SECONDS_BEFORE_SYNCING_DOWN = 0 @@ -24,8 +25,8 @@ class DataSiteWatcherCacheConfig(BaseModel): syftbox_folder: Path | None = None events_base_path: Path | None = None connection_configs: List[ConnectionConfig] = [] - # Subpath from owner_email to collections folder (e.g., "public/syft_datasets") - collection_subpath: Path | None = None + # Collections to sync down (prefix + local subpath per collection type) + collection_specs: list[CollectionSyncSpec] = [] class DataSiteWatcherCache(BaseModel): @@ -46,10 +47,10 @@ class DataSiteWatcherCache(BaseModel): last_event_timestamp_per_peer: Dict[str, float] = {} # Base syftbox folder syftbox_folder: Path | None = None - # Subpath from owner_email to collections folder (e.g., "public/syft_datasets") - collection_subpath: Path | None = None - # Cache of dataset collection hashes: path -> content_hash - dataset_collection_hashes: Dict[Path, str] = {} + # Collections to sync down (prefix + local subpath per collection type) + collection_specs: list[CollectionSyncSpec] = [] + # Cache of collection hashes: path -> content_hash + collection_hashes: Dict[Path, str] = {} # Optional pre-write filter: (path_in_syftbox, is_delete) -> allow? # Return True to allow the write, False to deny it. pre_write_filter: Callable[[str, bool], bool] | None = None @@ -65,17 +66,13 @@ def from_config(cls, config: DataSiteWatcherCacheConfig): connection_configs=config.connection_configs, ), syftbox_folder=config.syftbox_folder, - collection_subpath=config.collection_subpath, + collection_specs=config.collection_specs, ) return res else: if config.syftbox_folder is None: raise ValueError("syftbox_folder is required for non-in-memory cache") - if config.collection_subpath is None: - raise ValueError( - "collection_subpath is required for non-in-memory cache" - ) - + syftbox_folder_name = Path(config.syftbox_folder).name syftbox_parent = Path(config.syftbox_folder).parent events_folder = syftbox_parent / f"{syftbox_folder_name}-event-messages" @@ -90,15 +87,15 @@ def from_config(cls, config: DataSiteWatcherCacheConfig): connection_configs=config.connection_configs, ), syftbox_folder=config.syftbox_folder, - collection_subpath=config.collection_subpath, + collection_specs=config.collection_specs, ) cache._load_cached_state() return cache def _load_cached_state(self): - """Load cached state from disk: file hashes, timestamps, and dataset hashes.""" + """Load cached state from disk: file hashes, timestamps, and collection hashes.""" self._load_file_hashes_from_events() - self._load_dataset_hashes_from_disk() + self._load_collection_hashes_from_disk() def _load_file_hashes_from_events(self): """Load file hashes and timestamps from cached events.""" @@ -130,42 +127,47 @@ def _load_file_hashes_from_events(self): else: self.file_hashes[path_key] = event.new_hash - def _load_dataset_hashes_from_disk(self): - """Scan local dataset directories and compute hashes to populate dataset_collection_hashes.""" - for collection_path in self._get_local_dataset_folders(): - content_hash = self._compute_local_dataset_hash(collection_path) + def _load_collection_hashes_from_disk(self): + """Scan local collection directories and compute hashes to populate collection_hashes.""" + for collection_path in self._get_local_collection_folders(): + content_hash = self._compute_local_collection_hash(collection_path) if content_hash: - self.dataset_collection_hashes[collection_path] = content_hash + self.collection_hashes[collection_path] = content_hash def get_collection_owner_email(self, collection_path: Path) -> str: """Extract the owner email from a collection path.""" return collection_path.relative_to(self.syftbox_folder).parts[0] - def get_collection_path(self, owner_email: str, tag: str) -> Path | None: - """Get the full path to a collection for a given owner and tag.""" - if self.syftbox_folder is None or self.collection_subpath is None: + def get_collection_path( + self, owner_email: str, tag: str, local_subpath: Path + ) -> Path | None: + """Get the full path to a collection for a given owner, tag and subpath.""" + if self.syftbox_folder is None: return None - return self.syftbox_folder / owner_email / self.collection_subpath / tag + return self.syftbox_folder / owner_email / local_subpath / tag - def _get_local_dataset_folders(self): - """Yield paths to all local dataset folders.""" + def _get_local_collection_folders(self): + """Yield paths to all local collection folders across all specs.""" if self.syftbox_folder is None or not self.syftbox_folder.exists(): return - if self.collection_subpath is None: - return - for email_dir in self.syftbox_folder.iterdir(): - if not email_dir.is_dir() or "@" not in email_dir.name: + for spec in self.collection_specs: + if spec.owner_only: + # Owner-only collections (e.g. private data) are never pulled from + # peers, so the peer-facing watcher does not track them locally. continue - datasets_dir = email_dir / self.collection_subpath - if not datasets_dir.exists(): - continue - for tag_dir in datasets_dir.iterdir(): - if tag_dir.is_dir(): - yield tag_dir + for email_dir in self.syftbox_folder.iterdir(): + if not email_dir.is_dir() or "@" not in email_dir.name: + continue + collections_dir = email_dir / spec.local_subpath + if not collections_dir.exists(): + continue + for tag_dir in collections_dir.iterdir(): + if tag_dir.is_dir(): + yield tag_dir - def _compute_local_dataset_hash(self, collection_path: Path) -> str | None: - """Compute content hash from local dataset files on disk.""" + def _compute_local_collection_hash(self, collection_path: Path) -> str | None: + """Compute content hash from local collection files on disk.""" from syft_client.sync.file_utils import compute_directory_hash return compute_directory_hash(collection_path) @@ -178,7 +180,7 @@ def clear_cache(self): self.peers = [] self.current_check_point = None self.last_event_timestamp_per_peer = {} - self.dataset_collection_hashes = {} + self.collection_hashes = {} @property def last_event_timestamp(self) -> float | None: @@ -280,19 +282,28 @@ def current_hash_for_file(self, path: str) -> int | None: self.sync_down_if_needed(peer) return self.file_hashes.get(path, None) - def _cleanup_stale_dataset_collections( - self, peer_email: str, remote_collections: list[dict] + def _cleanup_stale_collections( + self, peer_email: str, remote_collections: list[dict], local_subpath: Path ): - """Remove locally cached dataset collections that no longer exist remotely.""" + """Remove locally cached collections that no longer exist remotely. + + Only considers collections under the given local_subpath so that + collections from other specs are not treated as stale. + """ remote_tags = {c["tag"] for c in remote_collections} - for local_collection_path in list(self.dataset_collection_hashes.keys()): + for local_collection_path in list(self.collection_hashes.keys()): owner_email = self.get_collection_owner_email(local_collection_path) if owner_email != peer_email: continue + # Only consider collections that live under this spec's subpath. + if self.syftbox_folder is not None: + expected_parent = self.syftbox_folder / owner_email / local_subpath + if local_collection_path.parent != expected_parent: + continue if local_collection_path.name in remote_tags: continue - del self.dataset_collection_hashes[local_collection_path] + del self.collection_hashes[local_collection_path] if self.syftbox_folder is not None: try: rel_path = local_collection_path.relative_to(self.syftbox_folder) @@ -300,112 +311,137 @@ def _cleanup_stale_dataset_collections( except ValueError: pass - def sync_down_datasets(self, peer_email: str): + def sync_down_collections(self, peer_email: str): """ - Sync dataset collections from peer. + Sync collections from peer across all configured specs. Separate from message sync. Uses hash to skip unchanged collections. """ - # Get list of collections shared with us (now returns list of dicts) - collections = self.connection_router.watcher_list_dataset_collections() + for spec in self.collection_specs: + if spec.owner_only: + # Never pull owner-only collections (e.g. private data) from a peer. + continue + # Get list of collections shared with us (returns list of dicts) + collections = self.connection_router.watcher_list_collections(spec.prefix) - # Filter by peer - peer_collections = [c for c in collections if c["owner_email"] == peer_email] + # Filter by peer + peer_collections = [ + c for c in collections if c["owner_email"] == peer_email + ] - self._cleanup_stale_dataset_collections(peer_email, peer_collections) + self._cleanup_stale_collections( + peer_email, peer_collections, spec.local_subpath + ) - for collection in peer_collections: - owner_email = collection["owner_email"] - tag = collection["tag"] - content_hash = collection["content_hash"] + for collection in peer_collections: + owner_email = collection["owner_email"] + tag = collection["tag"] + content_hash = collection["content_hash"] - # Check if hash changed - skip download if unchanged - collection_path = self.get_collection_path(owner_email, tag) - if collection_path is None: - continue - cached_hash = self.dataset_collection_hashes.get(collection_path) - if cached_hash == content_hash: - continue + # Check if hash changed - skip download if unchanged + collection_path = self.get_collection_path( + owner_email, tag, spec.local_subpath + ) + if collection_path is None: + continue + cached_hash = self.collection_hashes.get(collection_path) + if cached_hash == content_hash: + continue - # Download collection files - files = self.connection_router.watcher_download_dataset_collection( - tag, content_hash, owner_email - ) + # Download collection files + files = self.connection_router.watcher_download_collection( + spec.prefix, tag, content_hash, owner_email + ) - # Write files to local cache (path relative to syftbox_folder) - for file_name, content in files.items(): - rel_path = f"{owner_email}/{self.collection_subpath}/{tag}/{file_name}" - self.file_connection.write_file(rel_path, content) + # Write files to local cache (path relative to syftbox_folder) + for file_name, content in files.items(): + rel_path = ( + f"{owner_email}/{spec.local_subpath}/{tag}/{file_name}" + ) + self.file_connection.write_file(rel_path, content) - # Update hash cache - self.dataset_collection_hashes[collection_path] = content_hash + # Update hash cache + self.collection_hashes[collection_path] = content_hash - def sync_down_datasets_parallel( + def sync_down_collections_parallel( self, peer_email: str, executor: ThreadPoolExecutor, download_fn: Callable[[str], bytes], ): """ - Sync dataset collections from peer with parallel file downloads. - Downloads all files from all collections in a single parallel batch. + Sync collections from peer with parallel file downloads, across all specs. + For each spec, downloads all files from all collections in a single + parallel batch. """ - collections = self.connection_router.watcher_list_dataset_collections() - peer_collections = [c for c in collections if c["owner_email"] == peer_email] + for spec in self.collection_specs: + if spec.owner_only: + # Never pull owner-only collections (e.g. private data) from a peer. + continue + collections = self.connection_router.watcher_list_collections(spec.prefix) + peer_collections = [ + c for c in collections if c["owner_email"] == peer_email + ] - self._cleanup_stale_dataset_collections(peer_email, peer_collections) + self._cleanup_stale_collections( + peer_email, peer_collections, spec.local_subpath + ) - # Gather all files to download across all collections - all_downloads = [] # List of (collection_info, file_metadata) - collections_to_update = [] + # Gather all files to download across all collections for this spec + all_downloads = [] # List of (collection_info, file_metadata) + collections_to_update = [] - for collection in peer_collections: - owner_email = collection["owner_email"] - tag = collection["tag"] - content_hash = collection["content_hash"] + for collection in peer_collections: + owner_email = collection["owner_email"] + tag = collection["tag"] + content_hash = collection["content_hash"] - # Check if hash changed - skip download if unchanged - collection_path = self.get_collection_path(owner_email, tag) - if collection_path is None: - continue - cached_hash = self.dataset_collection_hashes.get(collection_path) - if cached_hash == content_hash: - continue + # Check if hash changed - skip download if unchanged + collection_path = self.get_collection_path( + owner_email, tag, spec.local_subpath + ) + if collection_path is None: + continue + cached_hash = self.collection_hashes.get(collection_path) + if cached_hash == content_hash: + continue - # Get file metadata (no download yet) - file_metadatas = ( - self.connection_router.watcher_get_dataset_collection_file_metadatas( - tag, content_hash, owner_email + # Get file metadata (no download yet) + file_metadatas = ( + self.connection_router.watcher_get_collection_file_metadatas( + spec.prefix, tag, content_hash, owner_email + ) ) - ) - if not file_metadatas: + if not file_metadatas: + continue + + collections_to_update.append(collection) + for metadata in file_metadatas: + all_downloads.append((collection, metadata)) + + if not all_downloads: continue - collections_to_update.append(collection) - for metadata in file_metadatas: - all_downloads.append((collection, metadata)) + # Download all files from all collections in parallel + file_ids = [metadata["file_id"] for _, metadata in all_downloads] + downloaded_contents = list(executor.map(download_fn, file_ids)) - if not all_downloads: - return + # Write files to local cache (path relative to syftbox_folder) + for (collection, metadata), content in zip( + all_downloads, downloaded_contents + ): + owner_email = collection["owner_email"] + tag = collection["tag"] + file_name = metadata["file_name"] + rel_path = f"{owner_email}/{spec.local_subpath}/{tag}/{file_name}" + self.file_connection.write_file(rel_path, content) - # Download all files from all collections in parallel - file_ids = [metadata["file_id"] for _, metadata in all_downloads] - downloaded_contents = list(executor.map(download_fn, file_ids)) - - # Write files to local cache (path relative to syftbox_folder) - for (collection, metadata), content in zip(all_downloads, downloaded_contents): - owner_email = collection["owner_email"] - tag = collection["tag"] - file_name = metadata["file_name"] - rel_path = f"{owner_email}/{self.collection_subpath}/{tag}/{file_name}" - self.file_connection.write_file(rel_path, content) - - # Update hash cache for all collections - for collection in collections_to_update: - collection_path = self.get_collection_path( - collection["owner_email"], collection["tag"] - ) - if collection_path is not None: - self.dataset_collection_hashes[collection_path] = collection[ - "content_hash" - ] + # Update hash cache for all collections in this spec + for collection in collections_to_update: + collection_path = self.get_collection_path( + collection["owner_email"], collection["tag"], spec.local_subpath + ) + if collection_path is not None: + self.collection_hashes[collection_path] = collection[ + "content_hash" + ] diff --git a/syft_client/sync/sync/collection_spec.py b/syft_client/sync/sync/collection_spec.py new file mode 100644 index 00000000000..1d17f67949b --- /dev/null +++ b/syft_client/sync/sync/collection_spec.py @@ -0,0 +1,56 @@ +"""The collection-sync primitive. + +``CollectionSyncSpec`` is the domain-free contract that tells the generic sync +engine HOW to sync a "collection" (a named, content-hashed folder like a dataset's +mock or private data) — WITHOUT the engine knowing what a dataset is. The concrete +specs are supplied by the domain layer (e.g. ``syft_rds.config.DATASET_COLLECTION_SPECS``). +""" + +from pathlib import Path + +from pydantic import BaseModel + + +class CollectionSyncSpec(BaseModel): + prefix: str + # Subpath from owner_email to the collection folder, + # e.g. Path("public/syft_datasets") + local_subpath: Path + # Download authority. False = mirror: re-download when the remote content-hash + # changes (the remote is the source of truth, e.g. published mock data). + # True = restore-only: download only when absent locally and never overwrite the + # local copy (the local copy is the source of truth, e.g. the owner's real data). + immutable: bool = False + # Sharing policy. False = shareable: a peer's watcher may pull it (e.g. mock data). + # True = owner-only: never shared with peers; the owner restores it for itself and + # peer-facing watchers skip it entirely (e.g. the owner's private data backup). + owner_only: bool = False + + @classmethod + def public(cls, prefix: str, local_subpath: "Path") -> "CollectionSyncSpec": + """A shareable, mirrored collection (e.g. a dataset's mock data). + + Peers' watchers pull it, and it re-downloads whenever the owner + republishes changed content. + """ + return cls( + prefix=prefix, + local_subpath=local_subpath, + immutable=False, + owner_only=False, + ) + + @classmethod + def private(cls, prefix: str, local_subpath: "Path") -> "CollectionSyncSpec": + """An owner-only, restore-only collection (e.g. a dataset's real data). + + Never shared with peers, peer-facing watchers skip it, and it is only + restored to the owner when absent locally (the local copy is authoritative + and is never overwritten by the backup). + """ + return cls( + prefix=prefix, + local_subpath=local_subpath, + immutable=True, + owner_only=True, + ) diff --git a/syft_client/sync/sync/datasite_owner_syncer.py b/syft_client/sync/sync/datasite_owner_syncer.py index 81e6b25241b..be7d7d9c533 100644 --- a/syft_client/sync/sync/datasite_owner_syncer.py +++ b/syft_client/sync/sync/datasite_owner_syncer.py @@ -1,6 +1,7 @@ import logging from pathlib import Path from uuid import uuid4 +from functools import partial from pydantic import ConfigDict, Field, BaseModel, PrivateAttr from concurrent.futures import ThreadPoolExecutor @@ -20,6 +21,7 @@ ) from syft_client.sync.connections.connection_router import ConnectionRouter from syft_client.sync.sync.caches.datasite_owner_cache import DataSiteOwnerEventCache +from syft_client.sync.sync.collection_spec import CollectionSyncSpec from syft_client.sync.callback_mixin import BaseModelCallbackMixin from syft_client.sync.messages.proposed_filechange import ProposedFileChangesMessage from syft_client.sync.utils.path_filters import is_normal_syncable_path @@ -53,6 +55,9 @@ class DatasiteOwnerSyncerConfig(BaseModel): write_files: bool = True # Full path to collections folder - must be provided explicitly collections_folder: Path | None = None + # Collection sync specs (public prefix + local subpath). Empty for a bare + # sync engine; the domain layer (e.g. syft-rds) supplies the concrete specs. + collection_specs: List[CollectionSyncSpec] = [] cache_config: DataSiteOwnerEventCacheConfig = Field( default_factory=DataSiteOwnerEventCacheConfig ) @@ -74,6 +79,8 @@ class DatasiteOwnerSyncer(BaseModelCallbackMixin): perm_context: SyftPermContext # Full path to collections folder collections_folder: Path | None = None + # Collection sync specs (public prefix + local subpath), supplied by rds. + collection_specs: List[CollectionSyncSpec] = [] syftbox_events_queue: Queue[FileChangeEventsMessage] = Field(default_factory=Queue) outbox_queue: Queue[Tuple[str, FileChangeEventsMessage]] = Field( @@ -83,8 +90,8 @@ class DatasiteOwnerSyncer(BaseModelCallbackMixin): _executor: ThreadPoolExecutor = PrivateAttr( default_factory=lambda: ThreadPoolExecutor(max_workers=10) ) - # Cache of datasets shared with "any" - list of (tag, content_hash) tuples - _any_shared_datasets: List[tuple] = PrivateAttr(default_factory=list) + # Cache of collections shared with "any" - list of (tag, content_hash) tuples + _any_shared_collections: List[tuple] = PrivateAttr(default_factory=list) # Cache of read permissions per file path → frozenset of peer emails _read_perm_cache: dict[str, frozenset[str]] = PrivateAttr(default_factory=dict) @@ -112,6 +119,7 @@ def from_config(cls, config: DatasiteOwnerSyncerConfig): syftbox_folder=config.syftbox_folder, perm_context=SyftPermContext(datasite=datasite), collections_folder=config.collections_folder, + collection_specs=config.collection_specs, ) @property @@ -289,11 +297,10 @@ def pull_initial_state(self): if restored_events and not self.event_cache.get_cached_events(): self._write_events_to_messages_cache(restored_events) - # Load datasets from connection and populate _any_shared_datasets cache - self._pull_datasets_for_initial_sync() - - # Restore private datasets from GDrive (owner-only collections) - self._pull_private_datasets_for_initial_sync() + # Restore ALL registered collections (public + private) generically. Each + # spec's flags decide the behaviour: `immutable` picks mirror vs restore-only; + # the local destination comes from the spec's local_subpath. + self._pull_collections_for_initial_sync() self.initial_sync_done = True @@ -316,43 +323,65 @@ def _apply_incremental_checkpoint_to_cache( str(event.path_in_datasite), event.content ) - def _pull_datasets_for_initial_sync(self): - """Load datasets from GDrive when DO connects. + def _collection_local_dir(self, spec: CollectionSyncSpec, tag: str) -> Path: + """Local directory a collection with the given tag restores to.""" + return self.syftbox_folder / self.email / spec.local_subpath / tag + + def _pull_collections_for_initial_sync(self): + """Restore the owner's own collections from the sync backend on connect. - Restores datasets to local filesystem and populates the _any_shared_datasets cache. + Iterates every registered spec (public, private, or any future kind). For + each it lists the owner's collections, refreshes the _any_shared_collections + cache, filters by the spec's download policy, and restores to disk. """ - collections = ( - self.connection_router.owner_list_all_dataset_collections_with_permissions() - ) + for spec in self.collection_specs: + collections = ( + self.connection_router.owner_list_all_collections_with_permissions( + spec.prefix + ) + ) + if not collections: + continue - self._update_any_shared_datasets_cache(collections) + self._update_any_shared_collections_cache(collections) - collections_to_download = self._filter_collections_needing_download(collections) - self._download_dataset_collections_parallel(collections_to_download) + collections_to_download = self._filter_collections_needing_download( + collections, spec + ) + self._download_collections_parallel(collections_to_download, spec) - def _update_any_shared_datasets_cache(self, collections: list[FileCollection]): - """Populate _any_shared_datasets cache from collections with 'any' permission.""" + def _update_any_shared_collections_cache(self, collections: list[FileCollection]): + """Populate _any_shared_collections cache from collections with 'any' permission.""" for collection in collections: if collection.has_any_permission: entry = (collection.tag, collection.content_hash) - if entry not in self._any_shared_datasets: - self._any_shared_datasets.append(entry) + if entry not in self._any_shared_collections: + self._any_shared_collections.append(entry) def _filter_collections_needing_download( - self, collections: list[FileCollection] + self, collections: list[FileCollection], spec: CollectionSyncSpec ) -> list[FileCollection]: - """Return collections that don't exist locally or have different content hash.""" + """Return the collections that need downloading, per the spec's policy. + + * immutable (restore-only): download only when absent locally — the local + copy is authoritative and must never be overwritten by the backup. + * mutable (mirror): download when the remote content-hash differs from ours. + """ from syft_client.sync.file_utils import compute_directory_hash result = [] for collection in collections: - # Use cached hash from event_cache first + local_dir = self._collection_local_dir(spec, collection.tag) + + if spec.immutable: + if not local_dir.exists() or not any(local_dir.iterdir()): + result.append(collection) + continue + + # Mirror: use the cached hash first, else compute it from local files. cached_hash = self.event_cache.get_collection_hash(collection.tag) - if cached_hash is None and self.collections_folder is not None: - # Fallback: compute hash from local filesystem (for locally created datasets) - local_dataset_dir = self.collections_folder / collection.tag - cached_hash = compute_directory_hash(local_dataset_dir) - # Update cache if we computed a hash + if cached_hash is None and local_dir.exists(): + cached_hash = compute_directory_hash(local_dir) if cached_hash is not None: self.event_cache.set_collection_hash(collection.tag, cached_hash) @@ -360,7 +389,9 @@ def _filter_collections_needing_download( result.append(collection) return result - def _download_dataset_collections_parallel(self, collections: list[FileCollection]): + def _download_collections_parallel( + self, collections: list[FileCollection], spec: CollectionSyncSpec + ): """Download all files from collections in parallel and write to disk.""" if not collections: return @@ -368,7 +399,10 @@ def _download_dataset_collections_parallel(self, collections: list[FileCollectio # Fetch file metadatas for all collections in parallel all_file_metadatas = list( self._executor.map( - self._get_file_metadatas_with_new_connection, collections + partial( + self._get_file_metadatas_with_new_connection, prefix=spec.prefix + ), + collections, ) ) @@ -388,24 +422,36 @@ def _download_dataset_collections_parallel(self, collections: list[FileCollectio self._executor.map(self._download_file_with_new_connection, file_ids) ) - # Write all files to disk + # Write all files to disk under the spec's local subpath. for (collection, metadata), content in zip(all_downloads, downloaded_contents): - local_dataset_dir = self.collections_folder / collection.tag - local_dataset_dir.mkdir(parents=True, exist_ok=True) - (local_dataset_dir / metadata["file_name"]).write_bytes(content) + local_dir = self._collection_local_dir(spec, collection.tag) + local_dir.mkdir(parents=True, exist_ok=True) + (local_dir / metadata["file_name"]).write_bytes(content) - # Update cached hashes for downloaded collections + # Update cached hashes ONLY for mutable specs + if not spec.immutable: + for collection in collections: + self.event_cache.set_collection_hash( + collection.tag, collection.content_hash + ) + + # Notify the domain layer (e.g. syft-rds) that these collections were restored, + # so it can run any collection-specific post-processing for collection in collections: - self.event_cache.set_collection_hash( - collection.tag, collection.content_hash + self._emit( + "collection_restored", + spec.prefix, + collection.tag, + self._collection_local_dir(spec, collection.tag), ) def _get_file_metadatas_with_new_connection( - self, collection: FileCollection + self, collection: FileCollection, prefix: str ) -> list: """Get file metadatas for a collection using a new connection for thread safety.""" connection = self.connection_router.connection_for_parallel_download() - return connection.watcher_get_dataset_collection_file_metadatas( + return connection.watcher_get_collection_file_metadatas( + prefix, tag=collection.tag, content_hash=collection.content_hash, owner_email=self.email, @@ -414,100 +460,7 @@ def _get_file_metadatas_with_new_connection( def _download_file_with_new_connection(self, file_id: str) -> bytes: """Download a file using a new connection for thread safety.""" connection = self.connection_router.connection_for_parallel_download() - return connection.watcher_download_dataset_file(file_id) - - # ========================================================================= - # PRIVATE DATASET RESTORE METHODS - # ========================================================================= - - def _pull_private_datasets_for_initial_sync(self): - """Restore private datasets from GDrive when DO reconnects. - - Downloads private data from owner-only collection folders - to {syftbox_folder}/private/syft_datasets/{tag}/. - """ - collections = self.connection_router.owner_list_private_dataset_collections() - if not collections: - return - - collections_to_download = self._filter_private_collections_needing_download( - collections - ) - self._download_private_collections_parallel(collections_to_download) - - def _private_dataset_local_dir(self, tag: str) -> Path: - return self.syftbox_folder / self.email / "private" / "syft_datasets" / tag - - def _filter_private_collections_needing_download( - self, collections: list[FileCollection] - ) -> list[FileCollection]: - """Return private collections that don't exist locally yet.""" - result = [] - for collection in collections: - local_dir = self._private_dataset_local_dir(collection.tag) - if not local_dir.exists() or not any(local_dir.iterdir()): - result.append(collection) - return result - - def _download_private_collections_parallel(self, collections: list[FileCollection]): - """Download private collection files in parallel and write to disk.""" - if not collections: - return - - all_file_metadatas = list( - self._executor.map( - self._get_private_file_metadatas_with_new_connection, collections - ) - ) - - all_downloads = [ - (collection, metadata) - for collection, file_metadatas in zip(collections, all_file_metadatas) - for metadata in file_metadatas - ] - - if not all_downloads: - return - - file_ids = [metadata["file_id"] for _, metadata in all_downloads] - downloaded_contents = list( - self._executor.map(self._download_file_with_new_connection, file_ids) - ) - - for (collection, metadata), content in zip(all_downloads, downloaded_contents): - local_dir = self._private_dataset_local_dir(collection.tag) - local_dir.mkdir(parents=True, exist_ok=True) - (local_dir / metadata["file_name"]).write_bytes(content) - - # Fix data_dir in private_metadata.yaml to point to current local path - for collection in collections: - self._fix_private_metadata_data_dir(collection.tag) - - def _get_private_file_metadatas_with_new_connection( - self, collection: FileCollection - ) -> list: - """Get file metadatas for a private collection using a new connection.""" - connection = self.connection_router.connection_for_parallel_download() - return connection.owner_get_private_collection_file_metadatas( - tag=collection.tag, - content_hash=collection.content_hash, - owner_email=self.email, - ) - - def _fix_private_metadata_data_dir(self, dataset_tag: str): - """Update data_dir in private_metadata.yaml to match the current syftbox path.""" - local_dir = self._private_dataset_local_dir(dataset_tag) - metadata_path = local_dir / "private_metadata.yaml" - if not metadata_path.exists(): - return - - import yaml - - data = yaml.safe_load(metadata_path.read_text()) - expected_dir = str(local_dir) - if data.get("data_dir") != expected_dir: - data["data_dir"] = expected_dir - metadata_path.write_text(yaml.safe_dump(data, indent=2, sort_keys=False)) + return connection.watcher_download_collection_file(file_id) def _get_readers(self, path: str, recipients: list[str]) -> frozenset[str]: """Return the set of recipients that have read access to the given path.""" diff --git a/syft_client/sync/sync/datasite_watcher_syncer.py b/syft_client/sync/sync/datasite_watcher_syncer.py index 85e9f279081..a26c064310d 100644 --- a/syft_client/sync/sync/datasite_watcher_syncer.py +++ b/syft_client/sync/sync/datasite_watcher_syncer.py @@ -135,12 +135,12 @@ def download_events_message_with_new_connection( ) return FileChangeEventsMessage.from_compressed_data(raw) - def download_dataset_file_with_new_connection( + def download_collection_file_with_new_connection( self, file_id: str, owner_email: str ) -> bytes: - """Download dataset file using a new connection (thread-safe).""" + """Download collection file using a new connection (thread-safe).""" connection = self.connection_router.connection_for_parallel_download() - data = connection.watcher_download_dataset_file(file_id) + data = connection.watcher_download_collection_file(file_id) data = self.connection_router.peer_store.decrypt_dataset_if_needed( owner_email, data ) @@ -159,10 +159,12 @@ def sync_down(self, peer_emails: list[str]): ) if event_count: print(f"Pulled {event_count} inbound event(s) from {peer_email}") - # Sync datasets with parallel download - self.datasite_watcher_cache.sync_down_datasets_parallel( + # Sync collections with parallel download + self.datasite_watcher_cache.sync_down_collections_parallel( peer_email, self._executor, lambda fid, - pe=peer_email: self.download_dataset_file_with_new_connection(fid, pe), + pe=peer_email: self.download_collection_file_with_new_connection( + fid, pe + ), ) diff --git a/tests/unit/syft_bg/test_email_approval_flow.py b/tests/unit/syft_bg/test_email_approval_flow.py index f1a6240d2ca..7fecde67bc8 100644 --- a/tests/unit/syft_bg/test_email_approval_flow.py +++ b/tests/unit/syft_bg/test_email_approval_flow.py @@ -17,7 +17,7 @@ from syft_bg.notify.handlers.job import JobHandler from syft_bg.notify.monitors.job import JobMonitor from syft_bg.notify.orchestrator import NotificationOrchestrator -from syft_client.sync.syftbox_manager import SyftboxManager +from syft_rds import SyftRDSClient from tests.unit.utils import create_test_project_folder, create_tmp_dataset_files @@ -25,7 +25,7 @@ def _make_notify_orchestrator( - do_manager: SyftboxManager, + do_manager: SyftRDSClient, tmp: Path, ) -> tuple[NotificationOrchestrator, JsonStateManager, MagicMock]: """Create a NotificationOrchestrator with a mocked GmailSender. @@ -64,7 +64,7 @@ def _make_notify_orchestrator( def _make_email_approve_orchestrator( - do_manager: SyftboxManager, + do_manager: SyftRDSClient, notify_state: JsonStateManager, tmp: Path, ) -> tuple[EmailApproveOrchestrator, EmailApproveMonitor, JsonStateManager, MagicMock]: @@ -132,7 +132,7 @@ def test_email_approval_e2e(): """Full flow: DS submits job -> notify detects it -> email approval -> DS gets results.""" # -- Step 1: Create DO/DS clients -- - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) diff --git a/tests/unit/syft_bg/test_email_auto_approve_flow.py b/tests/unit/syft_bg/test_email_auto_approve_flow.py index 0fb0fed921c..6593b774320 100644 --- a/tests/unit/syft_bg/test_email_auto_approve_flow.py +++ b/tests/unit/syft_bg/test_email_auto_approve_flow.py @@ -20,7 +20,7 @@ from syft_bg.notify.handlers.job import JobHandler from syft_bg.notify.monitors.job import JobMonitor from syft_bg.notify.orchestrator import NotificationOrchestrator -from syft_client.sync.syftbox_manager import SyftboxManager +from syft_rds import SyftRDSClient FAKE_THREAD_ID = "thread_auto_approve_123" @@ -44,7 +44,7 @@ def _temp_config_paths(): def _make_notify_orchestrator( - do_manager: SyftboxManager, + do_manager: SyftRDSClient, tmp: Path, ) -> tuple[NotificationOrchestrator, JsonStateManager, MagicMock]: """Create a NotificationOrchestrator with a mocked GmailSender.""" @@ -81,7 +81,7 @@ def _make_notify_orchestrator( def _make_email_approve_orchestrator( - do_manager: SyftboxManager, + do_manager: SyftRDSClient, notify_state: JsonStateManager, tmp: Path, reply_text: str = "auto-approve", @@ -146,7 +146,7 @@ def _create_project_code_files_with_json_contents(params: dict) -> Path: def test_email_auto_approve_creates_object_and_approves_future_jobs(): """Full flow: auto-approve reply creates approval object, second job is auto-approved.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, )