Merge pull request #319 from MerginMaps/sparse_checkout

Add sparse checkout support

Author: MarcelGeo

mergin/cli.py

@@ -249,16 +249,20 @@ def list_projects(ctx, name, namespace, order_params):
 @click.argument("project")
 @click.argument("directory", type=click.Path(), required=False)
 @click.option("--version", default=None, help="Version of project to download")
+@click.option("--include", multiple=True, help="Only download files matching this pattern, e.g. '*.gpkg'")
+@click.option("--exclude", multiple=True, help="Skip files matching this pattern, e.g. 'media/*'")
 @click.pass_context
-def download(ctx, project, directory, version):
+def download(ctx, project, directory, version, include, exclude):
     """Download last version of mergin project."""
     mc = ctx.obj["client"]
     if mc is None:
         return
+    if include and exclude:
+        raise click.UsageError("--include and --exclude cannot be used together")
     directory = directory or os.path.basename(project)
     click.echo("Downloading into {}".format(directory))
     try:
-        job = download_project_async(mc, project, directory, version)
+        job = download_project_async(mc, project, directory, version, include=include, exclude=exclude)
         with click.progressbar(length=job.total_size) as bar:
             last_transferred_size = 0
             while download_project_is_running(job):

mergin/client.py

@@ -64,6 +64,7 @@
 from .utils import DateTimeEncoder, get_versions_with_file_changes, int_version, is_version_acceptable
 from .utils import (
     DateTimeEncoder,
+    filter_files,
     get_versions_with_file_changes,
     int_version,
     is_version_acceptable,
@@ -903,7 +904,7 @@ def project_versions(self, project_path, since=1, to=None):
         filtered_versions = list(filter(lambda v: (num_since <= int_version(v["name"]) <= num_to), versions))
         return filtered_versions
 
-    def download_project(self, project_path, directory, version=None):
+    def download_project(self, project_path, directory, version=None, include=None, exclude=None):
         """
         Download project into given directory. If version is not specified, latest version is downloaded
 
@@ -915,8 +916,16 @@ def download_project(self, project_path, directory, version=None):
 
         :param version: Project version to download, e.g. v42
         :type version: String
+
+        :param include: Optional list of glob patterns (matched against each file's project path, e.g.
+            "media/*" or "*.gpkg") - only matching files are downloaded.
+        :type include: List[String]
+
+        :param exclude: Optional list of glob patterns - matching files are skipped. Mutually exclusive
+            with include.
+        :type exclude: List[String]
         """
-        job = download_project_async(self, project_path, directory, version)
+        job = download_project_async(self, project_path, directory, version, include=include, exclude=exclude)
         download_project_wait(job)
         download_project_finalize(job)
 
@@ -1159,6 +1168,10 @@ def project_status(self, directory):
         server_info = self.project_info(mp.project_full_name(), since=mp.version())
 
         pull_changes = mp.get_pull_changes(server_info.get("files", []), server_info.get("version"))
+        # on a sparse checkout, don't report excluded files as pending server changes -
+        # they were never meant to be pulled in the first place
+        file_filter = mp.file_filter()
+        pull_changes = {change_type: filter_files(files, **file_filter) for change_type, files in pull_changes.items()}
 
         push_changes = mp.get_push_changes()
         push_changes_summary = mp.get_list_of_push_changes(push_changes)

mergin/client_pull.py

@@ -25,7 +25,7 @@
 from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType
 from .models import ProjectDelta, ProjectDeltaChange, PullAction
 from .merginproject import MerginProject
-from .utils import cleanup_tmp_dir, is_versioned_file, save_to_file
+from .utils import cleanup_tmp_dir, filter_files, is_path_in_scope, save_to_file, is_versioned_file
 from typing import List, Optional, Union
 
 # status = download_project_async(...)
@@ -261,12 +261,18 @@ def _cleanup_failed_download(mergin_project: MerginProject = None):
     return dest_path
 
 
-def download_project_async(mc, project_path, directory, project_version=None):
+def download_project_async(mc, project_path, directory, project_version=None, include=None, exclude=None):
     """
     Starts project download in background and returns handle to the pending project download.
     Using that object it is possible to watch progress or cancel the ongoing work.
+
+    `include`/`exclude` are optional lists of glob patterns (matched against each file's project
+    path, e.g. "media/*" or "*.gpkg") to only download a subset of the project's files. They are
+    mutually exclusive.
     """
 
+    if include and exclude:
+        raise ClientError("Cannot use both include and exclude filters at the same time")
     if "/" not in project_path:
         raise ClientError("Project name needs to be fully qualified, e.g. <username>/<projectname>")
     if os.path.exists(directory):
@@ -295,6 +301,12 @@ def download_project_async(mc, project_path, directory, project_version=None):
 
     mp.log.info(f"got project info. version {version}")
 
+    # keep only the files matching the filter (if any)
+    project_info["files"] = filter_files(project_info["files"], include=include, exclude=exclude)
+    # persisted once since it must never change again for this checkout
+    if include or exclude:
+        mp.write_file_filter({"include": include, "exclude": exclude})
+
     # prepare download
     update_tasks = []  # stuff to do at the end of download
     for file in project_info["files"]:
@@ -544,6 +556,9 @@ def pull_project_async(mc, directory) -> Optional[PullJob]:
         mp.log.info("--- pull aborted")
         raise
 
+    file_filter = mp.file_filter()
+    delta.changes = [c for c in delta.changes if is_path_in_scope(c.path, **file_filter)]
+
     mp.log.info(f"got project versions: local version {local_version} / server version {server_version}")
 
     if local_version == server_version:
@@ -767,6 +782,9 @@ def pull_project_finalize(job: PullJob):
         cleanup_tmp_dir(job.mp, job.tmp_dir)  # delete our temporary dir and all its content
         raise ClientError("Failed to apply pull actions: " + str(e))
 
+    # keep only in-scope files in the metadata we're about to persist
+    job.project_info["files"] = filter_files(job.project_info["files"], **job.mp.file_filter())
+
     job.mp.update_metadata(job.project_info)
 
     if job.mp.has_unfinished_pull():

mergin/client_push.py

@@ -34,7 +34,7 @@
 )
 from .merginproject import MerginProject, pygeodiff
 from .editor import filter_changes
-from .utils import get_data_checksum, cleanup_tmp_dir
+from .utils import get_data_checksum, cleanup_tmp_dir, filter_files
 
 POST_JSON_HEADERS = {"Content-Type": "application/json"}
 
@@ -458,6 +458,9 @@ def push_project_finalize(job: UploadJob):
             cleanup_tmp_dir(job.mp, job.tmp_dir)  # delete our temporary dir and all its content
             raise err
 
+    # keep only in-scope files in the metadata we're about to persist
+    job.server_resp["files"] = filter_files(job.server_resp["files"], **job.mp.file_filter())
+
     job.mp.update_metadata(job.server_resp)
     try:
         job.mp.apply_push_changes(asdict(job.changes))

mergin/merginproject.py

@@ -24,6 +24,8 @@
     unique_path_name,
     conflicted_copy_file_name,
     edit_conflict_file_name,
+    filter_files,
+    is_path_in_scope,
 )
 from .local_changes import FileChange
 
@@ -211,9 +213,28 @@ def version(self) -> str:
         return self._metadata["version"]
 
     def files(self) -> list:
-        """Returns project's list of files (each file being a dictionary)"""
+        """Returns project's list of files (each file being a dictionary), scoped to this
+        project's file_filter() if one is set (sparse checkout)."""
         self._read_metadata()
-        return self._metadata["files"]
+        return filter_files(self._metadata["files"], **self.file_filter())
+
+    def file_filter(self) -> dict:
+        """
+        Returns the include/exclude file filter this project was downloaded with, as a dict
+        with "include" and "exclude" keys. Stored in its own file (.mergin/file_filter.json)
+        """
+        filter_file = self.fpath_meta("file_filter.json")
+        if not os.path.exists(filter_file):
+            return {"include": None, "exclude": None}
+        with open(filter_file, "r") as f:
+            return json.load(f)
+
+    def write_file_filter(self, file_filter: dict) -> None:
+        """
+        Persists the include/exclude file filter this project was downloaded with.
+        """
+        with open(self.fpath_meta("file_filter.json"), "w") as f:
+            json.dump(file_filter, f, indent=2)
 
     @property
     def metadata(self) -> dict:
@@ -304,10 +325,12 @@ def ignore_file(self, file):
     def inspect_files(self):
         """
         Inspect files in project directory and return metadata.
+        Only files matching this project's file_filter() are included.
 
         :returns: metadata for files in project directory in server required format
         :rtype: list[dict]
         """
+        file_filter = self.file_filter()
         files_meta = []
         for root, dirs, files in os.walk(self.dir, topdown=True):
             dirs[:] = [d for d in dirs if d not in [".mergin"]]
@@ -318,6 +341,8 @@ def inspect_files(self):
                 abs_path = os.path.abspath(os.path.join(root, file))
                 rel_path = os.path.relpath(abs_path, start=self.dir)
                 proj_path = "/".join(rel_path.split(os.path.sep))  # we need posix path
+                if not is_path_in_scope(proj_path, **file_filter):
+                    continue
                 files_meta.append(
                     {
                         "path": proj_path,