cover all Python filesystem operations

Author: harminius

mergin/client.py

@@ -66,6 +66,7 @@
     int_version,
     is_version_acceptable,
     normalize_role,
+    long_path,
 )
 from . import fs
 from .version import __version__
@@ -205,7 +206,7 @@ def setup_logging(self):
         self.log.setLevel(logging.DEBUG)  # log everything (it would otherwise log just warnings+errors)
         if not self.log.handlers:
             if client_log_file:
-                log_handler = logging.FileHandler(client_log_file)
+                log_handler = logging.FileHandler(long_path(client_log_file))
                 log_handler.setFormatter(logging.Formatter("%(asctime)s %(message)s"))
                 self.log.addHandler(log_handler)
             else:
@@ -532,7 +533,7 @@ def create_project_and_push(self, project_name, directory, is_public=False, name
         :param namespace: Deprecated. project_name should be full project name. Optional namespace for a new project. If empty username is used.
         :type namespace: String
         """
-        if os.path.exists(os.path.join(directory, ".mergin")):
+        if fs.exists(os.path.join(directory, ".mergin")):
             raise ClientError("Directory is already assigned to a Mergin Maps project (contains .mergin sub-dir)")
 
         if namespace and "/" not in project_name:
@@ -1242,7 +1243,7 @@ def get_file_diff(self, project_dir, file_path, output_diff, version_from, versi
         # concatenate diffs, if needed
         output_dir = os.path.dirname(output_diff)
         if len(diffs) >= 1:
-            os.makedirs(output_dir, exist_ok=True)
+            fs.makedirs(output_dir, exist_ok=True)
             if len(diffs) > 1:
                 mp.geodiff.concat_changes(diffs, output_diff)
             elif len(diffs) == 1:
@@ -1612,14 +1613,14 @@ def send_logs(
         local_logs_file_size_to_send = int(MAX_LOG_FILE_SIZE_TO_SEND * 0.8)
 
         global_logs = b""
-        if global_log_file and os.path.exists(global_log_file):
-            with open(global_log_file, "rb") as f:
-                if os.path.getsize(global_log_file) > global_logs_file_size_to_send:
+        if global_log_file and fs.exists(global_log_file):
+            with fs.open_file(global_log_file, "rb") as f:
+                if fs.getsize(global_log_file) > global_logs_file_size_to_send:
                     f.seek(-global_logs_file_size_to_send, os.SEEK_END)
                 global_logs = f.read() + b"\n--------------------------------\n\n"
 
-        with open(logfile, "rb") as f:
-            if os.path.getsize(logfile) > local_logs_file_size_to_send:
+        with fs.open_file(logfile, "rb") as f:
+            if fs.getsize(logfile) > local_logs_file_size_to_send:
                 f.seek(-local_logs_file_size_to_send, os.SEEK_END)
             logs = f.read()
 

mergin/client_pull.py

@@ -234,7 +234,7 @@ def _cleanup_failed_download(mergin_project: MerginProject = None):
     log_file = os.path.join(mergin_project.dir, ".mergin", "client-log.txt")
     dest_path = None
 
-    if os.path.exists(log_file):
+    if fs.exists(log_file):
         tmp_file = tempfile.NamedTemporaryFile(prefix="mergin-", suffix=".txt", delete=False)
         tmp_file.close()
         dest_path = tmp_file.name
@@ -251,9 +251,9 @@ def download_project_async(mc, project_path, directory, project_version=None):
 
     if "/" not in project_path:
         raise ClientError("Project name needs to be fully qualified, e.g. <username>/<projectname>")
-    if os.path.exists(directory):
+    if fs.exists(directory):
         raise ClientError("Project directory already exists")
-    os.makedirs(directory)
+    fs.makedirs(directory)
     mp = MerginProject(directory)
 
     mp.log.info("--- version: " + mc.user_agent_info())
@@ -409,7 +409,7 @@ def apply(self, directory, mp):
         else:
             file_dir = os.path.dirname(os.path.normpath(self.destination_file))
             dest_file_path = self.destination_file
-        os.makedirs(file_dir, exist_ok=True)
+        fs.makedirs(file_dir, exist_ok=True)
 
         # ignore check if we download not-latest version of gpkg file (possibly reconstructed on server on demand)
         check_size = self.latest_version or not mp.is_versioned_file(self.file_path)
@@ -993,5 +993,5 @@ def download_files_finalize(job: DownloadJob):
         task.apply(job.tmp_dir, job.mp)
 
     # Remove temporary download directory
-    if job.tmp_dir is not None and os.path.exists(job.tmp_dir.name):
+    if job.tmp_dir is not None and fs.exists(job.tmp_dir.name):
         cleanup_tmp_dir(job.mp, job.tmp_dir)

mergin/fs.py

@@ -43,6 +43,14 @@ def makedirs(path, exist_ok=False):
     os.makedirs(long_path(path), exist_ok=exist_ok)
 
 
+def mkdir(path):
+    os.mkdir(long_path(path))
+
+
+def rmtree(path):
+    shutil.rmtree(long_path(path))
+
+
 def connect(path):
     return sqlite3.connect(long_path(path))
 

mergin/merginproject.py

@@ -3,7 +3,6 @@
 import math
 import os
 import re
-import shutil
 from typing import List, Optional, Dict
 import typing
 import uuid
@@ -41,54 +40,6 @@
     import pygeodiff
 
 
-class GeoDiffLongPath:
-    """
-    Wraps a pygeodiff.GeoDiff instance so that every filesystem path passed to it is prefixed
-    with the Windows extended-length ("\\?\") marker.
-
-    Only the geodiff methods that take path arguments are listed explicitly.
-    """
-
-    def __init__(self, geodiff):
-        self._geodiff = geodiff
-
-    def create_changeset(self, base, modified, changeset):
-        return self._geodiff.create_changeset(long_path(base), long_path(modified), long_path(changeset))
-
-    def apply_changeset(self, base, changeset):
-        return self._geodiff.apply_changeset(long_path(base), long_path(changeset))
-
-    def rebase(self, base, modified_their, modified, conflict):
-        return self._geodiff.rebase(
-            long_path(base), long_path(modified_their), long_path(modified), long_path(conflict)
-        )
-
-    def make_copy_sqlite(self, src, dst):
-        return self._geodiff.make_copy_sqlite(long_path(src), long_path(dst))
-
-    def has_changes(self, changeset):
-        return self._geodiff.has_changes(long_path(changeset))
-
-    def changes_count(self, changeset):
-        return self._geodiff.changes_count(long_path(changeset))
-
-    def read_changeset(self, changeset):
-        return self._geodiff.read_changeset(long_path(changeset))
-
-    def list_changes_summary(self, changeset, json):
-        return self._geodiff.list_changes_summary(long_path(changeset), long_path(json))
-
-    def concat_changes(self, list_changesets, output_changeset):
-        return self._geodiff.concat_changes([long_path(p) for p in list_changesets], long_path(output_changeset))
-
-    def schema(self, driver, driver_info, src, json):
-        return self._geodiff.schema(driver, driver_info, long_path(src), long_path(json))
-
-    def __getattr__(self, name):
-        # everything without path arguments (set_logger_callback, level/table setters, version(), ...)
-        return getattr(self._geodiff, name)
-
-
 class MerginProject:
     """Base class for Mergin Maps local projects.
 
@@ -97,19 +48,19 @@ class MerginProject:
 
     def __init__(self, directory):
         self.dir = os.path.abspath(directory)
-        if not os.path.exists(self.dir):
+        if not fs.exists(self.dir):
             raise InvalidProject("Project directory does not exist")
 
         self.meta_dir = os.path.join(self.dir, ".mergin")
-        if not os.path.exists(self.meta_dir):
-            os.mkdir(self.meta_dir)
+        if not fs.exists(self.meta_dir):
+            fs.mkdir(self.meta_dir)
 
         # location for files from unfinished pull
         self.unfinished_pull_dir = os.path.join(self.meta_dir, "unfinished_pull")
 
         self.cache_dir = os.path.join(self.meta_dir, ".cache")
-        if not os.path.exists(self.cache_dir):
-            os.mkdir(self.cache_dir)
+        if not fs.exists(self.cache_dir):
+            fs.mkdir(self.cache_dir)
 
         # metadata from JSON are lazy loaded
         self._metadata = None
@@ -119,7 +70,7 @@ def __init__(self, directory):
 
         # make sure we can load correct pygeodiff
         try:
-            self.geodiff = GeoDiffLongPath(pygeodiff.GeoDiff())
+            self.geodiff = pygeodiff.GeoDiff()
         except pygeodiff.geodifflib.GeoDiffLibVersionError:
             # this is a fatal error, we can't live without geodiff
             self.log.error("Unable to load geodiff! (lib version error)")
@@ -145,7 +96,9 @@ def setup_logging(self, logger_name):
         if not self.log.handlers:
             # we only need to set the handler once
             # (otherwise we would get things logged multiple times as loggers are cached)
-            log_handler = logging.FileHandler(os.path.join(self.meta_dir, "client-log.txt"), encoding="utf-8")
+            log_handler = logging.FileHandler(
+                long_path(os.path.join(self.meta_dir, "client-log.txt")), encoding="utf-8"
+            )
             log_handler.setFormatter(logging.Formatter("%(asctime)s %(message)s"))
             self.log.addHandler(log_handler)
 
@@ -172,7 +125,7 @@ def fpath(self, file, other_dir=None):
         root = other_dir or self.dir
         abs_path = os.path.abspath(os.path.join(root, file))
         f_dir = os.path.dirname(abs_path)
-        os.makedirs(f_dir, exist_ok=True)
+        fs.makedirs(f_dir, exist_ok=True)
         return abs_path
 
     def fpath_meta(self, file):
@@ -280,9 +233,9 @@ def _read_metadata(self) -> None:
         """Loads the project's metadata from JSON"""
         if self._metadata is not None:
             return
-        if not os.path.exists(self.fpath_meta("mergin.json")):
+        if not fs.exists(self.fpath_meta("mergin.json")):
             raise InvalidProject("Project metadata has not been created yet")
-        with open(self.fpath_meta("mergin.json"), "r") as file:
+        with fs.open_file(self.fpath_meta("mergin.json"), "r") as file:
             self._metadata = json.load(file)
 
         self.is_old_metadata = "/" in self._metadata["name"]
@@ -302,9 +255,9 @@ def write_metadata(project_directory: str, data: dict):
         (and therefore creating MerginProject would fail).
         """
         meta_dir = os.path.join(project_directory, ".mergin")
-        os.makedirs(meta_dir, exist_ok=True)
+        fs.makedirs(meta_dir, exist_ok=True)
         metadata_json_file = os.path.abspath(os.path.join(meta_dir, "mergin.json"))
-        with open(metadata_json_file, "w") as file:
+        with fs.open_file(metadata_json_file, "w") as file:
             json.dump(data, file, indent=2)
 
     def is_versioned_file(self, file):
@@ -760,7 +713,7 @@ def copy_versioned_file_for_upload(self, f: FileChange, tmp_dir: str) -> str:
         path = f.path
         self.log.info("Making a temporary copy (full upload): " + path)
         tmp_file = os.path.join(tmp_dir, path)
-        os.makedirs(os.path.dirname(tmp_file), exist_ok=True)
+        fs.makedirs(os.path.dirname(tmp_file), exist_ok=True)
         self.geodiff.make_copy_sqlite(self.fpath(path), tmp_file)
         f.size = fs.getsize(tmp_file)
         f.checksum = generate_checksum(tmp_file)
@@ -777,10 +730,10 @@ def get_list_of_push_changes(self, push_changes):
                 result_file = self.fpath("change_list" + str(idx), self.meta_dir)
                 try:
                     self.geodiff.list_changes_summary(changeset, result_file)
-                    with open(result_file, "r") as f:
+                    with fs.open_file(result_file, "r") as f:
                         change = f.read()
                         changes[file["path"]] = json.loads(change)
-                    os.remove(result_file)
+                    fs.remove(result_file)
                 except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError):
                     pass
         return changes
@@ -1061,7 +1014,7 @@ def has_unfinished_pull(self):
         :returns: whether there is an unfinished pull
         :rtype: bool
         """
-        return os.path.exists(self.unfinished_pull_dir)
+        return fs.exists(self.unfinished_pull_dir)
 
     def resolve_unfinished_pull(self, user_name):
         """
@@ -1093,9 +1046,8 @@ def resolve_unfinished_pull(self, user_name):
 
         for root, dirs, files in fs.walk(self.unfinished_pull_dir):
             for file_name in files:
-                src = os.path.join(root, file_name)
-                # the relpath base must be prefixed as well to strip it correctly.
-                file_path = os.path.relpath(src, long_path(self.unfinished_pull_dir))
+                file_path = os.path.relpath(os.path.join(root, file_name), long_path(self.unfinished_pull_dir))
+                src = self.fpath_unfinished_pull(file_path)
                 dest = self.fpath(file_path)
                 basefile = self.fpath_meta(file_path)
 
@@ -1116,7 +1068,7 @@ def resolve_unfinished_pull(self, user_name):
                     self.log.error("unable to apply changes from previous unfinished pull!")
                     raise ClientError("Unable to resolve unfinished pull!")
 
-        shutil.rmtree(self.unfinished_pull_dir)
+        fs.rmtree(self.unfinished_pull_dir)
         self.log.info("unfinished pull resolved successfuly!")
         return conflicts
 
@@ -1162,7 +1114,7 @@ def get_geodiff_changes_count(self, diff_rel_path: str):
 
         diff_abs = self.fpath_meta(diff_rel_path)
         try:
-            return GeoDiffLongPath(pygeodiff.GeoDiff()).changes_count(diff_abs)
+            return pygeodiff.GeoDiff().changes_count(diff_abs)
         except (
             pygeodiff.GeoDiffLibError,
             pygeodiff.GeoDiffLibConflictError,

mergin/report.py

@@ -286,8 +286,8 @@ def create_report(mc, directory, since, to, out_file):
 
     # export report to csv file
     out_dir = os.path.dirname(out_file)
-    os.makedirs(out_dir, exist_ok=True)
-    with open(out_file, "w", newline="") as f_csv:
+    fs.makedirs(out_dir, exist_ok=True)
+    with fs.open_file(out_file, "w", newline="") as f_csv:
         writer = csv.DictWriter(f_csv, fieldnames=headers)
         writer.writeheader()
         writer.writerows(records)

mergin/utils.py

@@ -52,8 +52,8 @@ def save_to_file(stream, path):
 
 def move_file(src, dest):
     dest_dir = os.path.dirname(dest)
-    os.makedirs(dest_dir, exist_ok=True)
-    os.rename(src, dest)
+    os.makedirs(long_path(dest_dir), exist_ok=True)
+    os.rename(long_path(src), long_path(dest))
 
 
 class DateTimeEncoder(json.JSONEncoder):
@@ -167,13 +167,13 @@ def unique_path_name(path):
     """
     unique_path = str(path)
 
-    is_dir = os.path.isdir(path)
+    is_dir = os.path.isdir(long_path(path))
     head, tail = os.path.split(os.path.normpath(path))
     ext = "".join(Path(tail).suffixes)
     file_name = tail.replace(ext, "")
 
     i = 0
-    while os.path.exists(unique_path):
+    while os.path.exists(long_path(unique_path)):
         i += 1
 
         if is_dir: