fix replication repair files alignment in deep trees

Signed-off-by: Stephen L. <lrq3000@gmail.com>

Author: lrq3000

pyFileFixity/lib/aux_funcs.py

@@ -69,13 +69,17 @@ def sizeof_fmt(num, suffix='B', mod=1024.0):
         num /= mod
     return "%.1f%s%s" % (num, 'Y', suffix)
 
-def path2unix(path, fromwinpath=False):
+def path2unix(path, nojoin=False, fromwinpath=False):
     '''From a path given in any format, converts to posix path format
     fromwinpath=True forces the input path to be recognized as a Windows path (useful on Unix machines to unit test Windows paths)'''
     if fromwinpath:
-        return posixpath.join(*list(PureWindowsPath(path).parts))
+        pathparts = list(PureWindowsPath(path).parts)
     else:
-        return posixpath.join(*list(PurePath(path).parts))
+        pathparts = list(PurePath(path).parts)
+    if nojoin:
+        return pathparts
+    else:
+        return posixpath.join(*pathparts)
 
 def get_next_entry(file, entrymarker="\xFE\xFF\xFE\xFF\xFE\xFF\xFE\xFF\xFE\xFF", only_coord=True, blocksize=65535):
     '''Find or read the next ecc entry in a given ecc file.

pyFileFixity/replication_repair.py

@@ -61,15 +61,27 @@
 
 def relpath_posix(recwalk_result, pardir, fromwinpath=False):
     ''' Helper function to convert all paths to relative posix like paths (to ease comparison) '''
-    return recwalk_result[0], os.path.split(path2unix(os.path.join(os.path.relpath(recwalk_result[0], pardir),recwalk_result[1]), fromwinpath=fromwinpath))
+    return recwalk_result[0], path2unix(os.path.join(os.path.relpath(recwalk_result[0], pardir),recwalk_result[1]), nojoin=True, fromwinpath=fromwinpath)
 
 #def checkAllEqual(lst):
 #    return not lst or [lst[0]]*len(lst) == lst
 
+def sort_dict_of_paths(d):
+    """ Sort a dict containing paths parts (ie, paths divided in parts and stored as a list). Top paths will be given precedence over deeper paths. """
+    # Find the path that is the deepest, and count the number of parts
+    max_rec = max(len(x) if x else 0 for x in d.values())
+    # Pad other paths with empty parts to fill in, so that all paths will have the same number of parts (necessary to compare correctly, else deeper paths may get precedence over top ones, since the folder name will be compared to filenames!)
+    for key in d.keys():
+        if d[key]:
+            d[key] = ['']*(max_rec-len(d[key])) + d[key]
+    # Sort the dict relatively to the paths alphabetical order
+    d_sort = sorted(d.items(), key=lambda x: x[1])
+    return d_sort
+
 def sort_group(d, return_only_first=False):
     ''' Sort a dictionary of relative paths and cluster equal paths together at the same time '''
     # First, sort the paths in order (this must be a couple: (parent_dir, filename), so that there's no ambiguity because else a file at root will be considered as being after a folder/file since the ordering is done alphabetically without any notion of tree structure).
-    d_sort = sorted(d.items(), key=lambda x: x[1])
+    d_sort = sort_dict_of_paths(d)
     # Pop the first item in the ordered list
     base_elt = (-1, None)
     while (base_elt[1] is None and d_sort):
@@ -219,7 +231,9 @@ def majority_vote_byte_scan(relfilepath, fileslist, outpath, blocksize=65535, de
 
 def synchronize_files(inputpaths, outpath, database=None, tqdm_bar=None, report_file=None, ptee=None, verbose=False):
     ''' Main function to synchronize files contents by majority vote
-    The main job of this function is to walk through the input folders and align the files, so that we can compare every files across every folders, one by one.'''
+    The main job of this function is to walk through the input folders and align the files, so that we can compare every files across every folders, one by one.
+    The whole trick here is to align files, so that we don't need to memorize all the files in memory and we compare all equivalent files together: to do that, we ensure that we walk through the input directories in alphabetical order, and we pick the relative filepath at the top of the alphabetical order, this ensures the alignment of files between different folders, without memorizing  the whole trees structures.
+    '''
     # (Generator) Files Synchronization Algorithm:
     # Needs a function stable_dir_walking, which will walk through directories recursively but in always the same order on all platforms (same order for files but also for folders), whatever order it is, as long as it is stable.
     # Until there's no file in any of the input folders to be processed:

pyFileFixity/tests/test_aux_funcs.py

@@ -52,7 +52,8 @@ def test_sizeof_fmt():
 
 def test_path2unix():
     """ aux: test path2unix """
-    assert auxf.path2unix(r'test\some\folder\file.ext', True) == r'test/some/folder/file.ext'
+    assert auxf.path2unix(r'test\some\folder\file.ext', fromwinpath=True) == r'test/some/folder/file.ext'
+    assert auxf.path2unix(r'test\some\folder\file.ext', nojoin=True, fromwinpath=True) == ['test', 'some', 'folder', 'file.ext']
     assert auxf.path2unix(r'test/some/folder/file.ext') == r'test/some/folder/file.ext'
 
 def test_is_file():

pyFileFixity/tests/test_replication_repair.py

@@ -27,30 +27,44 @@ def test_relpath_posix():
     """ repli: test internal: relpath_posix()"""
     recwalk_result = [r'C:\test\some\path', r'relative\path\file.ext']
     pardir = r'C:\test\some'
-    assert rep.relpath_posix(recwalk_result, pardir, True) == ('C:\\test\\some\\path', ('path/relative/path', 'file.ext'))
+    print(rep.relpath_posix(recwalk_result, pardir, True))
+    assert rep.relpath_posix(recwalk_result, pardir, True) == ('C:\\test\\some\\path', ['path', 'relative', 'path', 'file.ext'])
     recwalk_result = [r'/test/some/path', r'relative/path/file.ext']
     pardir = r'/test/some'
-    assert rep.relpath_posix(recwalk_result, pardir, False) == ('/test/some/path', ('path/relative/path', 'file.ext'))
+    assert rep.relpath_posix(recwalk_result, pardir, False) == ('/test/some/path', ['path', 'relative', 'path', 'file.ext'])
     recwalk_result = [r'/test/some/path', r'relative\path\file.ext']
     pardir = r'/test/some'
-    assert rep.relpath_posix(recwalk_result, pardir, True) == ('/test/some/path', ('path/relative/path', 'file.ext'))
+    assert rep.relpath_posix(recwalk_result, pardir, True) == ('/test/some/path', ['path', 'relative', 'path', 'file.ext'])
+
+def test_sort_dict_of_paths():
+    d = {0: ['testoo.TXT'], 1: ['testoo.TXT'], 2: ['testbb-more.TXT'], 3: ['sub', 'testsub.TXT']}
+    d_sort = rep.sort_dict_of_paths(d)
+    assert d_sort == [(2, ['', 'testbb-more.TXT']),
+                      (0, ['', 'testoo.TXT']),
+                      (1, ['', 'testoo.TXT']),
+                      (3, ['sub', 'testsub.TXT'])]
 
 def test_sort_group():
     """ repli: test internal: sort_group()"""
+    # Generate an artificial tree, with some relative paths being the same across multiple folders,
+    # and some others in different tree depth (very import to check that it works OK!)
     curfiles = {
-                0: ('relative/path', 'file.ext'),
-                1: ('relative/path', 'file.ext'),
-                2: ('relative/path', 'zzzz.ext'),
-                3: ('relative/path', 'zzzz.ext'),
-                4: ('relative/path', 'bbbb.ext'),
+                0: ['relative', 'path', 'file.ext'],
+                1: ['relative', 'path', 'file.ext'],
+                2: ['relative', 'aaa', 'zzzz.ext'],
+                3: ['zzzz.ext'],
+                4: ['relative', 'zzzz.ext'],
+                5: ['relative', 'path', 'bbbb.ext'],
                }
     assert rep.sort_group(curfiles, return_only_first=False) == \
               [
-               [(4, ('relative/path', 'bbbb.ext'))],
-               [(0, ('relative/path', 'file.ext')), (1, ('relative/path', 'file.ext'))],
-               [(2, ('relative/path', 'zzzz.ext')), (3, ('relative/path', 'zzzz.ext'))]
+               [(3, ['', '', 'zzzz.ext'])],
+               [(4, ['', 'relative', 'zzzz.ext'])],
+               [(2, ['relative', 'aaa', 'zzzz.ext'])],
+               [(5, ['relative', 'path', 'bbbb.ext'])],
+               [(0, ['relative', 'path', 'file.ext']), (1, ['relative', 'path', 'file.ext'])]
               ]
-    assert rep.sort_group(curfiles, return_only_first=True) == [[(4, ('relative/path', 'bbbb.ext'))]]
+    assert rep.sort_group(curfiles, return_only_first=True) == [[(3, ['', '', 'zzzz.ext'])]]
 
 def test_majority_vote_byte_scan():
     """ repli: test internal: majority_vote_byte_scan()"""