diff --git a/FileDeduplicator.Test/DeduplicatorTests.cs b/FileDeduplicator.Test/DeduplicatorTests.cs new file mode 100644 index 0000000..89b7a34 --- /dev/null +++ b/FileDeduplicator.Test/DeduplicatorTests.cs @@ -0,0 +1,249 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.FileDeduplicator.Test; + +using ktsu.Semantics.Paths; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Tests for , the type that decides which files are deleted. +/// +[TestClass] +public sealed class DeduplicatorTests +{ + private static Dictionary Hash(TempTree tree, params string[] relativePaths) => + FileHasher.HashFiles([.. relativePaths.Select(p => tree.Write(p, "same content"))]); + + private static IReadOnlyList Duplicates(Dictionary hashes) => + Deduplicator.FindDuplicates(Deduplicator.GroupByHash(hashes)); + + /// + /// Identical content must group together regardless of filename or directory. + /// + [TestMethod] + public void IdenticalContentGroupsTogetherAcrossDirectories() + { + // Arrange + using TempTree tree = new(); + Dictionary hashes = Hash(tree, "a.txt", "nested/b.txt", "nested/deep/c.txt"); + + // Act + IReadOnlyList duplicates = Duplicates(hashes); + + // Assert + Assert.ContainsSingle(duplicates); + Assert.HasCount(3, duplicates[0].Files); + } + + /// + /// Distinct content must not be grouped, so nothing is proposed for deletion. + /// + [TestMethod] + public void DistinctContentProducesNoDuplicateGroups() + { + // Arrange + using TempTree tree = new(); + Dictionary hashes = FileHasher.HashFiles( + [ + tree.Write("a.txt", "one"), + tree.Write("b.txt", "two"), + tree.Write("c.txt", "three"), + ]); + + // Act + IReadOnlyList duplicates = Duplicates(hashes); + + // Assert + Assert.IsEmpty(duplicates); + } + + /// + /// A file with no twin must never be offered for deletion. + /// + [TestMethod] + public void ASingleCopyIsNeverADuplicate() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath lonely = tree.Write("lonely.txt", "unique"); + Dictionary hashes = FileHasher.HashFiles( + [ + lonely, + tree.Write("dup-a.txt", "shared"), + tree.Write("dup-bb.txt", "shared"), + ]); + + // Act + IReadOnlyList duplicates = Duplicates(hashes); + + // Assert + Assert.ContainsSingle(duplicates); + Assert.IsFalse(duplicates[0].Files.Contains(lonely), "A file with unique content must not appear in a duplicate group."); + } + + /// + /// Empty files hash identically to each other and must be treated as duplicates. + /// + [TestMethod] + public void EmptyFilesAreDuplicatesOfEachOther() + { + // Arrange + using TempTree tree = new(); + Dictionary hashes = FileHasher.HashFiles( + [ + tree.Write("empty-a.txt", string.Empty), + tree.Write("empty-b.txt", string.Empty), + ]); + + // Act + IReadOnlyList duplicates = Duplicates(hashes); + + // Assert + Assert.ContainsSingle(duplicates); + Assert.AreEqual(0, duplicates[0].FileSize); + } + + /// + /// The documented rule is "keep the copy with the shortest filename". + /// + [TestMethod] + public void TheShortestFileNameIsKept() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath shortest = tree.Write("a.txt", "x"); + List group = + [ + tree.Write("aaaa.txt", "x"), + shortest, + tree.Write("aa.txt", "x"), + ]; + + // Act + AbsoluteFilePath keeper = Deduplicator.SelectFileToKeep(group); + + // Assert + Assert.AreEqual(shortest, keeper); + } + + /// + /// Only the file name length is compared, not the length of the whole path -- a deeply nested + /// file with a short name still wins over a shallow one with a long name. + /// + [TestMethod] + public void OnlyTheFileNameLengthDecidesNotThePathLength() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath deepButShortName = tree.Write("one/two/three/four/a.txt", "x"); + List group = + [ + tree.Write("bbbbbbbb.txt", "x"), + deepButShortName, + ]; + + // Act + AbsoluteFilePath keeper = Deduplicator.SelectFileToKeep(group); + + // Assert + Assert.AreEqual(deepButShortName, keeper); + } + + /// + /// Ties on name length must break deterministically, or which copy survives would depend on + /// enumeration order and differ between runs and platforms. + /// + [TestMethod] + public void TiesOnNameLengthBreakDeterministicallyByPath() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath a = tree.Write("aa.txt", "x"); + AbsoluteFilePath b = tree.Write("bb.txt", "x"); + AbsoluteFilePath c = tree.Write("cc.txt", "x"); + + // Act -- the same set in three different orders + AbsoluteFilePath first = Deduplicator.SelectFileToKeep([a, b, c]); + AbsoluteFilePath second = Deduplicator.SelectFileToKeep([c, b, a]); + AbsoluteFilePath third = Deduplicator.SelectFileToKeep([b, c, a]); + + // Assert + Assert.AreEqual(first, second); + Assert.AreEqual(second, third); + Assert.AreEqual(a, first, "The ordinally-first path should win a tie on name length."); + } + + /// + /// Deletion must remove every copy except the keeper, and must not touch anything else. + /// + [TestMethod] + public void DeleteDuplicatesRemovesEveryCopyExceptTheKeeper() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath unrelated = tree.Write("unrelated.txt", "different content"); + Dictionary hashes = FileHasher.HashFiles( + [ + unrelated, + tree.Write("a.txt", "shared"), + tree.Write("bb.txt", "shared"), + tree.Write("ccc.txt", "shared"), + ]); + IReadOnlyList duplicates = Duplicates(hashes); + AbsoluteFilePath keeper = Deduplicator.SelectFileToKeep(duplicates[0].Files); + + // Act + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + + // Assert + Assert.AreEqual(2, result.DeletedCount); + Assert.IsEmpty(result.Errors); + Assert.IsTrue(TempTree.Exists(keeper), "The keeper must survive."); + Assert.IsTrue(TempTree.Exists(unrelated), "A file outside any duplicate group must be untouched."); + + foreach (AbsoluteFilePath file in duplicates[0].Files.Where(f => f != keeper)) + { + Assert.IsFalse(TempTree.Exists(file), $"{file} should have been deleted."); + } + } + + /// + /// The reclaimed byte count must reflect what was actually removed. + /// + [TestMethod] + public void ReclaimedBytesCountsOnlyTheDeletedCopies() + { + // Arrange + using TempTree tree = new(); + string content = new('x', 100); + Dictionary hashes = FileHasher.HashFiles( + [ + tree.Write("a.txt", content), + tree.Write("bb.txt", content), + tree.Write("ccc.txt", content), + ]); + IReadOnlyList duplicates = Duplicates(hashes); + + // Act + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + + // Assert -- three copies, two deleted, 100 bytes apiece + Assert.AreEqual(2, result.DeletedCount); + Assert.AreEqual(200, result.BytesReclaimed); + } + + /// + /// Deleting nothing must report nothing, rather than throwing on an empty group list. + /// + [TestMethod] + public void DeletingAnEmptyGroupListIsANoOp() + { + // Act + DeduplicationResult result = Deduplicator.DeleteDuplicates([]); + + // Assert + Assert.AreEqual(0, result.DeletedCount); + Assert.AreEqual(0, result.BytesReclaimed); + Assert.IsEmpty(result.Errors); + } +} diff --git a/FileDeduplicator.Test/DryRunEquivalenceTests.cs b/FileDeduplicator.Test/DryRunEquivalenceTests.cs new file mode 100644 index 0000000..ebf26e0 --- /dev/null +++ b/FileDeduplicator.Test/DryRunEquivalenceTests.cs @@ -0,0 +1,148 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.FileDeduplicator.Test; + +using ktsu.Semantics.Paths; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Tests that the set of files DryRun says it would delete is exactly the set Deduplicate does +/// delete. +/// +/// +/// DryRun is the safety net users rely on before letting this tool remove anything, so a +/// divergence between the two is the worst defect this codebase could have: the preview would be +/// a lie, and the thing it lied about is irreversible deletion. +/// +/// The two verbs share , +/// and , so +/// today they agree by construction. That is worth pinning rather than assuming: it holds only +/// while both keep calling the same three methods and while SelectFileToKeep stays deterministic, +/// and neither is enforced by anything but this test. +/// +[TestClass] +public sealed class DryRunEquivalenceTests +{ + /// + /// Computes the deletion set the way DryRun reports it -- every file in every duplicate group + /// except that group's keeper. + /// + private static HashSet PredictedDeletions(IReadOnlyList duplicates) + { + HashSet predicted = []; + + foreach (DuplicateGroup group in duplicates) + { + AbsoluteFilePath keeper = Deduplicator.SelectFileToKeep(group.Files); + foreach (AbsoluteFilePath file in group.Files.Where(f => f != keeper)) + { + _ = predicted.Add(file); + } + } + + return predicted; + } + + /// + /// Across a tree with several duplicate groups, unique files and nested directories, the + /// predicted deletion set must match the actual one exactly -- no file deleted that was not + /// predicted, and none predicted that survived. + /// + [TestMethod] + public void PredictedDeletionsMatchActualDeletionsExactly() + { + // Arrange -- three groups of duplicates plus two unique files, spread over nested folders + using TempTree tree = new(); + List all = + [ + tree.Write("group1/a.txt", "alpha"), + tree.Write("group1/aa.txt", "alpha"), + tree.Write("group1/nested/aaa.txt", "alpha"), + tree.Write("group2/b.txt", "beta"), + tree.Write("group2/bb.txt", "beta"), + tree.Write("group3/deep/c.txt", "gamma"), + tree.Write("group3/cc.txt", "gamma"), + tree.Write("unique-one.txt", "delta"), + tree.Write("nested/unique-two.txt", "epsilon"), + ]; + + Dictionary hashes = FileHasher.HashFiles(all); + IReadOnlyList duplicates = Deduplicator.FindDuplicates(Deduplicator.GroupByHash(hashes)); + HashSet predicted = PredictedDeletions(duplicates); + + // Act + DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates); + HashSet actuallyDeleted = [.. all.Where(f => !TempTree.Exists(f))]; + + // Assert + Assert.HasCount(3, duplicates, "Three groups of duplicates were created."); + Assert.AreEqual(predicted.Count, result.DeletedCount); + Assert.IsTrue( + predicted.SetEquals(actuallyDeleted), + $"Predicted [{string.Join(", ", predicted)}] but deleted [{string.Join(", ", actuallyDeleted)}]."); + } + + /// + /// Every duplicate group must keep exactly one survivor. Deleting a whole group would destroy + /// the content entirely, which is the failure mode that matters most here. + /// + [TestMethod] + public void EveryDuplicateGroupRetainsExactlyOneSurvivor() + { + // Arrange + using TempTree tree = new(); + List all = + [ + tree.Write("a.txt", "alpha"), + tree.Write("aa.txt", "alpha"), + tree.Write("aaa.txt", "alpha"), + tree.Write("b.txt", "beta"), + tree.Write("bb.txt", "beta"), + ]; + + Dictionary hashes = FileHasher.HashFiles(all); + IReadOnlyList duplicates = Deduplicator.FindDuplicates(Deduplicator.GroupByHash(hashes)); + + // Act + _ = Deduplicator.DeleteDuplicates(duplicates); + + // Assert + foreach (DuplicateGroup group in duplicates) + { + int survivors = group.Files.Count(TempTree.Exists); + Assert.AreEqual(1, survivors, $"Group {group.Hash[..12]} should retain exactly one file."); + } + } + + /// + /// Running the whole pipeline twice must be a no-op the second time: after deduplication there + /// are no duplicates left to find. + /// + [TestMethod] + public void DeduplicatingATreeTwiceDeletesNothingTheSecondTime() + { + // Arrange + using TempTree tree = new(); + _ = tree.Write("a.txt", "alpha"); + _ = tree.Write("aa.txt", "alpha"); + _ = tree.Write("b.txt", "beta"); + + // Act -- first pass + IReadOnlyList firstScan = FileScanner.ScanForFiles(tree.Root); + IReadOnlyList firstDuplicates = + Deduplicator.FindDuplicates(Deduplicator.GroupByHash(FileHasher.HashFiles(firstScan))); + DeduplicationResult first = Deduplicator.DeleteDuplicates(firstDuplicates); + + // Act -- second pass over the now-deduplicated tree + IReadOnlyList secondScan = FileScanner.ScanForFiles(tree.Root); + IReadOnlyList secondDuplicates = + Deduplicator.FindDuplicates(Deduplicator.GroupByHash(FileHasher.HashFiles(secondScan))); + DeduplicationResult second = Deduplicator.DeleteDuplicates(secondDuplicates); + + // Assert + Assert.AreEqual(1, first.DeletedCount); + Assert.IsEmpty(secondDuplicates); + Assert.AreEqual(0, second.DeletedCount); + Assert.HasCount(2, secondScan, "Both distinct contents should survive."); + } +} diff --git a/FileDeduplicator.Test/FileDeduplicator.Test.csproj b/FileDeduplicator.Test/FileDeduplicator.Test.csproj new file mode 100644 index 0000000..81934cd --- /dev/null +++ b/FileDeduplicator.Test/FileDeduplicator.Test.csproj @@ -0,0 +1,14 @@ + + + + + + true + net10.0 + + + + + + + diff --git a/FileDeduplicator.Test/FileScannerAndHasherTests.cs b/FileDeduplicator.Test/FileScannerAndHasherTests.cs new file mode 100644 index 0000000..9f65b9a --- /dev/null +++ b/FileDeduplicator.Test/FileScannerAndHasherTests.cs @@ -0,0 +1,167 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.FileDeduplicator.Test; + +using ktsu.Semantics.Paths; +using ktsu.Semantics.Strings; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Tests for and , the two stages that decide +/// what the deduplicator sees. +/// +[TestClass] +public sealed class FileScannerAndHasherTests +{ + /// + /// Scanning must recurse, since duplicates across nested folders are the main case. + /// + [TestMethod] + public void ScanFindsFilesInNestedDirectories() + { + // Arrange + using TempTree tree = new(); + _ = tree.Write("top.txt", "a"); + _ = tree.Write("one/mid.txt", "b"); + _ = tree.Write("one/two/three/deep.txt", "c"); + + // Act + IReadOnlyList files = FileScanner.ScanForFiles(tree.Root); + + // Assert + Assert.HasCount(3, files); + } + + /// + /// An empty directory must yield no files rather than throwing. + /// + [TestMethod] + public void ScanOfAnEmptyDirectoryReturnsNothing() + { + // Arrange + using TempTree tree = new(); + + // Act + IReadOnlyList files = FileScanner.ScanForFiles(tree.Root); + + // Assert + Assert.IsEmpty(files); + } + + /// + /// A path that does not exist must return an empty list rather than throwing, so a mistyped + /// argument does not crash the tool. + /// + [TestMethod] + public void ScanOfAMissingDirectoryReturnsNothing() + { + // Arrange + using TempTree tree = new(); + AbsoluteDirectoryPath missing = Path.Combine(tree.Root.WeakString, "does-not-exist").As(); + + // Act + IReadOnlyList files = FileScanner.ScanForFiles(missing); + + // Assert + Assert.IsEmpty(files); + } + + /// + /// Identical content must hash identically regardless of the file's name or location. + /// + [TestMethod] + public void IdenticalContentHashesIdenticallyRegardlessOfName() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath first = tree.Write("first.txt", "identical"); + AbsoluteFilePath second = tree.Write("nested/second-with-a-longer-name.txt", "identical"); + + // Act + string firstHash = FileHasher.ComputeHash(first); + string secondHash = FileHasher.ComputeHash(second); + + // Assert + Assert.AreEqual(firstHash, secondHash); + } + + /// + /// Differing content must hash differently, including a difference of a single character. + /// + [TestMethod] + public void DifferingContentHashesDifferently() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath first = tree.Write("a.txt", "content"); + AbsoluteFilePath second = tree.Write("b.txt", "contenu"); + + // Act & Assert + Assert.AreNotEqual(FileHasher.ComputeHash(first), FileHasher.ComputeHash(second)); + } + + /// + /// The hash is SHA-256 rendered as lowercase hex, which the console output slices to 12 + /// characters -- so it must be long enough for that slice and contain no uppercase. + /// + [TestMethod] + public void HashIsLowercaseHexOfTheExpectedLength() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath file = tree.Write("a.txt", "content"); + + // Act + string hash = FileHasher.ComputeHash(file); + + // Assert + Assert.HasCount(64, hash, "SHA-256 is 32 bytes, so 64 hex characters."); + Assert.AreEqual(hash.ToLowerInvariant(), hash, "Callers slice this for display and compare it as-is."); + Assert.IsTrue(hash.All(Uri.IsHexDigit), "Every character should be a hex digit."); + } + + /// + /// An empty file must hash to the well-known SHA-256 of zero bytes, rather than failing. + /// + [TestMethod] + public void AnEmptyFileHashesToTheEmptySha256() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath empty = tree.Write("empty.txt", string.Empty); + + // Act + string hash = FileHasher.ComputeHash(empty); + + // Assert + Assert.AreEqual("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", hash); + } + + /// + /// The parallel hashing path must return one entry per input file, with the same values the + /// single-file path produces. + /// + [TestMethod] + public void ParallelHashingAgreesWithSingleFileHashing() + { + // Arrange + using TempTree tree = new(); + List files = + [ + tree.Write("a.txt", "one"), + tree.Write("b.txt", "two"), + tree.Write("c.txt", "one"), + tree.Write("nested/d.txt", "three"), + ]; + + // Act + Dictionary hashes = FileHasher.HashFiles(files); + + // Assert + Assert.HasCount(files.Count, hashes); + foreach (AbsoluteFilePath file in files) + { + Assert.AreEqual(FileHasher.ComputeHash(file), hashes[file]); + } + } +} diff --git a/FileDeduplicator.Test/TempTree.cs b/FileDeduplicator.Test/TempTree.cs new file mode 100644 index 0000000..5e06822 --- /dev/null +++ b/FileDeduplicator.Test/TempTree.cs @@ -0,0 +1,76 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.FileDeduplicator.Test; + +using System.Runtime.CompilerServices; + +using ktsu.Semantics.Paths; +using ktsu.Semantics.Strings; + +/// +/// A throwaway directory tree under the system temp path, deleted on disposal. +/// +/// +/// These tests drive the real filesystem rather than an abstraction, because that is what the +/// production code uses -- calls +/// and calls directly. Faking those would +/// test a seam that does not exist and would not catch a mistake in the delete path, which is the +/// part of this tool that destroys data. +/// +internal sealed class TempTree : IDisposable +{ + /// + /// Gets the root of the throwaway tree. + /// + internal AbsoluteDirectoryPath Root { get; } + + /// + /// Creates a tree in a directory named after the calling test, so a leaked directory names + /// the test that leaked it. + /// + /// Supplied by the compiler; do not pass explicitly. + internal TempTree([CallerMemberName] string caller = "") + { + string path = Path.Combine(Path.GetTempPath(), $"ktsu-dedupe-{caller}-{Guid.NewGuid():N}"); + _ = Directory.CreateDirectory(path); + Root = path.As(); + } + + /// + /// Writes a file with the given content, creating any intermediate directories. + /// + /// Path relative to , using forward slashes. + /// The content to write. + /// The absolute path of the written file. + internal AbsoluteFilePath Write(string relativePath, string content) + { + string full = Path.Combine(Root.WeakString, relativePath.Replace('/', Path.DirectorySeparatorChar)); + _ = Directory.CreateDirectory(Path.GetDirectoryName(full)!); + File.WriteAllText(full, content); + return full.As(); + } + + /// + /// Gets whether a file written through still exists. + /// + /// The file to check. + /// if it is still on disk. + internal static bool Exists(AbsoluteFilePath path) => File.Exists(path.WeakString); + + /// + public void Dispose() + { + try + { + Directory.Delete(Root.WeakString, recursive: true); + } + catch (DirectoryNotFoundException) + { + // Already gone; nothing to clean up. + } + catch (IOException) + { + // A leaked temp directory is not worth failing an otherwise passing test over. + } + } +} diff --git a/FileDeduplicator.slnx b/FileDeduplicator.slnx index b1778f2..8eb3390 100644 --- a/FileDeduplicator.slnx +++ b/FileDeduplicator.slnx @@ -1,3 +1,4 @@ + diff --git a/FileDeduplicator/Properties/AssemblyInfo.cs b/FileDeduplicator/Properties/AssemblyInfo.cs index 7fc7e2a..9704eda 100644 --- a/FileDeduplicator/Properties/AssemblyInfo.cs +++ b/FileDeduplicator/Properties/AssemblyInfo.cs @@ -1,4 +1,9 @@ // Copyright (c) 2023-2026 ktsu-dev contributors +// NOTE: no "ktsu." prefix. This repository has no AUTHORS.md, so ktsu.Sdk resolves an empty +// AuthorsNamespace and the assembly is named FileDeduplicator rather than ktsu.FileDeduplicator +// the way every other repository in the organization is. Tracked separately; if that is corrected +// this literal has to move with it or the test project silently loses access to internals. +[assembly: System.Runtime.CompilerServices.InternalsVisibleTo("FileDeduplicator.Test")] [assembly: CLSCompliant(false)] [assembly: System.Runtime.InteropServices.ComVisible(false)]