Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
249 changes: 249 additions & 0 deletions FileDeduplicator.Test/DeduplicatorTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,249 @@
// Copyright (c) 2023-2026 ktsu-dev contributors

namespace ktsu.FileDeduplicator.Test;

using ktsu.Semantics.Paths;
using Microsoft.VisualStudio.TestTools.UnitTesting;

/// <summary>
/// Tests for <see cref="Deduplicator"/>, the type that decides which files are deleted.
/// </summary>
[TestClass]
public sealed class DeduplicatorTests
{
private static Dictionary<AbsoluteFilePath, string> Hash(TempTree tree, params string[] relativePaths) =>
FileHasher.HashFiles([.. relativePaths.Select(p => tree.Write(p, "same content"))]);

private static IReadOnlyList<DuplicateGroup> Duplicates(Dictionary<AbsoluteFilePath, string> hashes) =>
Deduplicator.FindDuplicates(Deduplicator.GroupByHash(hashes));

/// <summary>
/// Identical content must group together regardless of filename or directory.
/// </summary>
[TestMethod]
public void IdenticalContentGroupsTogetherAcrossDirectories()
{
// Arrange
using TempTree tree = new();
Dictionary<AbsoluteFilePath, string> hashes = Hash(tree, "a.txt", "nested/b.txt", "nested/deep/c.txt");

// Act
IReadOnlyList<DuplicateGroup> duplicates = Duplicates(hashes);

// Assert
Assert.ContainsSingle(duplicates);
Assert.HasCount(3, duplicates[0].Files);
}

/// <summary>
/// Distinct content must not be grouped, so nothing is proposed for deletion.
/// </summary>
[TestMethod]
public void DistinctContentProducesNoDuplicateGroups()
{
// Arrange
using TempTree tree = new();
Dictionary<AbsoluteFilePath, string> hashes = FileHasher.HashFiles(
[
tree.Write("a.txt", "one"),
tree.Write("b.txt", "two"),
tree.Write("c.txt", "three"),
]);

// Act
IReadOnlyList<DuplicateGroup> duplicates = Duplicates(hashes);

// Assert
Assert.IsEmpty(duplicates);
}

/// <summary>
/// A file with no twin must never be offered for deletion.
/// </summary>
[TestMethod]
public void ASingleCopyIsNeverADuplicate()
{
// Arrange
using TempTree tree = new();
AbsoluteFilePath lonely = tree.Write("lonely.txt", "unique");
Dictionary<AbsoluteFilePath, string> hashes = FileHasher.HashFiles(
[
lonely,
tree.Write("dup-a.txt", "shared"),
tree.Write("dup-bb.txt", "shared"),
]);

// Act
IReadOnlyList<DuplicateGroup> duplicates = Duplicates(hashes);

// Assert
Assert.ContainsSingle(duplicates);
Assert.IsFalse(duplicates[0].Files.Contains(lonely), "A file with unique content must not appear in a duplicate group.");

Check warning on line 81 in FileDeduplicator.Test/DeduplicatorTests.cs

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Use 'Assert.DoesNotContain' instead of 'Assert.IsFalse'

See more on https://sonarcloud.io/project/issues?id=ktsu-dev_FileDeduplicator&issues=AaBBrzg8Rb2kSLNmH5CU&open=AaBBrzg8Rb2kSLNmH5CU&pullRequest=98
}

/// <summary>
/// Empty files hash identically to each other and must be treated as duplicates.
/// </summary>
[TestMethod]
public void EmptyFilesAreDuplicatesOfEachOther()
{
// Arrange
using TempTree tree = new();
Dictionary<AbsoluteFilePath, string> hashes = FileHasher.HashFiles(
[
tree.Write("empty-a.txt", string.Empty),
tree.Write("empty-b.txt", string.Empty),
]);

// Act
IReadOnlyList<DuplicateGroup> duplicates = Duplicates(hashes);

// Assert
Assert.ContainsSingle(duplicates);
Assert.AreEqual(0, duplicates[0].FileSize);
}

/// <summary>
/// The documented rule is "keep the copy with the shortest filename".
/// </summary>
[TestMethod]
public void TheShortestFileNameIsKept()
{
// Arrange
using TempTree tree = new();
AbsoluteFilePath shortest = tree.Write("a.txt", "x");
List<AbsoluteFilePath> group =
[
tree.Write("aaaa.txt", "x"),
shortest,
tree.Write("aa.txt", "x"),
];

// Act
AbsoluteFilePath keeper = Deduplicator.SelectFileToKeep(group);

// Assert
Assert.AreEqual(shortest, keeper);
}

/// <summary>
/// Only the file name length is compared, not the length of the whole path -- a deeply nested
/// file with a short name still wins over a shallow one with a long name.
/// </summary>
[TestMethod]
public void OnlyTheFileNameLengthDecidesNotThePathLength()
{
// Arrange
using TempTree tree = new();
AbsoluteFilePath deepButShortName = tree.Write("one/two/three/four/a.txt", "x");
List<AbsoluteFilePath> group =
[
tree.Write("bbbbbbbb.txt", "x"),
deepButShortName,
];

// Act
AbsoluteFilePath keeper = Deduplicator.SelectFileToKeep(group);

// Assert
Assert.AreEqual(deepButShortName, keeper);
}

/// <summary>
/// Ties on name length must break deterministically, or which copy survives would depend on
/// enumeration order and differ between runs and platforms.
/// </summary>
[TestMethod]
public void TiesOnNameLengthBreakDeterministicallyByPath()
{
// Arrange
using TempTree tree = new();
AbsoluteFilePath a = tree.Write("aa.txt", "x");
AbsoluteFilePath b = tree.Write("bb.txt", "x");
AbsoluteFilePath c = tree.Write("cc.txt", "x");

// Act -- the same set in three different orders
AbsoluteFilePath first = Deduplicator.SelectFileToKeep([a, b, c]);
AbsoluteFilePath second = Deduplicator.SelectFileToKeep([c, b, a]);
AbsoluteFilePath third = Deduplicator.SelectFileToKeep([b, c, a]);

// Assert
Assert.AreEqual(first, second);
Assert.AreEqual(second, third);
Assert.AreEqual(a, first, "The ordinally-first path should win a tie on name length.");
}

/// <summary>
/// Deletion must remove every copy except the keeper, and must not touch anything else.
/// </summary>
[TestMethod]
public void DeleteDuplicatesRemovesEveryCopyExceptTheKeeper()
{
// Arrange
using TempTree tree = new();
AbsoluteFilePath unrelated = tree.Write("unrelated.txt", "different content");
Dictionary<AbsoluteFilePath, string> hashes = FileHasher.HashFiles(
[
unrelated,
tree.Write("a.txt", "shared"),
tree.Write("bb.txt", "shared"),
tree.Write("ccc.txt", "shared"),
]);
IReadOnlyList<DuplicateGroup> duplicates = Duplicates(hashes);
AbsoluteFilePath keeper = Deduplicator.SelectFileToKeep(duplicates[0].Files);

// Act
DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates);

// Assert
Assert.AreEqual(2, result.DeletedCount);
Assert.IsEmpty(result.Errors);
Assert.IsTrue(TempTree.Exists(keeper), "The keeper must survive.");
Assert.IsTrue(TempTree.Exists(unrelated), "A file outside any duplicate group must be untouched.");

foreach (AbsoluteFilePath file in duplicates[0].Files.Where(f => f != keeper))
{
Assert.IsFalse(TempTree.Exists(file), $"{file} should have been deleted.");
}
}

/// <summary>
/// The reclaimed byte count must reflect what was actually removed.
/// </summary>
[TestMethod]
public void ReclaimedBytesCountsOnlyTheDeletedCopies()
{
// Arrange
using TempTree tree = new();
string content = new('x', 100);
Dictionary<AbsoluteFilePath, string> hashes = FileHasher.HashFiles(
[
tree.Write("a.txt", content),
tree.Write("bb.txt", content),
tree.Write("ccc.txt", content),
]);
IReadOnlyList<DuplicateGroup> duplicates = Duplicates(hashes);

// Act
DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates);

// Assert -- three copies, two deleted, 100 bytes apiece
Assert.AreEqual(2, result.DeletedCount);
Assert.AreEqual(200, result.BytesReclaimed);
}

/// <summary>
/// Deleting nothing must report nothing, rather than throwing on an empty group list.
/// </summary>
[TestMethod]
public void DeletingAnEmptyGroupListIsANoOp()
{
// Act
DeduplicationResult result = Deduplicator.DeleteDuplicates([]);

// Assert
Assert.AreEqual(0, result.DeletedCount);
Assert.AreEqual(0, result.BytesReclaimed);
Assert.IsEmpty(result.Errors);
}
}
148 changes: 148 additions & 0 deletions FileDeduplicator.Test/DryRunEquivalenceTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
// Copyright (c) 2023-2026 ktsu-dev contributors

namespace ktsu.FileDeduplicator.Test;

using ktsu.Semantics.Paths;
using Microsoft.VisualStudio.TestTools.UnitTesting;

/// <summary>
/// Tests that the set of files DryRun says it would delete is exactly the set Deduplicate does
/// delete.
/// </summary>
/// <remarks>
/// DryRun is the safety net users rely on before letting this tool remove anything, so a
/// divergence between the two is the worst defect this codebase could have: the preview would be
/// a lie, and the thing it lied about is irreversible deletion.
///
/// The two verbs share <see cref="Deduplicator.GroupByHash"/>,
/// <see cref="Deduplicator.FindDuplicates"/> and <see cref="Deduplicator.SelectFileToKeep"/>, so
/// today they agree by construction. That is worth pinning rather than assuming: it holds only
/// while both keep calling the same three methods and while SelectFileToKeep stays deterministic,
/// and neither is enforced by anything but this test.
/// </remarks>
[TestClass]
public sealed class DryRunEquivalenceTests
{
/// <summary>
/// Computes the deletion set the way DryRun reports it -- every file in every duplicate group
/// except that group's keeper.
/// </summary>
private static HashSet<AbsoluteFilePath> PredictedDeletions(IReadOnlyList<DuplicateGroup> duplicates)
{
HashSet<AbsoluteFilePath> predicted = [];

foreach (DuplicateGroup group in duplicates)
{
AbsoluteFilePath keeper = Deduplicator.SelectFileToKeep(group.Files);
foreach (AbsoluteFilePath file in group.Files.Where(f => f != keeper))
{
_ = predicted.Add(file);
}
}

return predicted;
}

/// <summary>
/// Across a tree with several duplicate groups, unique files and nested directories, the
/// predicted deletion set must match the actual one exactly -- no file deleted that was not
/// predicted, and none predicted that survived.
/// </summary>
[TestMethod]
public void PredictedDeletionsMatchActualDeletionsExactly()
{
// Arrange -- three groups of duplicates plus two unique files, spread over nested folders
using TempTree tree = new();
List<AbsoluteFilePath> all =
[
tree.Write("group1/a.txt", "alpha"),
tree.Write("group1/aa.txt", "alpha"),
tree.Write("group1/nested/aaa.txt", "alpha"),
tree.Write("group2/b.txt", "beta"),
tree.Write("group2/bb.txt", "beta"),
tree.Write("group3/deep/c.txt", "gamma"),
tree.Write("group3/cc.txt", "gamma"),
tree.Write("unique-one.txt", "delta"),
tree.Write("nested/unique-two.txt", "epsilon"),
];

Dictionary<AbsoluteFilePath, string> hashes = FileHasher.HashFiles(all);
IReadOnlyList<DuplicateGroup> duplicates = Deduplicator.FindDuplicates(Deduplicator.GroupByHash(hashes));
HashSet<AbsoluteFilePath> predicted = PredictedDeletions(duplicates);

// Act
DeduplicationResult result = Deduplicator.DeleteDuplicates(duplicates);
HashSet<AbsoluteFilePath> actuallyDeleted = [.. all.Where(f => !TempTree.Exists(f))];

// Assert
Assert.HasCount(3, duplicates, "Three groups of duplicates were created.");
Assert.AreEqual(predicted.Count, result.DeletedCount);
Assert.IsTrue(
predicted.SetEquals(actuallyDeleted),
$"Predicted [{string.Join(", ", predicted)}] but deleted [{string.Join(", ", actuallyDeleted)}].");
}

/// <summary>
/// Every duplicate group must keep exactly one survivor. Deleting a whole group would destroy
/// the content entirely, which is the failure mode that matters most here.
/// </summary>
[TestMethod]
public void EveryDuplicateGroupRetainsExactlyOneSurvivor()
{
// Arrange
using TempTree tree = new();
List<AbsoluteFilePath> all =
[
tree.Write("a.txt", "alpha"),
tree.Write("aa.txt", "alpha"),
tree.Write("aaa.txt", "alpha"),
tree.Write("b.txt", "beta"),
tree.Write("bb.txt", "beta"),
];

Dictionary<AbsoluteFilePath, string> hashes = FileHasher.HashFiles(all);
IReadOnlyList<DuplicateGroup> duplicates = Deduplicator.FindDuplicates(Deduplicator.GroupByHash(hashes));

// Act
_ = Deduplicator.DeleteDuplicates(duplicates);

// Assert
foreach (DuplicateGroup group in duplicates)
{
int survivors = group.Files.Count(TempTree.Exists);
Assert.AreEqual(1, survivors, $"Group {group.Hash[..12]} should retain exactly one file.");
}
}

/// <summary>
/// Running the whole pipeline twice must be a no-op the second time: after deduplication there
/// are no duplicates left to find.
/// </summary>
[TestMethod]
public void DeduplicatingATreeTwiceDeletesNothingTheSecondTime()
{
// Arrange
using TempTree tree = new();
_ = tree.Write("a.txt", "alpha");
_ = tree.Write("aa.txt", "alpha");
_ = tree.Write("b.txt", "beta");

// Act -- first pass
IReadOnlyList<AbsoluteFilePath> firstScan = FileScanner.ScanForFiles(tree.Root);
IReadOnlyList<DuplicateGroup> firstDuplicates =
Deduplicator.FindDuplicates(Deduplicator.GroupByHash(FileHasher.HashFiles(firstScan)));
DeduplicationResult first = Deduplicator.DeleteDuplicates(firstDuplicates);

// Act -- second pass over the now-deduplicated tree
IReadOnlyList<AbsoluteFilePath> secondScan = FileScanner.ScanForFiles(tree.Root);
IReadOnlyList<DuplicateGroup> secondDuplicates =
Deduplicator.FindDuplicates(Deduplicator.GroupByHash(FileHasher.HashFiles(secondScan)));
DeduplicationResult second = Deduplicator.DeleteDuplicates(secondDuplicates);

// Assert
Assert.AreEqual(1, first.DeletedCount);
Assert.IsEmpty(secondDuplicates);
Assert.AreEqual(0, second.DeletedCount);
Assert.HasCount(2, secondScan, "Both distinct contents should survive.");
}
}
Loading