From 150611c8f43b41eb54c9f3abb44889ce255900ef Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 9 Apr 2026 11:34:42 -0700 Subject: [PATCH 01/12] Add coverage for ResultsComparer and CLI helpers Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../CommandLineOptions.cs | 4 +- .../BenchmarkDotNet.Extensions.Tests.csproj | 2 +- .../CommandLineOptionsTests.cs | 84 +++++++++ .../PartitionFilterTests.cs | 7 +- .../Reporting.Tests/Reporting.Tests.csproj | 2 +- src/tools/ResultsComparer.Tests/DataTests.cs | 160 ++++++++++++++++++ .../ResultsComparer.Tests/HelperTests.cs | 137 +++++++++++++++ .../ResultsComparer.Tests/ProgramTests.cs | 100 +++++++++++ .../ResultsComparer.Tests.csproj | 13 ++ .../ResultsComparerTestData.cs | 52 ++++++ src/tools/ResultsComparer.Tests/StatsTests.cs | 80 +++++++++ src/tools/ResultsComparer/Data.cs | 11 +- src/tools/ResultsComparer/Program.cs | 4 +- .../Properties/AssemblyInfo.cs | 3 + src/tools/ResultsComparer/ResultsComparer.sln | 29 ++++ .../ResultsComparer/TwoInputsComparer.cs | 6 +- .../Startup.Tests/StartupTests.cs | 23 ++- 17 files changed, 701 insertions(+), 16 deletions(-) create mode 100644 src/tools/ResultsComparer.Tests/DataTests.cs create mode 100644 src/tools/ResultsComparer.Tests/HelperTests.cs create mode 100644 src/tools/ResultsComparer.Tests/ProgramTests.cs create mode 100644 src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj create mode 100644 src/tools/ResultsComparer.Tests/ResultsComparerTestData.cs create mode 100644 src/tools/ResultsComparer.Tests/StatsTests.cs create mode 100644 src/tools/ResultsComparer/Properties/AssemblyInfo.cs diff --git a/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs b/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs index 3c8b343fc57..b76dd07dc29 100644 --- a/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs +++ b/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs @@ -35,7 +35,7 @@ public static List ParseAndRemoveStringsParameter(List argsList, int parameterIndex = argsList.IndexOf(parameter); parameterValue = new List(); - if (parameterIndex + 1 < argsList.Count) + if (parameterIndex != -1 && parameterIndex + 1 < argsList.Count) { while (parameterIndex + 1 < argsList.Count && !argsList[parameterIndex + 1].StartsWith("-")) { @@ -94,4 +94,4 @@ public static void ValidatePartitionParameters(int? count, int? index) } } } -} \ No newline at end of file +} diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj index 796d014d367..153b50473c5 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj @@ -1,7 +1,7 @@  - net11.0 + net10.0 false enable diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs index 4fbe6a7a10e..48e53620bcf 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs @@ -128,5 +128,89 @@ public void PartitionIndexValueGreaterThanCount() Assert.Throws(() => CommandLineOptions.ValidatePartitionParameters(count, index)); } + + [Fact] + public void ParseAndRemoveStringsParameterCollectsValuesUntilNextFlag() + { + List argsList = new List { + "--exclusion-filter", + "System.*", + "Microsoft.*", + "--partition-count", + "4" + }; + + var remaining = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out List filters); + + Assert.Equal(new[] { "System.*", "Microsoft.*" }, filters); + Assert.Equal(new[] { "--partition-count", "4" }, remaining); + } + + [Fact] + public void ParseAndRemoveBooleanParameterRemovesSwitchWhenPresent() + { + List argsList = new List { + "--wasm", + "--filter", + "*" + }; + + CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--wasm", out bool enabled); + + Assert.True(enabled); + Assert.Equal(new[] { "--filter", "*" }, argsList); + } + + [Fact] + public void ParseAndRemoveStringsParameterLeavesArgsUntouchedWhenSwitchIsMissing() + { + List argsList = new List { + "literal-value", + "--filter", + "*" + }; + + var remaining = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out List filters); + + Assert.Empty(filters); + Assert.Equal(new[] { "literal-value", "--filter", "*" }, remaining); + } + + [Fact] + public void ParseAndRemoveBooleanParameterReturnsFalseWhenSwitchIsMissing() + { + List argsList = new List { + "--filter", + "*" + }; + + CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--wasm", out bool enabled); + + Assert.False(enabled); + Assert.Equal(new[] { "--filter", "*" }, argsList); + } + + [Theory] + [InlineData("--partition-count")] + [InlineData("--partition-index")] + public void ParseAndRemoveIntParameterThrowsWhenValueIsMissing(string parameter) + { + List argsList = new List { parameter }; + + Assert.Throws(() => CommandLineOptions.ParseAndRemoveIntParameter(argsList, parameter, out int? _)); + } + + [Theory] + [InlineData("--partition-count", "abc")] + [InlineData("--partition-index", "3.14")] + public void ParseAndRemoveIntParameterThrowsWhenValueIsNotAnInteger(string parameter, string value) + { + List argsList = new List { + parameter, + value + }; + + Assert.Throws(() => CommandLineOptions.ParseAndRemoveIntParameter(argsList, parameter, out int? _)); + } } } diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs index 9319d50c607..f2a551c3bf7 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs @@ -63,8 +63,9 @@ public void NoBenchmarksAreOmitted_RealData() IConfig recommendedConfig = RecommendedConfig.Create( artifactsPath: new DirectoryInfo(Path.Combine(Path.GetDirectoryName(typeof(PartitionFilterTests).Assembly.Location)!, "BenchmarkDotNet.Artifacts")), mandatoryCategories: ImmutableHashSet.Create(Categories.Libraries, Categories.Runtime, Categories.ThirdParty)); - (bool isSuccess, IConfig parsedConfig, var _) = ConfigParser.Parse(new string[] { "--filter", "*" }, nullLogger, recommendedConfig); + (bool isSuccess, IConfig? parsedConfig, var _) = ConfigParser.Parse(new string[] { "--filter", "*" }, nullLogger, recommendedConfig); Assert.True(isSuccess); + Assert.NotNull(parsedConfig); Assembly microbenchmarksAssembly = typeof(Categories).Assembly; (bool allTypesValid, IReadOnlyList runnable) = Running.TypeFilter.GetTypesWithRunnableBenchmarks( @@ -73,7 +74,7 @@ public void NoBenchmarksAreOmitted_RealData() nullLogger); Assert.True(allTypesValid); - BenchmarkRunInfo[] allBenchmarks = GetAllBenchmarks(parsedConfig, runnable); + BenchmarkRunInfo[] allBenchmarks = GetAllBenchmarks(parsedConfig!, runnable); Dictionary idToPartitionIndex = new (); for (int i = 0; i < 10; i++) @@ -86,7 +87,7 @@ public void NoBenchmarksAreOmitted_RealData() { PartitionFilter filter = new(PartitionCount, partitionIndex); - foreach (BenchmarkCase benchmark in GetAllBenchmarks(parsedConfig, runnable).SelectMany(benchmark => benchmark.BenchmarksCases)) + foreach (BenchmarkCase benchmark in GetAllBenchmarks(parsedConfig!, runnable).SelectMany(benchmark => benchmark.BenchmarksCases)) { if (filter.Predicate(benchmark)) { diff --git a/src/tools/Reporting/Reporting.Tests/Reporting.Tests.csproj b/src/tools/Reporting/Reporting.Tests/Reporting.Tests.csproj index 28e90cbca3e..430d2b73476 100644 --- a/src/tools/Reporting/Reporting.Tests/Reporting.Tests.csproj +++ b/src/tools/Reporting/Reporting.Tests/Reporting.Tests.csproj @@ -1,7 +1,7 @@  - net11.0 + net10.0 false diff --git a/src/tools/ResultsComparer.Tests/DataTests.cs b/src/tools/ResultsComparer.Tests/DataTests.cs new file mode 100644 index 00000000000..6dea68b2a61 --- /dev/null +++ b/src/tools/ResultsComparer.Tests/DataTests.cs @@ -0,0 +1,160 @@ +using System.Formats.Tar; +using System.IO; +using System.IO.Compression; +using System.Text; +using Xunit; + +namespace ResultsComparer.Tests; + +public class DataTests +{ + [Fact] + public void DecompressExtractsJsonFilesFromNestedZipArchives() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var outerZipPath = Path.Combine(tempDir.FullName, "results.zip"); + var outputDirectory = new DirectoryInfo(Path.Combine(tempDir.FullName, "output")); + outputDirectory.Create(); + + var innerZipBytes = CreateInnerZip(("net10.0/SampleBenchmark.full.json", ResultsComparerTestData.CreateBdnJson())); + + using (var fileStream = File.Create(outerZipPath)) + using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create)) + { + var entry = archive.CreateEntry("Performance-Runs/net10.0/testuser/results.zip"); + using var entryStream = entry.Open(); + entryStream.Write(innerZipBytes, 0, innerZipBytes.Length); + } + + global::ResultsComparer.Data.Decompress(new FileInfo(outerZipPath), outputDirectory); + + var extractedFiles = Directory.GetFiles(outputDirectory.FullName, "*.full.json", SearchOption.AllDirectories); + var extractedFile = Assert.Single(extractedFiles); + var directoryName = Path.GetFileName(Path.GetDirectoryName(extractedFile)); + + Assert.Contains("testuser", directoryName, System.StringComparison.OrdinalIgnoreCase); + Assert.Contains("net10.0", directoryName, System.StringComparison.OrdinalIgnoreCase); + Assert.Contains("SampleBenchmark", File.ReadAllText(extractedFile)); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void DecompressExtractsJsonFilesFromTarGzArchives() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var outerZipPath = Path.Combine(tempDir.FullName, "results.zip"); + var outputDirectory = new DirectoryInfo(Path.Combine(tempDir.FullName, "output")); + outputDirectory.Create(); + + var tarGzBytes = CreateTarGzArchive( + ("payload/SampleBenchmark.full.json", ResultsComparerTestData.CreateBdnJson()), + ("payload/README.md", "ignored")); + + using (var fileStream = File.Create(outerZipPath)) + using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create)) + { + var entry = archive.CreateEntry("Performance-Runs/nativeaot10.0/testuser/arm64_win10-nativeaot10.0.tar.gz"); + using var entryStream = entry.Open(); + entryStream.Write(tarGzBytes, 0, tarGzBytes.Length); + } + + global::ResultsComparer.Data.Decompress(new FileInfo(outerZipPath), outputDirectory); + + var extractedFiles = Directory.GetFiles(outputDirectory.FullName, "*.full.json", SearchOption.AllDirectories); + var extractedFile = Assert.Single(extractedFiles); + var directoryName = Path.GetFileName(Path.GetDirectoryName(extractedFile)); + + Assert.Contains("nativeaot10.0", directoryName, System.StringComparison.OrdinalIgnoreCase); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void DecompressPrefersNewestBenchmarkDotNetVersionWhenDuplicatesExist() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var outerZipPath = Path.Combine(tempDir.FullName, "results.zip"); + var outputDirectory = new DirectoryInfo(Path.Combine(tempDir.FullName, "output")); + outputDirectory.Create(); + + var innerZipBytes = CreateInnerZip( + ("net10.0/SampleBenchmark-a.full.json", ResultsComparerTestData.CreateBdnJson(benchmarkDotNetVersion: "0.13.9")), + ("net10.0/SampleBenchmark-b.full.json", ResultsComparerTestData.CreateBdnJson(benchmarkDotNetVersion: "0.13.10"))); + + using (var fileStream = File.Create(outerZipPath)) + using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create)) + { + var entry = archive.CreateEntry("Performance-Runs/net10.0/testuser/results.zip"); + using var entryStream = entry.Open(); + entryStream.Write(innerZipBytes, 0, innerZipBytes.Length); + } + + global::ResultsComparer.Data.Decompress(new FileInfo(outerZipPath), outputDirectory); + + var extractedFile = Assert.Single(Directory.GetFiles(outputDirectory.FullName, "*.full.json", SearchOption.AllDirectories)); + var json = File.ReadAllText(extractedFile); + + Assert.Contains("\"BenchmarkDotNetVersion\": \"0.13.10\"", json); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + private static byte[] CreateInnerZip(params (string EntryName, string Content)[] entries) + { + using var stream = new MemoryStream(); + using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var (entryName, content) in entries) + { + var entry = archive.CreateEntry(entryName); + using var writer = new StreamWriter(entry.Open()); + writer.Write(content); + } + } + + return stream.ToArray(); + } + + private static byte[] CreateTarGzArchive(params (string EntryName, string Content)[] entries) + { + using var tarStream = new MemoryStream(); + using (var tarWriter = new TarWriter(tarStream, leaveOpen: true)) + { + foreach (var (entryName, content) in entries) + { + var tarEntry = new UstarTarEntry(TarEntryType.RegularFile, entryName) + { + DataStream = new MemoryStream(Encoding.UTF8.GetBytes(content)) + }; + + tarWriter.WriteEntry(tarEntry); + } + } + + tarStream.Position = 0; + + using var gzipStream = new MemoryStream(); + using (var compressor = new GZipStream(gzipStream, CompressionLevel.SmallestSize, leaveOpen: true)) + { + tarStream.CopyTo(compressor); + } + + return gzipStream.ToArray(); + } +} diff --git a/src/tools/ResultsComparer.Tests/HelperTests.cs b/src/tools/ResultsComparer.Tests/HelperTests.cs new file mode 100644 index 00000000000..b59b713fa29 --- /dev/null +++ b/src/tools/ResultsComparer.Tests/HelperTests.cs @@ -0,0 +1,137 @@ +using System.IO; +using System.Text; +using DataTransferContracts; +using Xunit; + +namespace ResultsComparer.Tests; + +public class HelperTests +{ + [Fact] + public void GetFilesToParseReturnsAllFullJsonFilesFromDirectory() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var nestedDirectory = Directory.CreateDirectory(Path.Combine(tempDir.FullName, "nested")); + var expectedA = Path.Combine(tempDir.FullName, "first.full.json"); + var expectedB = Path.Combine(nestedDirectory.FullName, "second.full.json"); + + File.WriteAllText(expectedA, "{}"); + File.WriteAllText(expectedB, "{}"); + File.WriteAllText(Path.Combine(tempDir.FullName, "ignored.json"), "{}"); + + var result = global::ResultsComparer.Helper.GetFilesToParse(tempDir.FullName); + + Assert.Equal(2, result.Length); + Assert.Contains(expectedA, result); + Assert.Contains(expectedB, result); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void GetFilesToParseThrowsForMissingFullJsonPath() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var missingPath = Path.Combine(tempDir.FullName, "missing.full.json"); + + Assert.Throws(() => global::ResultsComparer.Helper.GetFilesToParse(missingPath)); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void ReadFromStreamDeserializesBenchmarkResults() + { + var json = ResultsComparerTestData.CreateBdnJson(originalValues: [1.0, 1.1, 1.2], median: 1.1); + + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + + BdnResult result = global::ResultsComparer.Helper.ReadFromStream(stream); + + Assert.Equal("SampleBenchmark-20240504-182513", result.Title); + Assert.Equal("X64", result.HostEnvironmentInfo.Architecture); + Assert.Single(result.Benchmarks); + Assert.Equal("Demo.Namespace.SampleBenchmark", result.Benchmarks[0].FullName); + } + + [Fact] + public void ReadFromFileDeserializesBenchmarkResults() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var filePath = Path.Combine(tempDir.FullName, "sample.full.json"); + File.WriteAllText(filePath, ResultsComparerTestData.CreateBdnJson()); + + BdnResult result = global::ResultsComparer.Helper.ReadFromFile(filePath); + + Assert.Equal("SampleBenchmark-20240504-182513", result.Title); + Assert.Single(result.Benchmarks); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void GetFilesToParseReturnsExplicitFilePath() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var filePath = Path.Combine(tempDir.FullName, "custom-name.json"); + File.WriteAllText(filePath, ResultsComparerTestData.CreateBdnJson()); + + var result = global::ResultsComparer.Helper.GetFilesToParse(filePath); + + Assert.Equal([filePath], result); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void GetModalInfoReturnsNullForSmallSampleSets() + { + var benchmark = new Benchmark + { + Statistics = new Statistics + { + N = 3, + OriginalValues = [1.0, 1.1, 1.2] + } + }; + + Assert.Null(global::ResultsComparer.Helper.GetModalInfo(benchmark)); + } + + [Fact] + public void GetModalInfoDetectsMultiClusterData() + { + var benchmark = new Benchmark + { + Statistics = new Statistics + { + N = 16, + OriginalValues = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + } + }; + + var modality = global::ResultsComparer.Helper.GetModalInfo(benchmark); + + Assert.Equal("bimodal", modality); + } +} diff --git a/src/tools/ResultsComparer.Tests/ProgramTests.cs b/src/tools/ResultsComparer.Tests/ProgramTests.cs new file mode 100644 index 00000000000..8622871b722 --- /dev/null +++ b/src/tools/ResultsComparer.Tests/ProgramTests.cs @@ -0,0 +1,100 @@ +using System; +using System.IO; +using Xunit; + +namespace ResultsComparer.Tests; + +public class ProgramTests +{ + [Fact] + public void MainReportsInvalidThreshold() + { + var output = InvokeProgram(["--base", "base.json", "--diff", "diff.json", "--threshold", "not-a-threshold"]); + + Assert.Contains("Invalid Threshold", output); + } + + [Fact] + public void MainReportsMissingMatrixInputDirectory() + { + var missingDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); + + var output = InvokeProgram(["matrix", "--input", missingDirectory, "--base", "base", "--diff", "diff", "--threshold", "5%"]); + + Assert.Contains("does NOT exist", output); + } + + [Fact] + public void MainPrintsNoDifferencesForEquivalentInputs() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var baseFile = Path.Combine(tempDir.FullName, "base.full.json"); + var diffFile = Path.Combine(tempDir.FullName, "diff.full.json"); + var json = ResultsComparerTestData.CreateBdnJson(); + + File.WriteAllText(baseFile, json); + File.WriteAllText(diffFile, json); + + var output = InvokeProgram(["--base", baseFile, "--diff", diffFile, "--threshold", "5%"]); + + Assert.Contains("No differences found", output); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void MatrixCommandPrintsLegendAndBenchmarkTable() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var inputDirectory = Directory.CreateDirectory(Path.Combine(tempDir.FullName, "input")); + var baseDirectory = Directory.CreateDirectory(Path.Combine(inputDirectory.FullName, "run-base")); + var diffDirectory = Directory.CreateDirectory(Path.Combine(inputDirectory.FullName, "run-diff")); + + File.WriteAllText( + Path.Combine(baseDirectory.FullName, "SampleBenchmark.full.json"), + ResultsComparerTestData.CreateBdnJson( + originalValues: [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111], + median: 106, + allocatedBytes: 10)); + File.WriteAllText( + Path.Combine(diffDirectory.FullName, "SampleBenchmark.full.json"), + ResultsComparerTestData.CreateBdnJson( + originalValues: [200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211], + median: 206, + allocatedBytes: 30)); + + var output = InvokeProgram(["matrix", "--input", inputDirectory.FullName, "--base", "base", "--diff", "diff", "--threshold", "5%", "--ratio-only"]); + + Assert.Contains("# Legend", output); + Assert.Contains("Demo.Namespace.SampleBenchmark", output); + Assert.Contains("Slower", output); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + private static string InvokeProgram(string[] args) + { + using var writer = new StringWriter(); + var originalOut = Console.Out; + Console.SetOut(writer); + try + { + Program.Main(args); + return writer.ToString(); + } + finally + { + Console.SetOut(originalOut); + } + } +} diff --git a/src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj b/src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj new file mode 100644 index 00000000000..9a9c612083d --- /dev/null +++ b/src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + false + true + + + + + + diff --git a/src/tools/ResultsComparer.Tests/ResultsComparerTestData.cs b/src/tools/ResultsComparer.Tests/ResultsComparerTestData.cs new file mode 100644 index 00000000000..323f66a9455 --- /dev/null +++ b/src/tools/ResultsComparer.Tests/ResultsComparerTestData.cs @@ -0,0 +1,52 @@ +using System.Globalization; +using System.Linq; + +namespace ResultsComparer.Tests; + +internal static class ResultsComparerTestData +{ + internal static string CreateBdnJson( + string title = "SampleBenchmark-20240504-182513", + string fullName = "Demo.Namespace.SampleBenchmark", + string? benchmarkNamespace = "Demo.Namespace", + double[]? originalValues = null, + double? median = null, + long? allocatedBytes = 42, + string benchmarkDotNetVersion = "0.13.10", + string osVersion = "Windows 11 (10.0.26100)", + string processorName = "Intel Core i7-8700", + string architecture = "X64") + { + originalValues ??= [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111]; + median ??= originalValues.OrderBy(v => v).ElementAt(originalValues.Length / 2); + string values = string.Join(", ", originalValues.Select(v => v.ToString(CultureInfo.InvariantCulture))); + string namespaceField = benchmarkNamespace is null ? "null" : $"\"{benchmarkNamespace}\""; + string memoryField = allocatedBytes is null + ? "\"Memory\": {}" + : $"\"Memory\": {{ \"BytesAllocatedPerOperation\": {allocatedBytes.Value.ToString(CultureInfo.InvariantCulture)} }}"; + + return $$""" + { + "Title": "{{title}}", + "HostEnvironmentInfo": { + "BenchmarkDotNetVersion": "{{benchmarkDotNetVersion}}", + "OsVersion": "{{osVersion}}", + "ProcessorName": "{{processorName}}", + "Architecture": "{{architecture}}" + }, + "Benchmarks": [ + { + "Namespace": {{namespaceField}}, + "FullName": "{{fullName}}", + "Statistics": { + "OriginalValues": [{{values}}], + "N": {{originalValues.Length}}, + "Median": {{median.Value.ToString(CultureInfo.InvariantCulture)}} + }, + {{memoryField}} + } + ] + } + """; + } +} diff --git a/src/tools/ResultsComparer.Tests/StatsTests.cs b/src/tools/ResultsComparer.Tests/StatsTests.cs new file mode 100644 index 00000000000..d360817dbcc --- /dev/null +++ b/src/tools/ResultsComparer.Tests/StatsTests.cs @@ -0,0 +1,80 @@ +using System; +using System.IO; +using DataTransferContracts; +using Perfolizer.Mathematics.SignificanceTesting; +using Xunit; + +namespace ResultsComparer.Tests; + +public class StatsTests +{ + [Fact] + public void GetSimplifiedOSNameRemovesParentheticalSuffix() + { + Assert.Equal("Windows 11 ", global::ResultsComparer.Stats.GetSimplifiedOSName("Windows 11 (10.0.26100)")); + } + + [Fact] + public void PrintAggregatesTotalsAndEmitsSectionsOnce() + { + var stats = new global::ResultsComparer.Stats(); + var environment = new HostEnvironmentInfo + { + Architecture = "X64", + OsVersion = "Windows 11 (10.0.26100)", + ProcessorName = "Intel Core i7-8700" + }; + var benchmark = new Benchmark + { + Namespace = "Demo.Namespace", + Statistics = new Statistics + { + OriginalValues = new[] { 1.0, 1.1, 1.2 }, + N = 3, + Median = 1.1 + }, + Memory = new Memory() + }; + + stats.Record(EquivalenceTestConclusion.Same, environment, benchmark); + stats.Record(EquivalenceTestConclusion.Faster, environment, benchmark); + stats.Record(EquivalenceTestConclusion.Slower, environment, benchmark); + stats.Record(EquivalenceTestConclusion.Unknown, environment, benchmark); + stats.Record(global::ResultsComparer.Stats.Noise, environment, benchmark); + + using var writer = new StringWriter(); + var originalOut = Console.Out; + Console.SetOut(writer); + try + { + stats.Print(); + var firstOutput = writer.ToString(); + + Assert.Contains("## Statistics", firstOutput); + Assert.Contains("Total: 5", firstOutput); + Assert.Contains("## Statistics per Architecture", firstOutput); + Assert.Contains("## Statistics per Operating System", firstOutput); + Assert.Contains("## Statistics per Namespace", firstOutput); + Assert.Contains("Demo.Namespace", firstOutput); + + writer.GetStringBuilder().Clear(); + stats.Print(); + + Assert.Equal(string.Empty, writer.ToString()); + } + finally + { + Console.SetOut(originalOut); + } + } + + [Fact] + public void RecordThrowsForUnsupportedConclusion() + { + var stats = new global::ResultsComparer.Stats(); + var environment = new HostEnvironmentInfo { Architecture = "X64", OsVersion = "Windows 11 (10.0.26100)" }; + var benchmark = new Benchmark { Statistics = new Statistics { OriginalValues = [1.0], N = 1, Median = 1.0 }, Memory = new Memory() }; + + Assert.Throws(() => stats.Record((EquivalenceTestConclusion)999, environment, benchmark)); + } +} diff --git a/src/tools/ResultsComparer/Data.cs b/src/tools/ResultsComparer/Data.cs index 3a61ad01a03..b10d9c4b29a 100644 --- a/src/tools/ResultsComparer/Data.cs +++ b/src/tools/ResultsComparer/Data.cs @@ -130,6 +130,9 @@ static Version GetVersion(BdnResult bdnResult) private static string GetMoniker(string key) { + if (string.IsNullOrEmpty(key)) + return null; + if (key.Contains("net6")) // some files are net6.0, some are missing the dot (net60) return "net6.0"; if (key.Contains("nativeaot6")) @@ -158,13 +161,13 @@ private static string GetMoniker(string key) return "nativeaot9.0-preview" + key[key.IndexOf("nativeaot9.0-preview") + "nativeaot9.0-preview".Length]; if (key.Contains("net9.0")) return "net9.0"; - if (key.StartsWith("net10.0")) + if (key.Contains("net10.0")) return "net10.0"; - if (key.StartsWith("nativeaot10.0")) + if (key.Contains("nativeaot10.0")) return key; - if (key.StartsWith("net11.0")) + if (key.Contains("net11.0")) return "net11.0"; - if (key.StartsWith("nativeaot11.0")) + if (key.Contains("nativeaot11.0")) return key; return null; diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs index 8bb8d1294cf..92620cb3470 100644 --- a/src/tools/ResultsComparer/Program.cs +++ b/src/tools/ResultsComparer/Program.cs @@ -172,7 +172,9 @@ private static bool TryGetPaths(DirectoryInfo input, string basePattern, string } private static Regex[] GetFilters(string[] filters) - => filters.Select(pattern => new Regex(WildcardToRegex(pattern), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)).ToArray(); + => (filters ?? Array.Empty()) + .Select(pattern => new Regex(WildcardToRegex(pattern), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) + .ToArray(); // https://stackoverflow.com/a/6907849/5852046 not perfect but should work for all we need private static string WildcardToRegex(string pattern) => $"^{Regex.Escape(pattern).Replace(@"\*", ".*").Replace(@"\?", ".")}$"; diff --git a/src/tools/ResultsComparer/Properties/AssemblyInfo.cs b/src/tools/ResultsComparer/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..11fac6ab430 --- /dev/null +++ b/src/tools/ResultsComparer/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("ResultsComparer.Tests")] diff --git a/src/tools/ResultsComparer/ResultsComparer.sln b/src/tools/ResultsComparer/ResultsComparer.sln index 951a4d0fb5d..5985914bae2 100644 --- a/src/tools/ResultsComparer/ResultsComparer.sln +++ b/src/tools/ResultsComparer/ResultsComparer.sln @@ -2,15 +2,44 @@ Microsoft Visual Studio Solution File, Format Version 12.00 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ResultsComparer", "ResultsComparer.csproj", "{00859394-44F8-466B-8624-41578CA94009}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ResultsComparer.Tests", "..\ResultsComparer.Tests\ResultsComparer.Tests.csproj", "{F1CA7160-F99B-484F-975E-04C4638BCFAF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {00859394-44F8-466B-8624-41578CA94009}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {00859394-44F8-466B-8624-41578CA94009}.Debug|Any CPU.Build.0 = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Debug|x64.ActiveCfg = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Debug|x64.Build.0 = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Debug|x86.ActiveCfg = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Debug|x86.Build.0 = Debug|Any CPU {00859394-44F8-466B-8624-41578CA94009}.Release|Any CPU.ActiveCfg = Release|Any CPU {00859394-44F8-466B-8624-41578CA94009}.Release|Any CPU.Build.0 = Release|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|x64.ActiveCfg = Release|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|x64.Build.0 = Release|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|x86.ActiveCfg = Release|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|x86.Build.0 = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|x64.ActiveCfg = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|x64.Build.0 = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|x86.ActiveCfg = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|x86.Build.0 = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|Any CPU.Build.0 = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|x64.ActiveCfg = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|x64.Build.0 = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|x86.ActiveCfg = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE EndGlobalSection EndGlobal diff --git a/src/tools/ResultsComparer/TwoInputsComparer.cs b/src/tools/ResultsComparer/TwoInputsComparer.cs index 89209aefb58..c6e81a24bd2 100644 --- a/src/tools/ResultsComparer/TwoInputsComparer.cs +++ b/src/tools/ResultsComparer/TwoInputsComparer.cs @@ -36,11 +36,13 @@ internal static void Compare(TwoInputsOptions args) var diffValues = diffResult.Statistics.OriginalValues.ToArray(); var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); - if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same) + if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same + || userTresholdResult.Conclusion == EquivalenceTestConclusion.Base) continue; var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.NoiseThreshold); - if (noiseResult.Conclusion == EquivalenceTestConclusion.Same) + if (noiseResult.Conclusion == EquivalenceTestConclusion.Same + || noiseResult.Conclusion == EquivalenceTestConclusion.Base) continue; yield return (id, baseResult, diffResult, userTresholdResult.Conclusion); diff --git a/src/tools/ScenarioMeasurement/Startup.Tests/StartupTests.cs b/src/tools/ScenarioMeasurement/Startup.Tests/StartupTests.cs index 83ceb5c79c0..e94ceaa3b0d 100644 --- a/src/tools/ScenarioMeasurement/Startup.Tests/StartupTests.cs +++ b/src/tools/ScenarioMeasurement/Startup.Tests/StartupTests.cs @@ -3,7 +3,9 @@ using System; using System.Collections.Generic; using System.IO; +using System.Security.Principal; using System.Threading; +using System.Runtime.Versioning; using Xunit; @@ -131,10 +133,27 @@ public sealed class WindowsOnly : FactAttribute { public WindowsOnly() { - if (Environment.OSVersion.Platform != PlatformID.Win32NT) + if (!OperatingSystem.IsWindows()) { Skip = "Skip on non-windows platform"; } + else if (!IsRunningAsAdministrator()) + { + Skip = "Requires administrator privileges to start ETW sessions"; + } + } + + [SupportedOSPlatform("windows")] + private static bool IsRunningAsAdministrator() + { + if (!OperatingSystem.IsWindows()) + { + return false; + } + + using WindowsIdentity identity = WindowsIdentity.GetCurrent(); + WindowsPrincipal principal = new(identity); + return principal.IsInRole(WindowsBuiltInRole.Administrator); } } @@ -142,7 +161,7 @@ public sealed class LinuxOnly : FactAttribute { public LinuxOnly() { - if(Environment.OSVersion.Platform != PlatformID.Unix) + if (!OperatingSystem.IsLinux()) { Skip = "Skip on non-linux platform"; } From fe3c6bf587f06598e30252800429204f00172a06 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 9 Apr 2026 14:54:22 -0700 Subject: [PATCH 02/12] Restore test projects to net11.0 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../BenchmarkDotNet.Extensions.Tests.csproj | 2 +- src/tools/Reporting/Reporting.Tests/Reporting.Tests.csproj | 2 +- src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj index 153b50473c5..796d014d367 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj @@ -1,7 +1,7 @@  - net10.0 + net11.0 false enable diff --git a/src/tools/Reporting/Reporting.Tests/Reporting.Tests.csproj b/src/tools/Reporting/Reporting.Tests/Reporting.Tests.csproj index 430d2b73476..28e90cbca3e 100644 --- a/src/tools/Reporting/Reporting.Tests/Reporting.Tests.csproj +++ b/src/tools/Reporting/Reporting.Tests/Reporting.Tests.csproj @@ -1,7 +1,7 @@  - net10.0 + net11.0 false diff --git a/src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj b/src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj index 9a9c612083d..5611c64bee9 100644 --- a/src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj +++ b/src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj @@ -1,6 +1,6 @@ - net10.0 + net11.0 enable enable false From 6e12ccfac73432e7e7ebada8d5cb8f10ab45d70b Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 10 Apr 2026 10:49:50 -0700 Subject: [PATCH 03/12] Address PR review feedback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../ResultsComparer.Tests/ConsoleOutputCollection.cs | 8 ++++++++ src/tools/ResultsComparer.Tests/ProgramTests.cs | 1 + src/tools/ResultsComparer.Tests/StatsTests.cs | 3 ++- src/tools/ResultsComparer/Data.cs | 4 ++-- src/tools/ResultsComparer/MultipleInputsComparer.cs | 6 +++--- src/tools/ResultsComparer/Stats.cs | 2 +- src/tools/ResultsComparer/TwoInputsComparer.cs | 8 ++++---- 7 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 src/tools/ResultsComparer.Tests/ConsoleOutputCollection.cs diff --git a/src/tools/ResultsComparer.Tests/ConsoleOutputCollection.cs b/src/tools/ResultsComparer.Tests/ConsoleOutputCollection.cs new file mode 100644 index 00000000000..8f3c5aa69b6 --- /dev/null +++ b/src/tools/ResultsComparer.Tests/ConsoleOutputCollection.cs @@ -0,0 +1,8 @@ +using Xunit; + +namespace ResultsComparer.Tests; + +[CollectionDefinition("Console output", DisableParallelization = true)] +public sealed class ConsoleOutputCollection +{ +} diff --git a/src/tools/ResultsComparer.Tests/ProgramTests.cs b/src/tools/ResultsComparer.Tests/ProgramTests.cs index 8622871b722..d5bc708e001 100644 --- a/src/tools/ResultsComparer.Tests/ProgramTests.cs +++ b/src/tools/ResultsComparer.Tests/ProgramTests.cs @@ -4,6 +4,7 @@ namespace ResultsComparer.Tests; +[Collection("Console output")] public class ProgramTests { [Fact] diff --git a/src/tools/ResultsComparer.Tests/StatsTests.cs b/src/tools/ResultsComparer.Tests/StatsTests.cs index d360817dbcc..a8b8cfeb38e 100644 --- a/src/tools/ResultsComparer.Tests/StatsTests.cs +++ b/src/tools/ResultsComparer.Tests/StatsTests.cs @@ -6,12 +6,13 @@ namespace ResultsComparer.Tests; +[Collection("Console output")] public class StatsTests { [Fact] public void GetSimplifiedOSNameRemovesParentheticalSuffix() { - Assert.Equal("Windows 11 ", global::ResultsComparer.Stats.GetSimplifiedOSName("Windows 11 (10.0.26100)")); + Assert.Equal("Windows 11", global::ResultsComparer.Stats.GetSimplifiedOSName("Windows 11 (10.0.26100)")); } [Fact] diff --git a/src/tools/ResultsComparer/Data.cs b/src/tools/ResultsComparer/Data.cs index b10d9c4b29a..a663f1e8f63 100644 --- a/src/tools/ResultsComparer/Data.cs +++ b/src/tools/ResultsComparer/Data.cs @@ -164,11 +164,11 @@ private static string GetMoniker(string key) if (key.Contains("net10.0")) return "net10.0"; if (key.Contains("nativeaot10.0")) - return key; + return "nativeaot10.0"; if (key.Contains("net11.0")) return "net11.0"; if (key.Contains("nativeaot11.0")) - return key; + return "nativeaot11.0"; return null; } diff --git a/src/tools/ResultsComparer/MultipleInputsComparer.cs b/src/tools/ResultsComparer/MultipleInputsComparer.cs index 986a879e2b8..7347d22d216 100644 --- a/src/tools/ResultsComparer/MultipleInputsComparer.cs +++ b/src/tools/ResultsComparer/MultipleInputsComparer.cs @@ -141,13 +141,13 @@ static long GetMetricValue(Benchmark result) var baseValues = info.baseResult.Statistics.OriginalValues; var diffValues = info.diffResult.Statistics.OriginalValues; - var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); + var userThresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.NoiseThreshold); // filter noise (0.20 ns vs 0.25ns is 25% difference) - var conclusion = userTresholdResult.Conclusion != EquivalenceTestConclusion.Same && noiseResult.Conclusion == EquivalenceTestConclusion.Same + var conclusion = userThresholdResult.Conclusion != EquivalenceTestConclusion.Same && noiseResult.Conclusion == EquivalenceTestConclusion.Same ? Stats.Noise - : userTresholdResult.Conclusion == EquivalenceTestConclusion.Base ? EquivalenceTestConclusion.Same : userTresholdResult.Conclusion; + : userThresholdResult.Conclusion == EquivalenceTestConclusion.Base ? EquivalenceTestConclusion.Same : userThresholdResult.Conclusion; stats.Record(conclusion, info.baseEnv, info.baseResult); diff --git a/src/tools/ResultsComparer/Stats.cs b/src/tools/ResultsComparer/Stats.cs index e230c0478a4..4e0a472c33f 100644 --- a/src/tools/ResultsComparer/Stats.cs +++ b/src/tools/ResultsComparer/Stats.cs @@ -77,7 +77,7 @@ static void Print(Dictionary dictionary, string name) } } - internal static string GetSimplifiedOSName(string text) => text.Split('(')[0]; + internal static string GetSimplifiedOSName(string text) => text.Split('(')[0].TrimEnd(); private class PerConclusion { diff --git a/src/tools/ResultsComparer/TwoInputsComparer.cs b/src/tools/ResultsComparer/TwoInputsComparer.cs index c6e81a24bd2..a929e937d27 100644 --- a/src/tools/ResultsComparer/TwoInputsComparer.cs +++ b/src/tools/ResultsComparer/TwoInputsComparer.cs @@ -35,9 +35,9 @@ internal static void Compare(TwoInputsOptions args) var baseValues = baseResult.Statistics.OriginalValues.ToArray(); var diffValues = diffResult.Statistics.OriginalValues.ToArray(); - var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); - if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same - || userTresholdResult.Conclusion == EquivalenceTestConclusion.Base) + var userThresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); + if (userThresholdResult.Conclusion == EquivalenceTestConclusion.Same + || userThresholdResult.Conclusion == EquivalenceTestConclusion.Base) continue; var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.NoiseThreshold); @@ -45,7 +45,7 @@ internal static void Compare(TwoInputsOptions args) || noiseResult.Conclusion == EquivalenceTestConclusion.Base) continue; - yield return (id, baseResult, diffResult, userTresholdResult.Conclusion); + yield return (id, baseResult, diffResult, userThresholdResult.Conclusion); } } From e942a404d3bb3c4712cb1c289dc1423b096b46c3 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 13 Apr 2026 13:53:32 -0700 Subject: [PATCH 04/12] Fix Base classification in matrix comparisons Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../ResultsComparer.Tests/ProgramTests.cs | 25 +++++++++++++++++++ .../ResultsComparer/MultipleInputsComparer.cs | 6 +++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/tools/ResultsComparer.Tests/ProgramTests.cs b/src/tools/ResultsComparer.Tests/ProgramTests.cs index d5bc708e001..c292ef5df61 100644 --- a/src/tools/ResultsComparer.Tests/ProgramTests.cs +++ b/src/tools/ResultsComparer.Tests/ProgramTests.cs @@ -83,6 +83,31 @@ public void MatrixCommandPrintsLegendAndBenchmarkTable() } } + [Fact] + public void MatrixCommandTreatsEquivalentInputsAsSameNotNoise() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var inputDirectory = Directory.CreateDirectory(Path.Combine(tempDir.FullName, "input")); + var baseDirectory = Directory.CreateDirectory(Path.Combine(inputDirectory.FullName, "run-base")); + var diffDirectory = Directory.CreateDirectory(Path.Combine(inputDirectory.FullName, "run-diff")); + var identicalJson = ResultsComparerTestData.CreateBdnJson(); + + File.WriteAllText(Path.Combine(baseDirectory.FullName, "SampleBenchmark.full.json"), identicalJson); + File.WriteAllText(Path.Combine(diffDirectory.FullName, "SampleBenchmark.full.json"), identicalJson); + + var output = InvokeProgram(["matrix", "--input", inputDirectory.FullName, "--base", "base", "--diff", "diff", "--threshold", "5%", "--ratio-only"]); + + Assert.Contains("Same", output); + Assert.DoesNotContain("Noise", output); + } + finally + { + tempDir.Delete(recursive: true); + } + } + private static string InvokeProgram(string[] args) { using var writer = new StringWriter(); diff --git a/src/tools/ResultsComparer/MultipleInputsComparer.cs b/src/tools/ResultsComparer/MultipleInputsComparer.cs index 7347d22d216..ba3ee7656ed 100644 --- a/src/tools/ResultsComparer/MultipleInputsComparer.cs +++ b/src/tools/ResultsComparer/MultipleInputsComparer.cs @@ -143,11 +143,13 @@ static long GetMetricValue(Benchmark result) var userThresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.NoiseThreshold); + var userConclusion = userThresholdResult.Conclusion == EquivalenceTestConclusion.Base ? EquivalenceTestConclusion.Same : userThresholdResult.Conclusion; + var noiseConclusion = noiseResult.Conclusion == EquivalenceTestConclusion.Base ? EquivalenceTestConclusion.Same : noiseResult.Conclusion; // filter noise (0.20 ns vs 0.25ns is 25% difference) - var conclusion = userThresholdResult.Conclusion != EquivalenceTestConclusion.Same && noiseResult.Conclusion == EquivalenceTestConclusion.Same + var conclusion = userConclusion != EquivalenceTestConclusion.Same && noiseConclusion == EquivalenceTestConclusion.Same ? Stats.Noise - : userThresholdResult.Conclusion == EquivalenceTestConclusion.Base ? EquivalenceTestConclusion.Same : userThresholdResult.Conclusion; + : userConclusion; stats.Record(conclusion, info.baseEnv, info.baseResult); From a35c443e91ab70873978d62bc9ee66368178795b Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 15 Apr 2026 15:31:38 -0700 Subject: [PATCH 05/12] Run tooling tests in CI Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/sdk-perf-jobs.yml | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 4c3a62d1a72..acdf8f0c14e 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -17,6 +17,42 @@ jobs: # Public correctness jobs ###################################################### +- ${{ if or(parameters.runPublicJobs, parameters.runPrivateJobs, parameters.runScheduledPrivateJobs) }}: + - job: Tooling_Tests_Windows + displayName: Tooling Tests (Windows) + timeoutInMinutes: 30 + pool: + vmImage: windows-2022 + + steps: + - task: UseDotNet@2 + displayName: Install .NET 8.0 + inputs: + version: 8.0.x + + - task: UseDotNet@2 + displayName: Install .NET 10.0 + inputs: + version: 10.0.x + includePreviewVersions: true + + - task: UseDotNet@2 + displayName: Install .NET 11.0 + inputs: + version: 11.0.x + includePreviewVersions: true + + - pwsh: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + + dotnet test src\tools\ResultsComparer.Tests\ResultsComparer.Tests.csproj --configuration Release --framework net11.0 --nologo --verbosity minimal + dotnet test src\tools\Reporting\Reporting.Tests\Reporting.Tests.csproj --configuration Release --framework net11.0 --nologo --verbosity minimal + dotnet test src\tools\CertHelperTests\CertRotatorTests.csproj --configuration Release --framework net10.0 --nologo --verbosity minimal + dotnet test src\tests\harness\BenchmarkDotNet.Extensions.Tests\BenchmarkDotNet.Extensions.Tests.csproj --configuration Release --framework net11.0 -p:PERFLAB_TARGET_FRAMEWORKS=net11.0 --nologo --verbosity minimal + dotnet test src\tools\ScenarioMeasurement\Startup.Tests\Startup.Tests.csproj --configuration Release --nologo --verbosity minimal + displayName: Run tooling tests + - ${{ if parameters.runPublicJobs }}: # Scenario benchmarks From 83d303549cbd9e9e29353dd5184f92616a4a0038 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 15 Apr 2026 15:39:20 -0700 Subject: [PATCH 06/12] Skip tooling tests on main Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/sdk-perf-jobs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index acdf8f0c14e..84fe828d214 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -21,6 +21,7 @@ jobs: - job: Tooling_Tests_Windows displayName: Tooling Tests (Windows) timeoutInMinutes: 30 + condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/main')) pool: vmImage: windows-2022 From d76ef4193676a1ea6f08f91ef7347cc057a60a66 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 15 Apr 2026 15:40:03 -0700 Subject: [PATCH 07/12] Limit tooling tests to public non-main runs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/sdk-perf-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 84fe828d214..7d98eac467f 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -17,7 +17,7 @@ jobs: # Public correctness jobs ###################################################### -- ${{ if or(parameters.runPublicJobs, parameters.runPrivateJobs, parameters.runScheduledPrivateJobs) }}: +- ${{ if parameters.runPublicJobs }}: - job: Tooling_Tests_Windows displayName: Tooling Tests (Windows) timeoutInMinutes: 30 From 9cfdeb0962709265c8095dea58f4bfe2f707e90f Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 16 Apr 2026 11:21:37 -0700 Subject: [PATCH 08/12] Fix brittle matrix comparison test Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/tools/ResultsComparer.Tests/ProgramTests.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tools/ResultsComparer.Tests/ProgramTests.cs b/src/tools/ResultsComparer.Tests/ProgramTests.cs index c292ef5df61..850358a3dbc 100644 --- a/src/tools/ResultsComparer.Tests/ProgramTests.cs +++ b/src/tools/ResultsComparer.Tests/ProgramTests.cs @@ -98,9 +98,10 @@ public void MatrixCommandTreatsEquivalentInputsAsSameNotNoise() File.WriteAllText(Path.Combine(diffDirectory.FullName, "SampleBenchmark.full.json"), identicalJson); var output = InvokeProgram(["matrix", "--input", inputDirectory.FullName, "--base", "base", "--diff", "diff", "--threshold", "5%", "--ratio-only"]); + var lines = output.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); - Assert.Contains("Same", output); - Assert.DoesNotContain("Noise", output); + Assert.Contains(lines, line => line.TrimStart().StartsWith("| Same", StringComparison.Ordinal)); + Assert.DoesNotContain(lines, line => line.TrimStart().StartsWith("| Noise", StringComparison.Ordinal)); } finally { From 2a79e4cdaa949d70e8d0a0e42dd9b990722e6f69 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 16 Apr 2026 12:14:59 -0700 Subject: [PATCH 09/12] Restore culture after invoking ResultsComparer Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../ResultsComparer.Tests/ProgramTests.cs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/tools/ResultsComparer.Tests/ProgramTests.cs b/src/tools/ResultsComparer.Tests/ProgramTests.cs index 850358a3dbc..786f05209fc 100644 --- a/src/tools/ResultsComparer.Tests/ProgramTests.cs +++ b/src/tools/ResultsComparer.Tests/ProgramTests.cs @@ -1,4 +1,5 @@ using System; +using System.Globalization; using System.IO; using Xunit; @@ -109,10 +110,38 @@ public void MatrixCommandTreatsEquivalentInputsAsSameNotNoise() } } + [Fact] + public void InvokeProgramRestoresCurrentCulture() + { + var originalCulture = CultureInfo.CurrentCulture; + var originalUICulture = CultureInfo.CurrentUICulture; + + try + { + var expectedCulture = new CultureInfo("fr-FR"); + var expectedUICulture = new CultureInfo("de-DE"); + + CultureInfo.CurrentCulture = expectedCulture; + CultureInfo.CurrentUICulture = expectedUICulture; + + _ = InvokeProgram(["--base", "base.json", "--diff", "diff.json", "--threshold", "not-a-threshold"]); + + Assert.Equal(expectedCulture.Name, CultureInfo.CurrentCulture.Name); + Assert.Equal(expectedUICulture.Name, CultureInfo.CurrentUICulture.Name); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + CultureInfo.CurrentUICulture = originalUICulture; + } + } + private static string InvokeProgram(string[] args) { using var writer = new StringWriter(); var originalOut = Console.Out; + var originalCulture = CultureInfo.CurrentCulture; + var originalUICulture = CultureInfo.CurrentUICulture; Console.SetOut(writer); try { @@ -121,6 +150,8 @@ private static string InvokeProgram(string[] args) } finally { + CultureInfo.CurrentCulture = originalCulture; + CultureInfo.CurrentUICulture = originalUICulture; Console.SetOut(originalOut); } } From 4d66ea4389e583eb1e32e353eca8e3f2f3291656 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 20 Apr 2026 13:15:27 -0700 Subject: [PATCH 10/12] Bump Newtonsoft.Json for tooling tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/tools/Reporting/Directory.Packages.props | 2 +- src/tools/ResultsComparer/Directory.Packages.props | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/Reporting/Directory.Packages.props b/src/tools/Reporting/Directory.Packages.props index ab6543dc86b..0fde0cf995d 100644 --- a/src/tools/Reporting/Directory.Packages.props +++ b/src/tools/Reporting/Directory.Packages.props @@ -4,7 +4,7 @@ true - + diff --git a/src/tools/ResultsComparer/Directory.Packages.props b/src/tools/ResultsComparer/Directory.Packages.props index 28ef11f8a62..d910ce303ad 100644 --- a/src/tools/ResultsComparer/Directory.Packages.props +++ b/src/tools/ResultsComparer/Directory.Packages.props @@ -5,7 +5,7 @@ - + From 70a7531fa3f9a94cdd4307e43e802441fb3fd10b Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 20 Apr 2026 13:28:15 -0700 Subject: [PATCH 11/12] Use net10 MicroBenchmarks in harness tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/sdk-perf-jobs.yml | 2 +- .../BenchmarkDotNet.Extensions.Tests.csproj | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 7d98eac467f..c3c87fca015 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -50,7 +50,7 @@ jobs: dotnet test src\tools\ResultsComparer.Tests\ResultsComparer.Tests.csproj --configuration Release --framework net11.0 --nologo --verbosity minimal dotnet test src\tools\Reporting\Reporting.Tests\Reporting.Tests.csproj --configuration Release --framework net11.0 --nologo --verbosity minimal dotnet test src\tools\CertHelperTests\CertRotatorTests.csproj --configuration Release --framework net10.0 --nologo --verbosity minimal - dotnet test src\tests\harness\BenchmarkDotNet.Extensions.Tests\BenchmarkDotNet.Extensions.Tests.csproj --configuration Release --framework net11.0 -p:PERFLAB_TARGET_FRAMEWORKS=net11.0 --nologo --verbosity minimal + dotnet test src\tests\harness\BenchmarkDotNet.Extensions.Tests\BenchmarkDotNet.Extensions.Tests.csproj --configuration Release --framework net11.0 -p:PERFLAB_TARGET_FRAMEWORKS=net10.0 --nologo --verbosity minimal dotnet test src\tools\ScenarioMeasurement\Startup.Tests\Startup.Tests.csproj --configuration Release --nologo --verbosity minimal displayName: Run tooling tests diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj index 796d014d367..24a2526f6ec 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj @@ -7,7 +7,9 @@ - + From 92f74bc2721287118260fb566d1b5deedc23ba85 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 29 Apr 2026 10:55:22 -0700 Subject: [PATCH 12/12] Update src/tools/ResultsComparer.Tests/DataTests.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/tools/ResultsComparer.Tests/DataTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tools/ResultsComparer.Tests/DataTests.cs b/src/tools/ResultsComparer.Tests/DataTests.cs index 6dea68b2a61..a94592f139f 100644 --- a/src/tools/ResultsComparer.Tests/DataTests.cs +++ b/src/tools/ResultsComparer.Tests/DataTests.cs @@ -138,9 +138,10 @@ private static byte[] CreateTarGzArchive(params (string EntryName, string Conten { foreach (var (entryName, content) in entries) { + using var dataStream = new MemoryStream(Encoding.UTF8.GetBytes(content)); var tarEntry = new UstarTarEntry(TarEntryType.RegularFile, entryName) { - DataStream = new MemoryStream(Encoding.UTF8.GetBytes(content)) + DataStream = dataStream }; tarWriter.WriteEntry(tarEntry);