Add test coverage for ResultsComparer and CLI helper paths

eng/pipelines/sdk-perf-jobs.yml

@@ -17,6 +17,43 @@ jobs:
 # Public correctness jobs
 ######################################################
 
+- ${{ if parameters.runPublicJobs }}:
+  - job: Tooling_Tests_Windows
+    displayName: Tooling Tests (Windows)
+    timeoutInMinutes: 30
+    condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/main'))
+    pool:
+      vmImage: windows-2022
+
+    steps:
+      - task: UseDotNet@2
+        displayName: Install .NET 8.0
+        inputs:
+          version: 8.0.x
+
+      - task: UseDotNet@2
+        displayName: Install .NET 10.0
+        inputs:
+          version: 10.0.x
+          includePreviewVersions: true
+
+      - task: UseDotNet@2
+        displayName: Install .NET 11.0
+        inputs:
+          version: 11.0.x
+          includePreviewVersions: true
+
+      - pwsh: |
+          $ErrorActionPreference = 'Stop'
+          $PSNativeCommandUseErrorActionPreference = $true
+
+          dotnet test src\tools\ResultsComparer.Tests\ResultsComparer.Tests.csproj --configuration Release --framework net11.0 --nologo --verbosity minimal
+          dotnet test src\tools\Reporting\Reporting.Tests\Reporting.Tests.csproj --configuration Release --framework net11.0 --nologo --verbosity minimal
+          dotnet test src\tools\CertHelperTests\CertRotatorTests.csproj --configuration Release --framework net10.0 --nologo --verbosity minimal
+          dotnet test src\tests\harness\BenchmarkDotNet.Extensions.Tests\BenchmarkDotNet.Extensions.Tests.csproj --configuration Release --framework net11.0 -p:PERFLAB_TARGET_FRAMEWORKS=net10.0 --nologo --verbosity minimal
+          dotnet test src\tools\ScenarioMeasurement\Startup.Tests\Startup.Tests.csproj --configuration Release --nologo --verbosity minimal
+        displayName: Run tooling tests
+
 - ${{ if parameters.runPublicJobs }}:
 
   # Scenario benchmarks

src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs

@@ -35,7 +35,7 @@ public static List<string> ParseAndRemoveStringsParameter(List<string> argsList,
             int parameterIndex = argsList.IndexOf(parameter);
             parameterValue = new List<string>();
 
-            if (parameterIndex + 1 < argsList.Count)
+            if (parameterIndex != -1 && parameterIndex + 1 < argsList.Count)
             {
                 while (parameterIndex + 1 < argsList.Count && !argsList[parameterIndex + 1].StartsWith("-"))
                 {
@@ -94,4 +94,4 @@ public static void ValidatePartitionParameters(int? count, int? index)
             }
         }
     }
-}
+}

src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj

@@ -7,7 +7,9 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\..\benchmarks\micro\MicroBenchmarks.csproj" />
+    <ProjectReference Include="..\..\..\benchmarks\micro\MicroBenchmarks.csproj"
+                      SetTargetFramework="TargetFramework=net10.0"
+                      AdditionalProperties="PERFLAB_TARGET_FRAMEWORKS=net10.0" />
     <ProjectReference Include="..\..\..\harness\BenchmarkDotNet.Extensions\BenchmarkDotNet.Extensions.csproj" />
   </ItemGroup>
 </Project>

src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs

@@ -128,5 +128,89 @@ public void PartitionIndexValueGreaterThanCount()
 
             Assert.Throws<ArgumentException>(() => CommandLineOptions.ValidatePartitionParameters(count, index));
         }
+
+        [Fact]
+        public void ParseAndRemoveStringsParameterCollectsValuesUntilNextFlag()
+        {
+            List<string> argsList = new List<string> {
+                "--exclusion-filter",
+                "System.*",
+                "Microsoft.*",
+                "--partition-count",
+                "4"
+            };
+
+            var remaining = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out List<string> filters);
+
+            Assert.Equal(new[] { "System.*", "Microsoft.*" }, filters);
+            Assert.Equal(new[] { "--partition-count", "4" }, remaining);
+        }
+
+        [Fact]
+        public void ParseAndRemoveBooleanParameterRemovesSwitchWhenPresent()
+        {
+            List<string> argsList = new List<string> {
+                "--wasm",
+                "--filter",
+                "*"
+            };
+
+            CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--wasm", out bool enabled);
+
+            Assert.True(enabled);
+            Assert.Equal(new[] { "--filter", "*" }, argsList);
+        }
+
+        [Fact]
+        public void ParseAndRemoveStringsParameterLeavesArgsUntouchedWhenSwitchIsMissing()
+        {
+            List<string> argsList = new List<string> {
+                "literal-value",
+                "--filter",
+                "*"
+            };
+
+            var remaining = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out List<string> filters);
+
+            Assert.Empty(filters);
+            Assert.Equal(new[] { "literal-value", "--filter", "*" }, remaining);
+        }
+
+        [Fact]
+        public void ParseAndRemoveBooleanParameterReturnsFalseWhenSwitchIsMissing()
+        {
+            List<string> argsList = new List<string> {
+                "--filter",
+                "*"
+            };
+
+            CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--wasm", out bool enabled);
+
+            Assert.False(enabled);
+            Assert.Equal(new[] { "--filter", "*" }, argsList);
+        }
+
+        [Theory]
+        [InlineData("--partition-count")]
+        [InlineData("--partition-index")]
+        public void ParseAndRemoveIntParameterThrowsWhenValueIsMissing(string parameter)
+        {
+            List<string> argsList = new List<string> { parameter };
+
+            Assert.Throws<ArgumentException>(() => CommandLineOptions.ParseAndRemoveIntParameter(argsList, parameter, out int? _));
+        }
+
+        [Theory]
+        [InlineData("--partition-count", "abc")]
+        [InlineData("--partition-index", "3.14")]
+        public void ParseAndRemoveIntParameterThrowsWhenValueIsNotAnInteger(string parameter, string value)
+        {
+            List<string> argsList = new List<string> {
+                parameter,
+                value
+            };
+
+            Assert.Throws<ArgumentException>(() => CommandLineOptions.ParseAndRemoveIntParameter(argsList, parameter, out int? _));
+        }
     }
 }

src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs

@@ -63,8 +63,9 @@ public void NoBenchmarksAreOmitted_RealData()
             IConfig recommendedConfig = RecommendedConfig.Create(
                 artifactsPath: new DirectoryInfo(Path.Combine(Path.GetDirectoryName(typeof(PartitionFilterTests).Assembly.Location)!, "BenchmarkDotNet.Artifacts")),
                 mandatoryCategories: ImmutableHashSet.Create(Categories.Libraries, Categories.Runtime, Categories.ThirdParty));
-            (bool isSuccess, IConfig parsedConfig, var _) = ConfigParser.Parse(new string[] { "--filter", "*" }, nullLogger, recommendedConfig);
+            (bool isSuccess, IConfig? parsedConfig, var _) = ConfigParser.Parse(new string[] { "--filter", "*" }, nullLogger, recommendedConfig);
             Assert.True(isSuccess);
+            Assert.NotNull(parsedConfig);
 
             Assembly microbenchmarksAssembly = typeof(Categories).Assembly;
             (bool allTypesValid, IReadOnlyList<Type> runnable) = Running.TypeFilter.GetTypesWithRunnableBenchmarks(
@@ -73,7 +74,7 @@ public void NoBenchmarksAreOmitted_RealData()
                 nullLogger);
             Assert.True(allTypesValid);
 
-            BenchmarkRunInfo[] allBenchmarks = GetAllBenchmarks(parsedConfig, runnable);
+            BenchmarkRunInfo[] allBenchmarks = GetAllBenchmarks(parsedConfig!, runnable);
             Dictionary<string, int> idToPartitionIndex = new ();
 
             for (int i = 0; i < 10; i++)
@@ -86,7 +87,7 @@ public void NoBenchmarksAreOmitted_RealData()
                 {
                     PartitionFilter filter = new(PartitionCount, partitionIndex);
 
-                    foreach (BenchmarkCase benchmark in GetAllBenchmarks(parsedConfig, runnable).SelectMany(benchmark => benchmark.BenchmarksCases))
+                    foreach (BenchmarkCase benchmark in GetAllBenchmarks(parsedConfig!, runnable).SelectMany(benchmark => benchmark.BenchmarksCases))
                     {
                         if (filter.Predicate(benchmark))
                         {

src/tools/Reporting/Directory.Packages.props

@@ -4,7 +4,7 @@
     <CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
   </PropertyGroup>
   <ItemGroup>
-    <PackageVersion Include="Newtonsoft.Json" Version="13.0.1" />
+    <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="Microsoft.DotNet.PlatformAbstractions" Version="2.1.0" />
   </ItemGroup>
 </Project>

src/tools/ResultsComparer.Tests/ConsoleOutputCollection.cs

@@ -0,0 +1,8 @@
+using Xunit;
+
+namespace ResultsComparer.Tests;
+
+[CollectionDefinition("Console output", DisableParallelization = true)]
+public sealed class ConsoleOutputCollection
+{
+}

src/tools/ResultsComparer.Tests/DataTests.cs

@@ -0,0 +1,161 @@
+using System.Formats.Tar;
+using System.IO;
+using System.IO.Compression;
+using System.Text;
+using Xunit;
+
+namespace ResultsComparer.Tests;
+
+public class DataTests
+{
+    [Fact]
+    public void DecompressExtractsJsonFilesFromNestedZipArchives()
+    {
+        var tempDir = Directory.CreateTempSubdirectory();
+        try
+        {
+            var outerZipPath = Path.Combine(tempDir.FullName, "results.zip");
+            var outputDirectory = new DirectoryInfo(Path.Combine(tempDir.FullName, "output"));
+            outputDirectory.Create();
+
+            var innerZipBytes = CreateInnerZip(("net10.0/SampleBenchmark.full.json", ResultsComparerTestData.CreateBdnJson()));
+
+            using (var fileStream = File.Create(outerZipPath))
+            using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create))
+            {
+                var entry = archive.CreateEntry("Performance-Runs/net10.0/testuser/results.zip");
+                using var entryStream = entry.Open();
+                entryStream.Write(innerZipBytes, 0, innerZipBytes.Length);
+            }
+
+            global::ResultsComparer.Data.Decompress(new FileInfo(outerZipPath), outputDirectory);
+
+            var extractedFiles = Directory.GetFiles(outputDirectory.FullName, "*.full.json", SearchOption.AllDirectories);
+            var extractedFile = Assert.Single(extractedFiles);
+            var directoryName = Path.GetFileName(Path.GetDirectoryName(extractedFile));
+
+            Assert.Contains("testuser", directoryName, System.StringComparison.OrdinalIgnoreCase);
+            Assert.Contains("net10.0", directoryName, System.StringComparison.OrdinalIgnoreCase);
+            Assert.Contains("SampleBenchmark", File.ReadAllText(extractedFile));
+        }
+        finally
+        {
+            tempDir.Delete(recursive: true);
+        }
+    }
+
+    [Fact]
+    public void DecompressExtractsJsonFilesFromTarGzArchives()
+    {
+        var tempDir = Directory.CreateTempSubdirectory();
+        try
+        {
+            var outerZipPath = Path.Combine(tempDir.FullName, "results.zip");
+            var outputDirectory = new DirectoryInfo(Path.Combine(tempDir.FullName, "output"));
+            outputDirectory.Create();
+
+            var tarGzBytes = CreateTarGzArchive(
+                ("payload/SampleBenchmark.full.json", ResultsComparerTestData.CreateBdnJson()),
+                ("payload/README.md", "ignored"));
+
+            using (var fileStream = File.Create(outerZipPath))
+            using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create))
+            {
+                var entry = archive.CreateEntry("Performance-Runs/nativeaot10.0/testuser/arm64_win10-nativeaot10.0.tar.gz");
+                using var entryStream = entry.Open();
+                entryStream.Write(tarGzBytes, 0, tarGzBytes.Length);
+            }
+
+            global::ResultsComparer.Data.Decompress(new FileInfo(outerZipPath), outputDirectory);
+
+            var extractedFiles = Directory.GetFiles(outputDirectory.FullName, "*.full.json", SearchOption.AllDirectories);
+            var extractedFile = Assert.Single(extractedFiles);
+            var directoryName = Path.GetFileName(Path.GetDirectoryName(extractedFile));
+
+            Assert.Contains("nativeaot10.0", directoryName, System.StringComparison.OrdinalIgnoreCase);
+        }
+        finally
+        {
+            tempDir.Delete(recursive: true);
+        }
+    }
+
+    [Fact]
+    public void DecompressPrefersNewestBenchmarkDotNetVersionWhenDuplicatesExist()
+    {
+        var tempDir = Directory.CreateTempSubdirectory();
+        try
+        {
+            var outerZipPath = Path.Combine(tempDir.FullName, "results.zip");
+            var outputDirectory = new DirectoryInfo(Path.Combine(tempDir.FullName, "output"));
+            outputDirectory.Create();
+
+            var innerZipBytes = CreateInnerZip(
+                ("net10.0/SampleBenchmark-a.full.json", ResultsComparerTestData.CreateBdnJson(benchmarkDotNetVersion: "0.13.9")),
+                ("net10.0/SampleBenchmark-b.full.json", ResultsComparerTestData.CreateBdnJson(benchmarkDotNetVersion: "0.13.10")));
+
+            using (var fileStream = File.Create(outerZipPath))
+            using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create))
+            {
+                var entry = archive.CreateEntry("Performance-Runs/net10.0/testuser/results.zip");
+                using var entryStream = entry.Open();
+                entryStream.Write(innerZipBytes, 0, innerZipBytes.Length);
+            }
+
+            global::ResultsComparer.Data.Decompress(new FileInfo(outerZipPath), outputDirectory);
+
+            var extractedFile = Assert.Single(Directory.GetFiles(outputDirectory.FullName, "*.full.json", SearchOption.AllDirectories));
+            var json = File.ReadAllText(extractedFile);
+
+            Assert.Contains("\"BenchmarkDotNetVersion\": \"0.13.10\"", json);
+        }
+        finally
+        {
+            tempDir.Delete(recursive: true);
+        }
+    }
+
+    private static byte[] CreateInnerZip(params (string EntryName, string Content)[] entries)
+    {
+        using var stream = new MemoryStream();
+        using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true))
+        {
+            foreach (var (entryName, content) in entries)
+            {
+                var entry = archive.CreateEntry(entryName);
+                using var writer = new StreamWriter(entry.Open());
+                writer.Write(content);
+            }
+        }
+
+        return stream.ToArray();
+    }
+
+    private static byte[] CreateTarGzArchive(params (string EntryName, string Content)[] entries)
+    {
+        using var tarStream = new MemoryStream();
+        using (var tarWriter = new TarWriter(tarStream, leaveOpen: true))
+        {
+            foreach (var (entryName, content) in entries)
+            {
+                using var dataStream = new MemoryStream(Encoding.UTF8.GetBytes(content));
+                var tarEntry = new UstarTarEntry(TarEntryType.RegularFile, entryName)
+                {
+                    DataStream = dataStream
+                };
+
+                tarWriter.WriteEntry(tarEntry);
+            }
+        }
+
+        tarStream.Position = 0;
+
+        using var gzipStream = new MemoryStream();
+        using (var compressor = new GZipStream(gzipStream, CompressionLevel.SmallestSize, leaveOpen: true))
+        {
+            tarStream.CopyTo(compressor);
+        }
+
+        return gzipStream.ToArray();
+    }
+}