From 1f50ec82ba40fc560c33f6f0373a5554cfb7192c Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Fri, 21 Oct 2022 13:16:06 +0300 Subject: [PATCH 01/11] add tests --- .../Columns/JobColumnsApprovalTests.cs | 87 +++++++++++++++++++ .../Mocks/MockFactory.cs | 11 ++- 2 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs diff --git a/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs b/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs new file mode 100644 index 0000000000..ac9207d6cc --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs @@ -0,0 +1,87 @@ +using System; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Threading; +using ApprovalTests; +using ApprovalTests.Namers; +using ApprovalTests.Reporters; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Tests.Mocks; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using JetBrains.Annotations; +using Xunit; + +namespace BenchmarkDotNet.Tests.Columns +{ + // In case of failed approval tests, use the following reporter: + // [UseReporter(typeof(KDiffReporter))] + [UseReporter(typeof(XUnit2Reporter))] + [UseApprovalSubdirectory("ApprovedFiles")] + [Collection("ApprovalTests")] + public class JobColumnsApprovalTests : IDisposable + { + private readonly CultureInfo initCulture; + + public JobColumnsApprovalTests() => initCulture = Thread.CurrentThread.CurrentCulture; + + [UsedImplicitly] + public static TheoryData GetConfigs() + { + var data = new TheoryData + { + DefaultConfig.Instance + .AddJob(Job.InProcess), + + DefaultConfig.Instance + .AddJob(Job.Default) + .AddJob(Job.InProcess), + + DefaultConfig.Instance + .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core70).WithId("net7")), + + DefaultConfig.Instance + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core70).WithId("net7")) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core60).WithId("net6")), + + DefaultConfig.Instance + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core70).WithId("net7")) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core60).WithId("net6").WithToolchain(InProcessEmitToolchain.Instance)) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core60).WithId("net6")), + }; + + return data; + } + + [Theory] + [MemberData(nameof(GetConfigs))] + [MethodImpl(MethodImplOptions.NoInlining)] + public void ColumnsDisplayTest(IConfig config) + { + var fileName = string.Join("-", config.GetJobs()); + + NamerFactory.AdditionalInformation = fileName; + Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; + + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + fileName + " ==="); + + var exporter = MarkdownExporter.Mock; + var summary = MockFactory.CreateSummary(config); + exporter.ExportToLog(summary, logger); + + Approvals.Verify(logger.GetLog()); + } + + public void Dispose() => Thread.CurrentThread.CurrentCulture = initCulture; + + public class BenchmarkClass + { + [Benchmark] public void Method() { } + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs index f1393f8b62..5cb9e695e1 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs @@ -32,9 +32,12 @@ public static Summary CreateSummary(Type benchmarkType) ImmutableArray.Empty); } - public static Summary CreateSummary(IConfig config) => new Summary( + public static Summary CreateSummary(IConfig config) + => CreateSummary(config); + + public static Summary CreateSummary(IConfig config) => new Summary( "MockSummary", - CreateReports(config), + CreateReports(config), new HostEnvironmentInfoBuilder().WithoutDotNetSdkVersion().Build(), string.Empty, string.Empty, @@ -57,8 +60,8 @@ public static Summary CreateSummary(IConfig config, bool hugeSd, Metric[] metric ImmutableArray.Empty, ImmutableArray.Empty); - private static ImmutableArray CreateReports(IConfig config) - => CreateBenchmarks(config).Select(CreateSimpleReport).ToImmutableArray(); + private static ImmutableArray CreateReports(IConfig config) + => CreateBenchmarks(config).Select(CreateSimpleReport).ToImmutableArray(); private static BenchmarkCase[] CreateBenchmarks(IConfig config) => BenchmarkConverter.TypeToBenchmarks(typeof(TBenchmarks), config).BenchmarksCases; From 365a1d3d3dca6f6d2b5eec6c7fe1d5f991bf683d Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Fri, 21 Oct 2022 13:18:22 +0300 Subject: [PATCH 02/11] add approved files --- ...umnsDisplayTest.Default-InProcess.approved.txt | 14 ++++++++++++++ ...ests.ColumnsDisplayTest.InProcess.approved.txt | 13 +++++++++++++ ...ColumnsDisplayTest.net7-net6-net6.approved.txt | 15 +++++++++++++++ ...ests.ColumnsDisplayTest.net7-net6.approved.txt | 14 ++++++++++++++ ...ovalTests.ColumnsDisplayTest.net7.approved.txt | 15 +++++++++++++++ 5 files changed, 71 insertions(+) create mode 100644 tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.Default-InProcess.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.InProcess.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6-net6.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7.approved.txt diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.Default-InProcess.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.Default-InProcess.approved.txt new file mode 100644 index 0000000000..ce989dc159 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.Default-InProcess.approved.txt @@ -0,0 +1,14 @@ +=== Default-InProcess === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + InProcess : extra output line + + + Method | Job | Toolchain | Mean | Error | +------- |----------- |----------------------- |---------:|------:| + Method | DefaultJob | Default | 1.000 ns | NA | + Method | InProcess | InProcessEmitToolchain | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.InProcess.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.InProcess.approved.txt new file mode 100644 index 0000000000..907c87f921 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.InProcess.approved.txt @@ -0,0 +1,13 @@ +=== InProcess === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + InProcess : extra output line + +Job=InProcess Toolchain=InProcessEmitToolchain + + Method | Mean | Error | +------- |---------:|------:| + Method | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6-net6.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6-net6.approved.txt new file mode 100644 index 0000000000..3f6bd4c7f2 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6-net6.approved.txt @@ -0,0 +1,15 @@ +=== net7-net6-net6 === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + net6 : extra output line + net7 : extra output line + + + Method | Job | Runtime | Mean | Error | +------- |----- |--------- |---------:|------:| + Method | net6 | .NET 6.0 | 1.000 ns | NA | + Method | net6 | .NET 6.0 | 1.000 ns | NA | + Method | net7 | .NET 7.0 | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6.approved.txt new file mode 100644 index 0000000000..e0d3508293 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6.approved.txt @@ -0,0 +1,14 @@ +=== net7-net6 === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + net6 : extra output line + net7 : extra output line + + + Method | Job | Runtime | Mean | Error | +------- |----- |--------- |---------:|------:| + Method | net6 | .NET 6.0 | 1.000 ns | NA | + Method | net7 | .NET 7.0 | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7.approved.txt new file mode 100644 index 0000000000..99b375623b --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7.approved.txt @@ -0,0 +1,15 @@ +=== net7 === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + net7 : extra output line + +Job=net7 Runtime=.NET 7.0 IterationCount=1 +LaunchCount=1 RunStrategy=ColdStart UnrollFactor=1 +WarmupCount=1 + + Method | Mean | Error | +------- |---------:|------:| + Method | 1.000 ns | NA | From a6c5746763a4d7d393fff4fec474ba218692a234 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Fri, 21 Oct 2022 13:20:07 +0300 Subject: [PATCH 03/11] Rework Job.IsAvailable --- .../Columns/JobCharacteristicColumn.cs | 23 +++++++++++-------- src/BenchmarkDotNet/Reports/Summary.cs | 4 ---- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs b/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs index 5a92fac13e..23b00d2320 100644 --- a/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs +++ b/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; namespace BenchmarkDotNet.Columns { @@ -39,19 +40,21 @@ private JobCharacteristicColumn(Characteristic characteristic) public bool IsAvailable(Summary summary) { - if (summary.IsMultipleRuntimes) + switch (ColumnName) { - if (nameof(Toolchains.Toolchain).Equals(ColumnName)) - { + case Column.Job: return false; - } - if (nameof(Job).Equals(ColumnName)) - { - return summary.BenchmarksCases.Any(x => x.Job.HasValue(CharacteristicObject.IdCharacteristic)); - } - } + case Column.Toolchain: + var groups = summary.BenchmarksCases.GroupBy(b => b.GetRuntime(), b => b.GetToolchain().Name).ToArray(); + + bool isOneRuntime = groups.Length <= 1; + if (isOneRuntime) + return true; - return true; + return groups.Any(toolchainNames => toolchainNames.Distinct().Count() > 1); + default: + return true; + } } public string GetValue(Summary summary, BenchmarkCase benchmarkCase) diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index 08a1efecc9..b3b5150c37 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -36,7 +36,6 @@ public class Summary private ImmutableDictionary ReportMap { get; } private BaseliningStrategy BaseliningStrategy { get; } - private bool? isMultipleRuntimes; public Summary( string title, @@ -80,9 +79,6 @@ public Summary( public int GetNumberOfExecutedBenchmarks() => Reports.Count(report => report.ExecuteResults.Any(result => result.FoundExecutable)); - public bool IsMultipleRuntimes - => isMultipleRuntimes ??= BenchmarksCases.Length > 1 ? BenchmarksCases.Select(benchmark => benchmark.GetRuntime()).Distinct().Count() > 1 : false; - internal static Summary ValidationFailed(string title, string resultsDirectoryPath, string logFilePath, ImmutableArray? validationErrors = null) => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, validationErrors ?? ImmutableArray.Empty, ImmutableArray.Empty); From d763110c7bbc97b39623b411d2274f4d1cc06ee0 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Fri, 21 Oct 2022 13:28:13 +0300 Subject: [PATCH 04/11] Update new approved files --- ...s.ColumnsDisplayTest.Default-InProcess.approved.txt | 8 ++++---- ...ovalTests.ColumnsDisplayTest.InProcess.approved.txt | 2 +- ...ests.ColumnsDisplayTest.net7-net6-net6.approved.txt | 10 +++++----- ...ovalTests.ColumnsDisplayTest.net7-net6.approved.txt | 8 ++++---- ...sApprovalTests.ColumnsDisplayTest.net7.approved.txt | 5 ++--- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.Default-InProcess.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.Default-InProcess.approved.txt index ce989dc159..d7adbe51df 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.Default-InProcess.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.Default-InProcess.approved.txt @@ -8,7 +8,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC InProcess : extra output line - Method | Job | Toolchain | Mean | Error | -------- |----------- |----------------------- |---------:|------:| - Method | DefaultJob | Default | 1.000 ns | NA | - Method | InProcess | InProcessEmitToolchain | 1.000 ns | NA | + Method | Toolchain | Mean | Error | +------- |----------------------- |---------:|------:| + Method | Default | 1.000 ns | NA | + Method | InProcessEmitToolchain | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.InProcess.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.InProcess.approved.txt index 907c87f921..a92af8f304 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.InProcess.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.InProcess.approved.txt @@ -6,7 +6,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION InProcess : extra output line -Job=InProcess Toolchain=InProcessEmitToolchain +Toolchain=InProcessEmitToolchain Method | Mean | Error | ------- |---------:|------:| diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6-net6.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6-net6.approved.txt index 3f6bd4c7f2..a2af82901f 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6-net6.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6-net6.approved.txt @@ -8,8 +8,8 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC net7 : extra output line - Method | Job | Runtime | Mean | Error | -------- |----- |--------- |---------:|------:| - Method | net6 | .NET 6.0 | 1.000 ns | NA | - Method | net6 | .NET 6.0 | 1.000 ns | NA | - Method | net7 | .NET 7.0 | 1.000 ns | NA | + Method | Runtime | Toolchain | Mean | Error | +------- |--------- |----------------------- |---------:|------:| + Method | .NET 6.0 | Default | 1.000 ns | NA | + Method | .NET 6.0 | InProcessEmitToolchain | 1.000 ns | NA | + Method | .NET 7.0 | Default | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6.approved.txt index e0d3508293..5646cdd94e 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6.approved.txt @@ -8,7 +8,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC net7 : extra output line - Method | Job | Runtime | Mean | Error | -------- |----- |--------- |---------:|------:| - Method | net6 | .NET 6.0 | 1.000 ns | NA | - Method | net7 | .NET 7.0 | 1.000 ns | NA | + Method | Runtime | Mean | Error | +------- |--------- |---------:|------:| + Method | .NET 6.0 | 1.000 ns | NA | + Method | .NET 7.0 | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7.approved.txt index 99b375623b..22ab82b405 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7.approved.txt @@ -6,9 +6,8 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION net7 : extra output line -Job=net7 Runtime=.NET 7.0 IterationCount=1 -LaunchCount=1 RunStrategy=ColdStart UnrollFactor=1 -WarmupCount=1 +Runtime=.NET 7.0 IterationCount=1 LaunchCount=1 +RunStrategy=ColdStart UnrollFactor=1 WarmupCount=1 Method | Mean | Error | ------- |---------:|------:| From eacda8115f71f5031aeabcbb981c0d1b93d9c33d Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Fri, 21 Oct 2022 13:39:10 +0300 Subject: [PATCH 05/11] Update old approved files --- ...ovalTests.Exporters.Invariant.approved.txt | 21 +++++++------------ ...ApprovalTests.Exporters.en-US.approved.txt | 21 +++++++------------ ...ApprovalTests.Exporters.ru-RU.approved.txt | 21 +++++++------------ 3 files changed, 21 insertions(+), 42 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.Invariant.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.Invariant.approved.txt index 50556461f8..c8d7d109bf 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.Invariant.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.Invariant.approved.txt @@ -8,8 +8,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 .... [options="header"] |=== @@ -41,8 +40,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -
Job=LongRun  IterationCount=100  LaunchCount=3  
-WarmupCount=15  
+
IterationCount=100  LaunchCount=3  WarmupCount=15  
 
@@ -407,8 +405,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 Method | Mean | Error | StdDev | P67 | ------- |---------:|------:|---------:|---------:| @@ -425,8 +422,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 {noformat} ||Method || Mean ||Error || StdDev || P67 || @@ -442,8 +438,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 | Method | Mean | Error | StdDev | P67 | |------- |---------:|------:|---------:|---------:| @@ -460,8 +455,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 ``` | Method | Mean | Error | StdDev | P67 | @@ -478,8 +472,7 @@ MarkdownExporter-stackoverflow [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line - Job=LongRun IterationCount=100 LaunchCount=3 - WarmupCount=15 + IterationCount=100 LaunchCount=3 WarmupCount=15 Method | Mean | Error | StdDev | P67 | ------- |---------:|------:|---------:|---------:| diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.en-US.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.en-US.approved.txt index 50556461f8..c8d7d109bf 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.en-US.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.en-US.approved.txt @@ -8,8 +8,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 .... [options="header"] |=== @@ -41,8 +40,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -
Job=LongRun  IterationCount=100  LaunchCount=3  
-WarmupCount=15  
+
IterationCount=100  LaunchCount=3  WarmupCount=15  
 
@@ -407,8 +405,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 Method | Mean | Error | StdDev | P67 | ------- |---------:|------:|---------:|---------:| @@ -425,8 +422,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 {noformat} ||Method || Mean ||Error || StdDev || P67 || @@ -442,8 +438,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 | Method | Mean | Error | StdDev | P67 | |------- |---------:|------:|---------:|---------:| @@ -460,8 +455,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 ``` | Method | Mean | Error | StdDev | P67 | @@ -478,8 +472,7 @@ MarkdownExporter-stackoverflow [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line - Job=LongRun IterationCount=100 LaunchCount=3 - WarmupCount=15 + IterationCount=100 LaunchCount=3 WarmupCount=15 Method | Mean | Error | StdDev | P67 | ------- |---------:|------:|---------:|---------:| diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.ru-RU.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.ru-RU.approved.txt index b28fd9dd15..6dea3c88e3 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.ru-RU.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.ru-RU.approved.txt @@ -8,8 +8,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 .... [options="header"] |=== @@ -41,8 +40,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -
Job=LongRun  IterationCount=100  LaunchCount=3  
-WarmupCount=15  
+
IterationCount=100  LaunchCount=3  WarmupCount=15  
 
@@ -407,8 +405,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 Method | Mean | Error | StdDev | P67 | ------- |---------:|------:|---------:|---------:| @@ -425,8 +422,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 {noformat} ||Method || Mean ||Error || StdDev || P67 || @@ -442,8 +438,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 | Method | Mean | Error | StdDev | P67 | |------- |---------:|------:|---------:|---------:| @@ -460,8 +455,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 ``` | Method | Mean | Error | StdDev | P67 | @@ -478,8 +472,7 @@ MarkdownExporter-stackoverflow [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line - Job=LongRun IterationCount=100 LaunchCount=3 - WarmupCount=15 + IterationCount=100 LaunchCount=3 WarmupCount=15 Method | Mean | Error | StdDev | P67 | ------- |---------:|------:|---------:|---------:| From 88e091cadde6b2a38cb58a2588141872a20e92d3 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Fri, 21 Oct 2022 13:34:25 +0300 Subject: [PATCH 06/11] Display nice name instead tfm when set runtimes from cmd --- .../ConsoleArguments/ConfigParser.cs | 31 ++++++++----- .../CsProj/CsProjClassicNetToolchain.cs | 4 +- .../Toolchains/ToolchainExtensions.cs | 2 +- .../ConfigParserTests.cs | 46 ++++++++++++++++--- 4 files changed, 62 insertions(+), 21 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index e4b730aca3..f37d3092fd 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -107,7 +107,7 @@ private static bool Validate(CommandLineOptions options, ILogger logger) foreach (string runtime in options.Runtimes) { - if (!TryParse(runtime, out RuntimeMoniker runtimeMoniker)) + if (!TryParse(runtime, out RuntimeMoniker runtimeMoniker, out _)) { logger.WriteLineError($"The provided runtime \"{runtime}\" is invalid. Available options are: {string.Join(", ", Enum.GetNames(typeof(RuntimeMoniker)).Select(name => name.ToLower()))}."); return false; @@ -365,7 +365,7 @@ private static IEnumerable Expand(Job baseJob, CommandLineOptions options, private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, CommandLineOptions options) { - if (!TryParse(runtimeId, out RuntimeMoniker runtimeMoniker)) + if (!TryParse(runtimeId, out RuntimeMoniker runtimeMoniker, out string platformSpecificPostfix)) { throw new InvalidOperationException("Impossible, already validated by the Validate method"); } @@ -379,10 +379,10 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.Net472: case RuntimeMoniker.Net48: case RuntimeMoniker.Net481: + var clrRuntime = runtimeMoniker.GetRuntime(); return baseJob - .WithRuntime(runtimeMoniker.GetRuntime()) - .WithToolchain(CsProjClassicNetToolchain.From(runtimeId, options.RestorePath?.FullName)); - + .WithRuntime(clrRuntime) + .WithToolchain(CsProjClassicNetToolchain.From(clrRuntime.MsBuildMoniker, clrRuntime.Name, options.RestorePath?.FullName)); case RuntimeMoniker.NetCoreApp20: case RuntimeMoniker.NetCoreApp21: case RuntimeMoniker.NetCoreApp22: @@ -395,9 +395,10 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.Net60: case RuntimeMoniker.Net70: case RuntimeMoniker.Net80: + var coreRuntime = runtimeMoniker.GetRuntime(); return baseJob - .WithRuntime(runtimeMoniker.GetRuntime()) - .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(runtimeId, null, runtimeId, options.CliPath?.FullName, options.RestorePath?.FullName))); + .WithRuntime(coreRuntime) + .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(coreRuntime.MsBuildMoniker + platformSpecificPostfix, null, coreRuntime.Name + platformSpecificPostfix, options.CliPath?.FullName, options.RestorePath?.FullName))); case RuntimeMoniker.Mono: return baseJob.WithRuntime(new MonoRuntime("Mono", options.MonoPath?.FullName)); @@ -469,6 +470,7 @@ private static Job CreateAotJob(Job baseJob, CommandLineOptions options, Runtime builder.UseNuGet(ilCompilerVersion, nuGetFeedUrl); var runtime = runtimeMoniker.GetRuntime(); + builder.DisplayName(runtime.Name); builder.TargetFrameworkMoniker(runtime.MsBuildMoniker); return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain()); @@ -613,13 +615,20 @@ private static string GetCoreRunToolchainDisplayName(IReadOnlyList pat return coreRunPath.FullName.Substring(lastCommonDirectorySeparatorIndex); } - private static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) + private static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker, out string platformSpecificPostfix) { int index = runtime.IndexOf('-'); - return index < 0 - ? Enum.TryParse(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker) - : Enum.TryParse(runtime.Substring(0, index).Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker); + if (index < 0) + { + platformSpecificPostfix = ""; + return Enum.TryParse(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker); + } + else + { + platformSpecificPostfix = runtime.Substring(index).ToLower(); + return Enum.TryParse(runtime.Substring(0, index).Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker); + } } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs index be6fae3f4b..f7a7216e60 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs @@ -31,8 +31,8 @@ private CsProjClassicNetToolchain(string targetFrameworkMoniker, string name, st { } - public static IToolchain From(string targetFrameworkMoniker, string packagesPath = null) - => new CsProjClassicNetToolchain(targetFrameworkMoniker, targetFrameworkMoniker, packagesPath); + public static IToolchain From(string targetFrameworkMoniker, string name, string packagesPath = null) + => new CsProjClassicNetToolchain(targetFrameworkMoniker, name, packagesPath); public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) { diff --git a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs index c2af278c76..208d6ccc5f 100644 --- a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs +++ b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs @@ -37,7 +37,7 @@ internal static IToolchain GetToolchain(this Runtime runtime, Descriptor descrip if (RuntimeInformation.IsNetCore || preferMsBuildToolchains) return clrRuntime.RuntimeMoniker != RuntimeMoniker.NotRecognized ? GetToolchain(clrRuntime.RuntimeMoniker) - : CsProjClassicNetToolchain.From(clrRuntime.MsBuildMoniker); + : CsProjClassicNetToolchain.From(clrRuntime.MsBuildMoniker, clrRuntime.Name); return RoslynToolchain.Instance; diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index e3f32cede2..6cd71d3414 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -370,23 +370,40 @@ public void NetFrameworkMonikerParsedCorrectly(string tfm) } [Theory] - [InlineData("net50")] - [InlineData("net60")] - [InlineData("net70")] - [InlineData("net80")] - public void NetMonikersAreRecognizedAsNetCoreMonikers(string tfm) + [InlineData("net50", "net5.0")] + [InlineData("net60", "net6.0")] + [InlineData("net70", "net7.0")] + [InlineData("net80", "net8.0")] + public void NetMonikersAreRecognizedAsNetCoreMonikers(string input, string expected) { - var config = ConfigParser.Parse(new[] { "-r", tfm }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "-r", input }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); CsProjCoreToolchain toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; Assert.NotNull(toolchain); - Assert.Equal(tfm, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); + Assert.Equal(expected, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); + } + + [Theory] + [InlineData("net481", ".NET Framework 4.8.1")] + [InlineData("net7.0", ".NET 7.0")] + [InlineData("nativeaot7.0", "NativeAOT 7.0")] + public void NetMonikersHaveNiceNames(string input, string expected) + { + // CoreRuntime. + var config = ConfigParser.Parse(new[] { "-r", input }, new OutputLogger(Output)).config; + + Assert.Single(config.GetJobs()); + var job = config.GetJobs().Single(); + Assert.NotNull(job); + Assert.Equal(expected, job.Environment.Runtime.Name); + Assert.Equal(expected, job.Infrastructure.Toolchain.Name); } [Theory] [InlineData("net5.0-windows")] [InlineData("net5.0-ios")] + [InlineData("net5.0-ios15.0")] public void PlatformSpecificMonikersAreSupported(string msBuildMoniker) { var config = ConfigParser.Parse(new[] { "-r", msBuildMoniker }, new OutputLogger(Output)).config; @@ -397,6 +414,21 @@ public void PlatformSpecificMonikersAreSupported(string msBuildMoniker) Assert.Equal(msBuildMoniker, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); } + [Theory] + [InlineData("net7.0-ios", ".NET 7.0-ios")] + [InlineData("net7.0-IOS", ".NET 7.0-ios")] + [InlineData("net7.0-ios15.0", ".NET 7.0-ios15.0")] + public void PlatformSpecificMonikersHaveNiceNames(string input, string expected) + { + var config = ConfigParser.Parse(new[] { "-r", input }, new OutputLogger(Output)).config; + + Assert.Single(config.GetJobs()); + var job = config.GetJobs().Single(); + Assert.NotNull(job); + //todo: add postfix to runtime name instead toolchain name + Assert.Equal(expected, job.Infrastructure.Toolchain.Name); + } + [Fact] public void CanCompareFewDifferentRuntimes() { From 26e93151485fa9f922f8f7dcb5d0ffefe6ecb7d9 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Fri, 21 Oct 2022 17:04:15 +0300 Subject: [PATCH 07/11] revert: do not change NativeAOT toolchain name --- src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index f37d3092fd..4ffcc1cd88 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -470,7 +470,6 @@ private static Job CreateAotJob(Job baseJob, CommandLineOptions options, Runtime builder.UseNuGet(ilCompilerVersion, nuGetFeedUrl); var runtime = runtimeMoniker.GetRuntime(); - builder.DisplayName(runtime.Name); builder.TargetFrameworkMoniker(runtime.MsBuildMoniker); return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain()); From 0316820f9e211636f5c5ee5c28c96b890c1b56f3 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Fri, 21 Oct 2022 17:17:35 +0300 Subject: [PATCH 08/11] Fix tests --- .../ConfigParserTests.cs | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index 6cd71d3414..48e47e0f58 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -387,7 +387,6 @@ public void NetMonikersAreRecognizedAsNetCoreMonikers(string input, string expec [Theory] [InlineData("net481", ".NET Framework 4.8.1")] [InlineData("net7.0", ".NET 7.0")] - [InlineData("nativeaot7.0", "NativeAOT 7.0")] public void NetMonikersHaveNiceNames(string input, string expected) { // CoreRuntime. @@ -397,7 +396,22 @@ public void NetMonikersHaveNiceNames(string input, string expected) var job = config.GetJobs().Single(); Assert.NotNull(job); Assert.Equal(expected, job.Environment.Runtime.Name); - Assert.Equal(expected, job.Infrastructure.Toolchain.Name); + Assert.Equal(expected, job.GetToolchain().Name); + } + + [Theory] + [InlineData("nativeaot6.0", "NativeAOT 6.0", "ILCompiler 6.0.0-*")] + [InlineData("nativeaot7.0", "NativeAOT 7.0", "Latest ILCompiler")] + public void NativeAotMonikersHaveNiceNames(string input, string runtimeName, string toolchainName) + { + // CoreRuntime. + var config = ConfigParser.Parse(new[] { "-r", input }, new OutputLogger(Output)).config; + + Assert.Single(config.GetJobs()); + var job = config.GetJobs().Single(); + Assert.NotNull(job); + Assert.Equal(runtimeName, job.Environment.Runtime.Name); + Assert.Equal(toolchainName, job.GetToolchain().Name); } [Theory] @@ -426,7 +440,7 @@ public void PlatformSpecificMonikersHaveNiceNames(string input, string expected) var job = config.GetJobs().Single(); Assert.NotNull(job); //todo: add postfix to runtime name instead toolchain name - Assert.Equal(expected, job.Infrastructure.Toolchain.Name); + Assert.Equal(expected, job.GetToolchain().Name); } [Fact] From 516723f1a2420ef5792f46025a7318de12f1565e Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Fri, 21 Oct 2022 17:05:05 +0300 Subject: [PATCH 09/11] Add multiple input test --- ...ltipleInputColumnsDisplayTest.approved.txt | 26 ++++++++ .../Columns/JobColumnsApprovalTests.cs | 61 +++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.MultipleInputColumnsDisplayTest.MultipleInputColumnsDisplayTest.approved.txt diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.MultipleInputColumnsDisplayTest.MultipleInputColumnsDisplayTest.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.MultipleInputColumnsDisplayTest.MultipleInputColumnsDisplayTest.approved.txt new file mode 100644 index 0000000000..50fd289287 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.MultipleInputColumnsDisplayTest.MultipleInputColumnsDisplayTest.approved.txt @@ -0,0 +1,26 @@ +=== MultipleInputColumnsDisplayTest === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + Job-rndId0 : extra output line + Job-rndId1 : extra output line + Job-rndId2 : extra output line + Job-rndId3 : extra output line + Job-rndId4 : extra output line + Job-rndId5 : extra output line + Dry : extra output line + + + Method | Runtime | IterationCount | LaunchCount | RunStrategy | UnrollFactor | WarmupCount | Mean | Error | Ratio | +------- |--------------------- |--------------- |------------ |------------ |------------- |------------ |---------:|------:|------:| + Method | .NET 6.0 | Default | Default | ColdStart | 1 | Default | 1.000 ns | NA | 1.00 | + Method | .NET 6.0 | Default | Default | Default | 16 | Default | 1.000 ns | NA | 1.00 | + Method | .NET Framework 4.8.1 | Default | Default | ColdStart | 1 | Default | 1.000 ns | NA | 1.00 | + Method | .NET Framework 4.8.1 | Default | Default | Default | 16 | Default | 1.000 ns | NA | 1.00 | + Method | NativeAOT 6.0 | Default | Default | ColdStart | 1 | Default | 1.000 ns | NA | 1.00 | + Method | NativeAOT 6.0 | Default | Default | Default | 16 | Default | 1.000 ns | NA | 1.00 | + Method | .NET 6.0 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | + Method | .NET Framework 4.8.1 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | + Method | NativeAOT 6.0 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | diff --git a/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs b/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs index ac9207d6cc..0ff0d8ea97 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs +++ b/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs @@ -1,17 +1,22 @@ using System; using System.Globalization; using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; using System.Threading; using ApprovalTests; using ApprovalTests.Namers; using ApprovalTests.Reporters; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Tests.Mocks; +using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; using JetBrains.Annotations; using Xunit; @@ -83,5 +88,61 @@ public class BenchmarkClass { [Benchmark] public void Method() { } } + + [FactDotNetCoreOnly("In the .Net Framework cmd job uses CsProjClassicNetToolchain while fluent and attribute jobs use RoslynToolchain by default")] + [MethodImpl(MethodImplOptions.NoInlining)] + public void MultipleInputColumnsDisplayTest() + { + var cmdConfig = ConfigParser.Parse( + "--join --runtimes net481 net6.0 nativeaot6.0".Split(), NullLogger.Instance).config; + + var fluentConfig = ManualConfig.CreateEmpty().AddColumnProvider(DefaultColumnProviders.Instance) + .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core60)) + .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net481)) + .AddJob(Job.Dry.WithRuntime(NativeAotRuntime.Net60)); + + var config = ManualConfig.Union(cmdConfig, fluentConfig); + + NamerFactory.AdditionalInformation = nameof(MultipleInputColumnsDisplayTest); + Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; + + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + nameof(MultipleInputColumnsDisplayTest) + " ==="); + + var exporter = MarkdownExporter.Mock; + var summary = MockFactory.CreateSummary(config); + exporter.ExportToLog(summary, logger); + + var log = logger.GetLog(); + log = ReplaceRandomIDs(log); + Approvals.Verify(log); + } + + private static string ReplaceRandomIDs(string log) + { + var regex = new Regex(@"Job-\w*"); + + var index = 0; + foreach (Match match in regex.Matches(log)) + { + var randomGeneratedJobName = match.Value; + + // JobIdGenerator.GenerateRandomId() generates Job-ABCDEF + // respect the length for proper table formatting + var persistantName = $"Job-rndId{index}"; + log = log.Replace(randomGeneratedJobName, persistantName); + index++; + } + + return log; + } + + [SimpleJob(RunStrategy.ColdStart, RuntimeMoniker.Net60)] + [SimpleJob(RunStrategy.ColdStart, RuntimeMoniker.Net481)] + [SimpleJob(RunStrategy.ColdStart, RuntimeMoniker.NativeAot60)] + public class BenchmarkClass1 + { + [Benchmark] public void Method() { } + } } } \ No newline at end of file From 6f5ebe6637628395f02867aa53a67bc46cfb8839 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Mon, 21 Nov 2022 10:39:40 +0300 Subject: [PATCH 10/11] Remove toolchain postfix from name --- .../Toolchains/InProcess.Emit/InProcessEmitToolchain.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitToolchain.cs b/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitToolchain.cs index 8ca8f77897..89f519ea64 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitToolchain.cs @@ -23,7 +23,7 @@ public InProcessEmitToolchain(bool logOutput) : /// true if the output should be logged. public InProcessEmitToolchain(TimeSpan timeout, bool logOutput) : base( - nameof(InProcessEmitToolchain), + "InProcessEmit", new InProcessEmitGenerator(), new InProcessEmitBuilder(), new InProcessEmitExecutor(timeout, logOutput)) From 2c99d26ec4cd36ad33cd4c1c00a2a4b87068684e Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Sun, 20 Nov 2022 00:29:53 +0300 Subject: [PATCH 11/11] Add implicitId characteristics --- .../Attributes/Jobs/JobConfigbaseAttribute.cs | 6 +- .../Attributes/Jobs/SimpleJobAttribute.cs | 2 +- .../Columns/JobCharacteristicColumn.cs | 17 ++- src/BenchmarkDotNet/Jobs/Job.cs | 25 +-- src/BenchmarkDotNet/Jobs/JobExtensions.cs | 5 +- ...VMToolChain.cs => MonoAotLLVMToolchain.cs} | 0 ...mnsDisplayTest.net7-net6-net6.approved.txt | 15 -- ...sts.ExplicitIdsAreDiscovered.approved.txt} | 7 +- ...ests.ImplicitIdsAreDiscovered.approved.txt | 13 ++ ...ltipleInputJobsAreDiscovered.approved.txt} | 29 ++-- ...ovalTests.RuntimeIsDiscovered.approved.txt | 12 ++ ...lTests.RuntimesAreDiscovered.approved.txt} | 9 +- ...esWithToolchainsAreDiscovered.approved.txt | 13 ++ ...lTests.ToolchainIsDiscovered.approved.txt} | 5 +- ...ests.ToolchainsAreDiscovered.approved.txt} | 11 +- .../Columns/JobColumnsApprovalTests.cs | 143 ++++++++++-------- 16 files changed, 183 insertions(+), 129 deletions(-) rename src/BenchmarkDotNet/Toolchains/MonoAotLLVM/{MonoAotLLVMToolChain.cs => MonoAotLLVMToolchain.cs} (100%) delete mode 100644 tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6-net6.approved.txt rename tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/{JobColumnsApprovalTests.ColumnsDisplayTest.net7.approved.txt => JobColumnsApprovalTests.ExplicitIdsAreDiscovered.approved.txt} (81%) create mode 100644 tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ImplicitIdsAreDiscovered.approved.txt rename tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/{JobColumnsApprovalTests.MultipleInputColumnsDisplayTest.MultipleInputColumnsDisplayTest.approved.txt => JobColumnsApprovalTests.MultipleInputJobsAreDiscovered.approved.txt} (56%) create mode 100644 tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimeIsDiscovered.approved.txt rename tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/{JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6.approved.txt => JobColumnsApprovalTests.RuntimesAreDiscovered.approved.txt} (72%) create mode 100644 tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimesWithToolchainsAreDiscovered.approved.txt rename tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/{JobColumnsApprovalTests.ColumnsDisplayTest.InProcess.approved.txt => JobColumnsApprovalTests.ToolchainIsDiscovered.approved.txt} (87%) rename tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/{JobColumnsApprovalTests.ColumnsDisplayTest.Default-InProcess.approved.txt => JobColumnsApprovalTests.ToolchainsAreDiscovered.approved.txt} (59%) diff --git a/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs index 1630da4fe5..101fca151a 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs @@ -21,8 +21,8 @@ public class JobConfigBaseAttribute : Attribute, IConfigSource protected static Job GetJob(Job sourceJob, RuntimeMoniker runtimeMoniker, Jit? jit, Platform? platform) { var runtime = runtimeMoniker.GetRuntime(); - var baseJob = sourceJob.WithRuntime(runtime).WithId($"{sourceJob.Id}-{runtime.Name}"); - var id = baseJob.Id; + var baseJob = sourceJob.WithRuntime(runtime); + var id = $"{sourceJob.Id}-{runtime.Name}"; if (jit.HasValue) { @@ -36,7 +36,7 @@ protected static Job GetJob(Job sourceJob, RuntimeMoniker runtimeMoniker, Jit? j id += "-" + platform.Value; } - return baseJob.WithId(id).Freeze(); + return baseJob.WithImplicitId(id).Freeze(); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs index e0b80cf482..d2b277198f 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs @@ -108,7 +108,7 @@ private static Job CreateJob(string id, int launchCount, int warmupCount, int it } if (id == null && manualValuesCount == 1 && runtimeMoniker != RuntimeMoniker.HostProcess) - job = job.WithId(runtimeMoniker.GetRuntime().Name); + job = job.WithImplicitId(runtimeMoniker.GetRuntime().Name); return job.Freeze(); } diff --git a/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs b/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs index 23b00d2320..680db42d06 100644 --- a/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs +++ b/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs @@ -2,10 +2,10 @@ using System.Linq; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains; namespace BenchmarkDotNet.Columns { @@ -43,15 +43,20 @@ public bool IsAvailable(Summary summary) switch (ColumnName) { case Column.Job: - return false; + return summary.BenchmarksCases + .Any(b => b.Job.HasValue(Job.IdCharacteristic) && !b.Job.GetValue(Job.ImplicitIdCharacteristic)); case Column.Toolchain: - var groups = summary.BenchmarksCases.GroupBy(b => b.GetRuntime(), b => b.GetToolchain().Name).ToArray(); - bool isOneRuntime = groups.Length <= 1; - if (isOneRuntime) + var toolchainsByRuntime = summary.BenchmarksCases + .GroupBy(b => b.GetRuntime().Name, b => b.Job.GetValue(InfrastructureMode.ToolchainCharacteristic)) + .ToArray(); + + if (toolchainsByRuntime.Length <= 1) return true; - return groups.Any(toolchainNames => toolchainNames.Distinct().Count() > 1); + return toolchainsByRuntime.Any(toolchains => toolchains.Where(toolchain => toolchain != null) + .DistinctBy(toolchain => toolchain.Name) + .Count() > 1); default: return true; } diff --git a/src/BenchmarkDotNet/Jobs/Job.cs b/src/BenchmarkDotNet/Jobs/Job.cs index 6ee47f7150..bd03ff74e9 100644 --- a/src/BenchmarkDotNet/Jobs/Job.cs +++ b/src/BenchmarkDotNet/Jobs/Job.cs @@ -14,22 +14,24 @@ public sealed class Job : JobMode [PublicAPI] public static readonly Characteristic AccuracyCharacteristic = CreateCharacteristic(nameof(Accuracy)); [PublicAPI] public static readonly Characteristic MetaCharacteristic = CreateCharacteristic(nameof(Meta)); - public static readonly Job LegacyJitX86 = new Job(nameof(LegacyJitX86), EnvironmentMode.LegacyJitX86).Freeze(); - public static readonly Job LegacyJitX64 = new Job(nameof(LegacyJitX64), EnvironmentMode.LegacyJitX64).Freeze(); - public static readonly Job RyuJitX64 = new Job(nameof(RyuJitX64), EnvironmentMode.RyuJitX64).Freeze(); - public static readonly Job RyuJitX86 = new Job(nameof(RyuJitX86), EnvironmentMode.RyuJitX86).Freeze(); + internal static readonly Characteristic ImplicitIdCharacteristic = CreateHiddenCharacteristic("ImplicitId"); + + public static readonly Job LegacyJitX86 = new Job(EnvironmentMode.LegacyJitX86).WithImplicitId(nameof(LegacyJitX86)).Freeze(); + public static readonly Job LegacyJitX64 = new Job(EnvironmentMode.LegacyJitX64).WithImplicitId(nameof(LegacyJitX64)).Freeze(); + public static readonly Job RyuJitX64 = new Job(EnvironmentMode.RyuJitX64).WithImplicitId(nameof(RyuJitX64)).Freeze(); + public static readonly Job RyuJitX86 = new Job(EnvironmentMode.RyuJitX86).WithImplicitId(nameof(RyuJitX86)).Freeze(); // Run - public static readonly Job Dry = new Job(nameof(Dry), RunMode.Dry).Freeze(); + public static readonly Job Dry = new Job(RunMode.Dry).WithImplicitId(nameof(Dry)).Freeze(); - public static readonly Job ShortRun = new Job(nameof(ShortRun), RunMode.Short).Freeze(); - public static readonly Job MediumRun = new Job(nameof(MediumRun), RunMode.Medium).Freeze(); - public static readonly Job LongRun = new Job(nameof(LongRun), RunMode.Long).Freeze(); - public static readonly Job VeryLongRun = new Job(nameof(VeryLongRun), RunMode.VeryLong).Freeze(); + public static readonly Job ShortRun = new Job(RunMode.Short).WithImplicitId(nameof(ShortRun)).Freeze(); + public static readonly Job MediumRun = new Job(RunMode.Medium).WithImplicitId(nameof(MediumRun)).Freeze(); + public static readonly Job LongRun = new Job(RunMode.Long).WithImplicitId(nameof(LongRun)).Freeze(); + public static readonly Job VeryLongRun = new Job(RunMode.VeryLong).WithImplicitId(nameof(VeryLongRun)).Freeze(); // Infrastructure - public static readonly Job InProcess = new Job(nameof(InProcess), InfrastructureMode.InProcess); - public static readonly Job InProcessDontLogOutput = new Job(nameof(InProcessDontLogOutput), InfrastructureMode.InProcessDontLogOutput); + public static readonly Job InProcess = new Job(InfrastructureMode.InProcess).WithImplicitId(nameof(InProcess)); + public static readonly Job InProcessDontLogOutput = new Job(InfrastructureMode.InProcessDontLogOutput).WithImplicitId(nameof(InProcessDontLogOutput)); public Job() : this((string)null) { } @@ -40,6 +42,7 @@ public Job(string id) : base(id) InfrastructureCharacteristic[this] = new InfrastructureMode(); AccuracyCharacteristic[this] = new AccuracyMode(); MetaCharacteristic[this] = new MetaMode(); + ImplicitIdCharacteristic[this] = false; } public Job(CharacteristicObject other) : this((string)null, other) diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index ca731ad723..0b8738773c 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -21,7 +21,10 @@ public static class JobExtensions public static Job With(this Job job, Platform platform) => job.WithPlatform(platform); public static Job WithPlatform(this Job job, Platform platform) => job.WithCore(j => j.Environment.Platform = platform); - public static Job WithId(this Job job, string id) => new Job(id, job); + public static Job WithId(this Job job, string id) => new Job(id, job).WithImplicitId(false); + + internal static Job WithImplicitId(this Job job, string id) => new Job(id, job).WithImplicitId(true); + private static Job WithImplicitId(this Job job, bool isImplicit) => job.WithCore(j => Job.ImplicitIdCharacteristic[j] = isImplicit); // Env [EditorBrowsable(EditorBrowsableState.Never)] diff --git a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolchain.cs similarity index 100% rename from src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs rename to src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolchain.cs diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6-net6.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6-net6.approved.txt deleted file mode 100644 index a2af82901f..0000000000 --- a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6-net6.approved.txt +++ /dev/null @@ -1,15 +0,0 @@ -=== net7-net6-net6 === - -BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V -MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - net6 : extra output line - net7 : extra output line - - - Method | Runtime | Toolchain | Mean | Error | -------- |--------- |----------------------- |---------:|------:| - Method | .NET 6.0 | Default | 1.000 ns | NA | - Method | .NET 6.0 | InProcessEmitToolchain | 1.000 ns | NA | - Method | .NET 7.0 | Default | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ExplicitIdsAreDiscovered.approved.txt similarity index 81% rename from tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7.approved.txt rename to tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ExplicitIdsAreDiscovered.approved.txt index 22ab82b405..3c088241cb 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ExplicitIdsAreDiscovered.approved.txt @@ -1,12 +1,11 @@ -=== net7 === - + BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - net7 : extra output line + Dry : extra output line -Runtime=.NET 7.0 IterationCount=1 LaunchCount=1 +Job=Dry IterationCount=1 LaunchCount=1 RunStrategy=ColdStart UnrollFactor=1 WarmupCount=1 Method | Mean | Error | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ImplicitIdsAreDiscovered.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ImplicitIdsAreDiscovered.approved.txt new file mode 100644 index 0000000000..bfe110757c --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ImplicitIdsAreDiscovered.approved.txt @@ -0,0 +1,13 @@ + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + Dry : extra output line + +IterationCount=1 LaunchCount=1 RunStrategy=ColdStart +UnrollFactor=1 WarmupCount=1 + + Method | Mean | Error | +------- |---------:|------:| + Method | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.MultipleInputColumnsDisplayTest.MultipleInputColumnsDisplayTest.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.MultipleInputJobsAreDiscovered.approved.txt similarity index 56% rename from tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.MultipleInputColumnsDisplayTest.MultipleInputColumnsDisplayTest.approved.txt rename to tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.MultipleInputJobsAreDiscovered.approved.txt index 50fd289287..fd2768b16f 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.MultipleInputColumnsDisplayTest.MultipleInputColumnsDisplayTest.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.MultipleInputJobsAreDiscovered.approved.txt @@ -1,26 +1,25 @@ -=== MultipleInputColumnsDisplayTest === - + BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - Job-rndId0 : extra output line - Job-rndId1 : extra output line - Job-rndId2 : extra output line - Job-rndId3 : extra output line - Job-rndId4 : extra output line - Job-rndId5 : extra output line - Dry : extra output line + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + Job-rndId0 : extra output line + Job-rndId1 : extra output line + Job-rndId2 : extra output line + Dry : extra output line + Dry-.NET 6.0 : extra output line + Dry-.NET Framework 4.8.1 : extra output line + Dry-NativeAOT 6.0 : extra output line Method | Runtime | IterationCount | LaunchCount | RunStrategy | UnrollFactor | WarmupCount | Mean | Error | Ratio | ------- |--------------------- |--------------- |------------ |------------ |------------- |------------ |---------:|------:|------:| - Method | .NET 6.0 | Default | Default | ColdStart | 1 | Default | 1.000 ns | NA | 1.00 | - Method | .NET 6.0 | Default | Default | Default | 16 | Default | 1.000 ns | NA | 1.00 | - Method | .NET Framework 4.8.1 | Default | Default | ColdStart | 1 | Default | 1.000 ns | NA | 1.00 | + Method | .NET 7.0 | Default | Default | Default | 16 | Default | 1.000 ns | NA | 1.00 | Method | .NET Framework 4.8.1 | Default | Default | Default | 16 | Default | 1.000 ns | NA | 1.00 | - Method | NativeAOT 6.0 | Default | Default | ColdStart | 1 | Default | 1.000 ns | NA | 1.00 | - Method | NativeAOT 6.0 | Default | Default | Default | 16 | Default | 1.000 ns | NA | 1.00 | + Method | NativeAOT 7.0 | Default | Default | Default | 16 | Default | 1.000 ns | NA | 1.00 | + Method | .NET 7.0 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | + Method | .NET Framework 4.8.1 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | + Method | NativeAOT 7.0 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | Method | .NET 6.0 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | Method | .NET Framework 4.8.1 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | Method | NativeAOT 6.0 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimeIsDiscovered.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimeIsDiscovered.approved.txt new file mode 100644 index 0000000000..f87a0283ae --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimeIsDiscovered.approved.txt @@ -0,0 +1,12 @@ + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + Job-rndId0 : extra output line + +Runtime=.NET 7.0 + + Method | Mean | Error | +------- |---------:|------:| + Method | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimesAreDiscovered.approved.txt similarity index 72% rename from tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6.approved.txt rename to tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimesAreDiscovered.approved.txt index 5646cdd94e..23a72cac47 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.net7-net6.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimesAreDiscovered.approved.txt @@ -1,11 +1,10 @@ -=== net7-net6 === - + BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - net6 : extra output line - net7 : extra output line + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + Job-rndId0 : extra output line + Job-rndId1 : extra output line Method | Runtime | Mean | Error | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimesWithToolchainsAreDiscovered.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimesWithToolchainsAreDiscovered.approved.txt new file mode 100644 index 0000000000..23a72cac47 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimesWithToolchainsAreDiscovered.approved.txt @@ -0,0 +1,13 @@ + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + Job-rndId0 : extra output line + Job-rndId1 : extra output line + + + Method | Runtime | Mean | Error | +------- |--------- |---------:|------:| + Method | .NET 6.0 | 1.000 ns | NA | + Method | .NET 7.0 | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.InProcess.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ToolchainIsDiscovered.approved.txt similarity index 87% rename from tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.InProcess.approved.txt rename to tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ToolchainIsDiscovered.approved.txt index a92af8f304..e4965071f9 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.InProcess.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ToolchainIsDiscovered.approved.txt @@ -1,12 +1,11 @@ -=== InProcess === - + BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION InProcess : extra output line -Toolchain=InProcessEmitToolchain +Toolchain=InProcessEmit Method | Mean | Error | ------- |---------:|------:| diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.Default-InProcess.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ToolchainsAreDiscovered.approved.txt similarity index 59% rename from tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.Default-InProcess.approved.txt rename to tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ToolchainsAreDiscovered.approved.txt index d7adbe51df..6348d6ff81 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ColumnsDisplayTest.Default-InProcess.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ToolchainsAreDiscovered.approved.txt @@ -1,5 +1,4 @@ -=== Default-InProcess === - + BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC @@ -8,7 +7,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC InProcess : extra output line - Method | Toolchain | Mean | Error | -------- |----------------------- |---------:|------:| - Method | Default | 1.000 ns | NA | - Method | InProcessEmitToolchain | 1.000 ns | NA | + Method | Toolchain | Mean | Error | +------- |-------------- |---------:|------:| + Method | Default | 1.000 ns | NA | + Method | InProcessEmit | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs b/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs index 0ff0d8ea97..ed05d7be65 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs +++ b/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs @@ -7,7 +7,6 @@ using ApprovalTests.Namers; using ApprovalTests.Reporters; using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.ConsoleArguments; using BenchmarkDotNet.Engines; @@ -18,7 +17,6 @@ using BenchmarkDotNet.Tests.Mocks; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; -using JetBrains.Annotations; using Xunit; namespace BenchmarkDotNet.Tests.Columns @@ -34,87 +32,122 @@ public class JobColumnsApprovalTests : IDisposable public JobColumnsApprovalTests() => initCulture = Thread.CurrentThread.CurrentCulture; - [UsedImplicitly] - public static TheoryData GetConfigs() - { - var data = new TheoryData - { - DefaultConfig.Instance - .AddJob(Job.InProcess), + public void Dispose() => Thread.CurrentThread.CurrentCulture = initCulture; - DefaultConfig.Instance - .AddJob(Job.Default) - .AddJob(Job.InProcess), + [Fact] + [MethodImpl(MethodImplOptions.NoInlining)] + public void ImplicitIdsAreDiscovered() + { + var config = DefaultConfig.Instance.AddJob(Job.Dry); - DefaultConfig.Instance - .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core70).WithId("net7")), + Verify(config); + } - DefaultConfig.Instance - .AddJob(Job.Default.WithRuntime(CoreRuntime.Core70).WithId("net7")) - .AddJob(Job.Default.WithRuntime(CoreRuntime.Core60).WithId("net6")), + [Fact] + [MethodImpl(MethodImplOptions.NoInlining)] + public void ExplicitIdsAreDiscovered() + { + var config = DefaultConfig.Instance.AddJob(Job.Dry.WithId("Dry")); - DefaultConfig.Instance - .AddJob(Job.Default.WithRuntime(CoreRuntime.Core70).WithId("net7")) - .AddJob(Job.Default.WithRuntime(CoreRuntime.Core60).WithId("net6").WithToolchain(InProcessEmitToolchain.Instance)) - .AddJob(Job.Default.WithRuntime(CoreRuntime.Core60).WithId("net6")), - }; + Verify(config); + } - return data; + [DryJob(RuntimeMoniker.Net60)] + [SimpleJob(RuntimeMoniker.Net60)] + [SimpleJob(RunStrategy.ColdStart, RuntimeMoniker.Net60, 1, 1, 1)] + public class BenchmarkWithAttributeJobs + { + [Benchmark] public void Method() { } } - [Theory] - [MemberData(nameof(GetConfigs))] - [MethodImpl(MethodImplOptions.NoInlining)] - public void ColumnsDisplayTest(IConfig config) + [Fact] + public void ToolchainIsDiscovered() { - var fileName = string.Join("-", config.GetJobs()); + var config = DefaultConfig.Instance.AddJob(Job.InProcess); - NamerFactory.AdditionalInformation = fileName; - Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; + Verify(config); + } - var logger = new AccumulationLogger(); - logger.WriteLine("=== " + fileName + " ==="); + [Fact] + public void ToolchainsAreDiscovered() + { + var config = DefaultConfig.Instance + .AddJob(Job.Default) + .AddJob(Job.InProcess); - var exporter = MarkdownExporter.Mock; - var summary = MockFactory.CreateSummary(config); - exporter.ExportToLog(summary, logger); + Verify(config); + } + + [Fact] + public void RuntimeIsDiscovered() + { + var config = DefaultConfig.Instance + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core70)); - Approvals.Verify(logger.GetLog()); + Verify(config); } - public void Dispose() => Thread.CurrentThread.CurrentCulture = initCulture; + [Fact] + public void RuntimesAreDiscovered() + { + var config = DefaultConfig.Instance + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core70)) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core60)); - public class BenchmarkClass + Verify(config); + } + + [Fact] + public void RuntimesWithToolchainsAreDiscovered() { - [Benchmark] public void Method() { } + var config = DefaultConfig.Instance + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core70).WithToolchain(InProcessEmitToolchain.Instance)) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core60)); + + Verify(config); } [FactDotNetCoreOnly("In the .Net Framework cmd job uses CsProjClassicNetToolchain while fluent and attribute jobs use RoslynToolchain by default")] [MethodImpl(MethodImplOptions.NoInlining)] - public void MultipleInputColumnsDisplayTest() + public void MultipleInputJobsAreDiscovered() { var cmdConfig = ConfigParser.Parse( - "--join --runtimes net481 net6.0 nativeaot6.0".Split(), NullLogger.Instance).config; + "--join --runtimes net481 net7.0 nativeaot7.0".Split(), NullLogger.Instance).config; - var fluentConfig = ManualConfig.CreateEmpty().AddColumnProvider(DefaultColumnProviders.Instance) - .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core60)) + var fluentConfig = DefaultConfig.Instance + .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core70)) .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net481)) - .AddJob(Job.Dry.WithRuntime(NativeAotRuntime.Net60)); + .AddJob(Job.Dry.WithRuntime(NativeAotRuntime.Net70)); var config = ManualConfig.Union(cmdConfig, fluentConfig); - NamerFactory.AdditionalInformation = nameof(MultipleInputColumnsDisplayTest); - Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; + Verify(config); + } - var logger = new AccumulationLogger(); - logger.WriteLine("=== " + nameof(MultipleInputColumnsDisplayTest) + " ==="); + public class Benchmark + { + [Benchmark] public void Method() { } + } + + [DryJob(RuntimeMoniker.Net60)] + [DryJob(RuntimeMoniker.Net481)] + [DryJob(RuntimeMoniker.NativeAot60)] + public class BenchmarkWithDryJobs + { + [Benchmark] public void Method() { } + } + + private static void Verify(IConfig config) + { + Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; var exporter = MarkdownExporter.Mock; - var summary = MockFactory.CreateSummary(config); + var summary = MockFactory.CreateSummary(config); + + var logger = new AccumulationLogger(); exporter.ExportToLog(summary, logger); - var log = logger.GetLog(); - log = ReplaceRandomIDs(log); + var log = ReplaceRandomIDs(logger.GetLog()); Approvals.Verify(log); } @@ -136,13 +169,5 @@ private static string ReplaceRandomIDs(string log) return log; } - - [SimpleJob(RunStrategy.ColdStart, RuntimeMoniker.Net60)] - [SimpleJob(RunStrategy.ColdStart, RuntimeMoniker.Net481)] - [SimpleJob(RunStrategy.ColdStart, RuntimeMoniker.NativeAot60)] - public class BenchmarkClass1 - { - [Benchmark] public void Method() { } - } } } \ No newline at end of file