diff --git a/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs index 1630da4fe5..101fca151a 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs @@ -21,8 +21,8 @@ public class JobConfigBaseAttribute : Attribute, IConfigSource protected static Job GetJob(Job sourceJob, RuntimeMoniker runtimeMoniker, Jit? jit, Platform? platform) { var runtime = runtimeMoniker.GetRuntime(); - var baseJob = sourceJob.WithRuntime(runtime).WithId($"{sourceJob.Id}-{runtime.Name}"); - var id = baseJob.Id; + var baseJob = sourceJob.WithRuntime(runtime); + var id = $"{sourceJob.Id}-{runtime.Name}"; if (jit.HasValue) { @@ -36,7 +36,7 @@ protected static Job GetJob(Job sourceJob, RuntimeMoniker runtimeMoniker, Jit? j id += "-" + platform.Value; } - return baseJob.WithId(id).Freeze(); + return baseJob.WithImplicitId(id).Freeze(); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs index e0b80cf482..d2b277198f 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs @@ -108,7 +108,7 @@ private static Job CreateJob(string id, int launchCount, int warmupCount, int it } if (id == null && manualValuesCount == 1 && runtimeMoniker != RuntimeMoniker.HostProcess) - job = job.WithId(runtimeMoniker.GetRuntime().Name); + job = job.WithImplicitId(runtimeMoniker.GetRuntime().Name); return job.Freeze(); } diff --git a/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs b/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs index 5a92fac13e..680db42d06 100644 --- a/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs +++ b/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs @@ -2,6 +2,7 @@ using System.Linq; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; @@ -39,19 +40,26 @@ private JobCharacteristicColumn(Characteristic characteristic) public bool IsAvailable(Summary summary) { - if (summary.IsMultipleRuntimes) + switch (ColumnName) { - if (nameof(Toolchains.Toolchain).Equals(ColumnName)) - { - return false; - } - if (nameof(Job).Equals(ColumnName)) - { - return summary.BenchmarksCases.Any(x => x.Job.HasValue(CharacteristicObject.IdCharacteristic)); - } - } + case Column.Job: + return summary.BenchmarksCases + .Any(b => b.Job.HasValue(Job.IdCharacteristic) && !b.Job.GetValue(Job.ImplicitIdCharacteristic)); + case Column.Toolchain: + + var toolchainsByRuntime = summary.BenchmarksCases + .GroupBy(b => b.GetRuntime().Name, b => b.Job.GetValue(InfrastructureMode.ToolchainCharacteristic)) + .ToArray(); - return true; + if (toolchainsByRuntime.Length <= 1) + return true; + + return toolchainsByRuntime.Any(toolchains => toolchains.Where(toolchain => toolchain != null) + .DistinctBy(toolchain => toolchain.Name) + .Count() > 1); + default: + return true; + } } public string GetValue(Summary summary, BenchmarkCase benchmarkCase) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index e4b730aca3..4ffcc1cd88 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -107,7 +107,7 @@ private static bool Validate(CommandLineOptions options, ILogger logger) foreach (string runtime in options.Runtimes) { - if (!TryParse(runtime, out RuntimeMoniker runtimeMoniker)) + if (!TryParse(runtime, out RuntimeMoniker runtimeMoniker, out _)) { logger.WriteLineError($"The provided runtime \"{runtime}\" is invalid. Available options are: {string.Join(", ", Enum.GetNames(typeof(RuntimeMoniker)).Select(name => name.ToLower()))}."); return false; @@ -365,7 +365,7 @@ private static IEnumerable Expand(Job baseJob, CommandLineOptions options, private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, CommandLineOptions options) { - if (!TryParse(runtimeId, out RuntimeMoniker runtimeMoniker)) + if (!TryParse(runtimeId, out RuntimeMoniker runtimeMoniker, out string platformSpecificPostfix)) { throw new InvalidOperationException("Impossible, already validated by the Validate method"); } @@ -379,10 +379,10 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.Net472: case RuntimeMoniker.Net48: case RuntimeMoniker.Net481: + var clrRuntime = runtimeMoniker.GetRuntime(); return baseJob - .WithRuntime(runtimeMoniker.GetRuntime()) - .WithToolchain(CsProjClassicNetToolchain.From(runtimeId, options.RestorePath?.FullName)); - + .WithRuntime(clrRuntime) + .WithToolchain(CsProjClassicNetToolchain.From(clrRuntime.MsBuildMoniker, clrRuntime.Name, options.RestorePath?.FullName)); case RuntimeMoniker.NetCoreApp20: case RuntimeMoniker.NetCoreApp21: case RuntimeMoniker.NetCoreApp22: @@ -395,9 +395,10 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.Net60: case RuntimeMoniker.Net70: case RuntimeMoniker.Net80: + var coreRuntime = runtimeMoniker.GetRuntime(); return baseJob - .WithRuntime(runtimeMoniker.GetRuntime()) - .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(runtimeId, null, runtimeId, options.CliPath?.FullName, options.RestorePath?.FullName))); + .WithRuntime(coreRuntime) + .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(coreRuntime.MsBuildMoniker + platformSpecificPostfix, null, coreRuntime.Name + platformSpecificPostfix, options.CliPath?.FullName, options.RestorePath?.FullName))); case RuntimeMoniker.Mono: return baseJob.WithRuntime(new MonoRuntime("Mono", options.MonoPath?.FullName)); @@ -613,13 +614,20 @@ private static string GetCoreRunToolchainDisplayName(IReadOnlyList pat return coreRunPath.FullName.Substring(lastCommonDirectorySeparatorIndex); } - private static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) + private static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker, out string platformSpecificPostfix) { int index = runtime.IndexOf('-'); - return index < 0 - ? Enum.TryParse(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker) - : Enum.TryParse(runtime.Substring(0, index).Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker); + if (index < 0) + { + platformSpecificPostfix = ""; + return Enum.TryParse(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker); + } + else + { + platformSpecificPostfix = runtime.Substring(index).ToLower(); + return Enum.TryParse(runtime.Substring(0, index).Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker); + } } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/Job.cs b/src/BenchmarkDotNet/Jobs/Job.cs index 6ee47f7150..bd03ff74e9 100644 --- a/src/BenchmarkDotNet/Jobs/Job.cs +++ b/src/BenchmarkDotNet/Jobs/Job.cs @@ -14,22 +14,24 @@ public sealed class Job : JobMode [PublicAPI] public static readonly Characteristic AccuracyCharacteristic = CreateCharacteristic(nameof(Accuracy)); [PublicAPI] public static readonly Characteristic MetaCharacteristic = CreateCharacteristic(nameof(Meta)); - public static readonly Job LegacyJitX86 = new Job(nameof(LegacyJitX86), EnvironmentMode.LegacyJitX86).Freeze(); - public static readonly Job LegacyJitX64 = new Job(nameof(LegacyJitX64), EnvironmentMode.LegacyJitX64).Freeze(); - public static readonly Job RyuJitX64 = new Job(nameof(RyuJitX64), EnvironmentMode.RyuJitX64).Freeze(); - public static readonly Job RyuJitX86 = new Job(nameof(RyuJitX86), EnvironmentMode.RyuJitX86).Freeze(); + internal static readonly Characteristic ImplicitIdCharacteristic = CreateHiddenCharacteristic("ImplicitId"); + + public static readonly Job LegacyJitX86 = new Job(EnvironmentMode.LegacyJitX86).WithImplicitId(nameof(LegacyJitX86)).Freeze(); + public static readonly Job LegacyJitX64 = new Job(EnvironmentMode.LegacyJitX64).WithImplicitId(nameof(LegacyJitX64)).Freeze(); + public static readonly Job RyuJitX64 = new Job(EnvironmentMode.RyuJitX64).WithImplicitId(nameof(RyuJitX64)).Freeze(); + public static readonly Job RyuJitX86 = new Job(EnvironmentMode.RyuJitX86).WithImplicitId(nameof(RyuJitX86)).Freeze(); // Run - public static readonly Job Dry = new Job(nameof(Dry), RunMode.Dry).Freeze(); + public static readonly Job Dry = new Job(RunMode.Dry).WithImplicitId(nameof(Dry)).Freeze(); - public static readonly Job ShortRun = new Job(nameof(ShortRun), RunMode.Short).Freeze(); - public static readonly Job MediumRun = new Job(nameof(MediumRun), RunMode.Medium).Freeze(); - public static readonly Job LongRun = new Job(nameof(LongRun), RunMode.Long).Freeze(); - public static readonly Job VeryLongRun = new Job(nameof(VeryLongRun), RunMode.VeryLong).Freeze(); + public static readonly Job ShortRun = new Job(RunMode.Short).WithImplicitId(nameof(ShortRun)).Freeze(); + public static readonly Job MediumRun = new Job(RunMode.Medium).WithImplicitId(nameof(MediumRun)).Freeze(); + public static readonly Job LongRun = new Job(RunMode.Long).WithImplicitId(nameof(LongRun)).Freeze(); + public static readonly Job VeryLongRun = new Job(RunMode.VeryLong).WithImplicitId(nameof(VeryLongRun)).Freeze(); // Infrastructure - public static readonly Job InProcess = new Job(nameof(InProcess), InfrastructureMode.InProcess); - public static readonly Job InProcessDontLogOutput = new Job(nameof(InProcessDontLogOutput), InfrastructureMode.InProcessDontLogOutput); + public static readonly Job InProcess = new Job(InfrastructureMode.InProcess).WithImplicitId(nameof(InProcess)); + public static readonly Job InProcessDontLogOutput = new Job(InfrastructureMode.InProcessDontLogOutput).WithImplicitId(nameof(InProcessDontLogOutput)); public Job() : this((string)null) { } @@ -40,6 +42,7 @@ public Job(string id) : base(id) InfrastructureCharacteristic[this] = new InfrastructureMode(); AccuracyCharacteristic[this] = new AccuracyMode(); MetaCharacteristic[this] = new MetaMode(); + ImplicitIdCharacteristic[this] = false; } public Job(CharacteristicObject other) : this((string)null, other) diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index ca731ad723..0b8738773c 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -21,7 +21,10 @@ public static class JobExtensions public static Job With(this Job job, Platform platform) => job.WithPlatform(platform); public static Job WithPlatform(this Job job, Platform platform) => job.WithCore(j => j.Environment.Platform = platform); - public static Job WithId(this Job job, string id) => new Job(id, job); + public static Job WithId(this Job job, string id) => new Job(id, job).WithImplicitId(false); + + internal static Job WithImplicitId(this Job job, string id) => new Job(id, job).WithImplicitId(true); + private static Job WithImplicitId(this Job job, bool isImplicit) => job.WithCore(j => Job.ImplicitIdCharacteristic[j] = isImplicit); // Env [EditorBrowsable(EditorBrowsableState.Never)] diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index 08a1efecc9..b3b5150c37 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -36,7 +36,6 @@ public class Summary private ImmutableDictionary ReportMap { get; } private BaseliningStrategy BaseliningStrategy { get; } - private bool? isMultipleRuntimes; public Summary( string title, @@ -80,9 +79,6 @@ public Summary( public int GetNumberOfExecutedBenchmarks() => Reports.Count(report => report.ExecuteResults.Any(result => result.FoundExecutable)); - public bool IsMultipleRuntimes - => isMultipleRuntimes ??= BenchmarksCases.Length > 1 ? BenchmarksCases.Select(benchmark => benchmark.GetRuntime()).Distinct().Count() > 1 : false; - internal static Summary ValidationFailed(string title, string resultsDirectoryPath, string logFilePath, ImmutableArray? validationErrors = null) => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, validationErrors ?? ImmutableArray.Empty, ImmutableArray.Empty); diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs index be6fae3f4b..f7a7216e60 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs @@ -31,8 +31,8 @@ private CsProjClassicNetToolchain(string targetFrameworkMoniker, string name, st { } - public static IToolchain From(string targetFrameworkMoniker, string packagesPath = null) - => new CsProjClassicNetToolchain(targetFrameworkMoniker, targetFrameworkMoniker, packagesPath); + public static IToolchain From(string targetFrameworkMoniker, string name, string packagesPath = null) + => new CsProjClassicNetToolchain(targetFrameworkMoniker, name, packagesPath); public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) { diff --git a/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitToolchain.cs b/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitToolchain.cs index 8ca8f77897..89f519ea64 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitToolchain.cs @@ -23,7 +23,7 @@ public InProcessEmitToolchain(bool logOutput) : /// true if the output should be logged. public InProcessEmitToolchain(TimeSpan timeout, bool logOutput) : base( - nameof(InProcessEmitToolchain), + "InProcessEmit", new InProcessEmitGenerator(), new InProcessEmitBuilder(), new InProcessEmitExecutor(timeout, logOutput)) diff --git a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolchain.cs similarity index 100% rename from src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs rename to src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolchain.cs diff --git a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs index c2af278c76..208d6ccc5f 100644 --- a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs +++ b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs @@ -37,7 +37,7 @@ internal static IToolchain GetToolchain(this Runtime runtime, Descriptor descrip if (RuntimeInformation.IsNetCore || preferMsBuildToolchains) return clrRuntime.RuntimeMoniker != RuntimeMoniker.NotRecognized ? GetToolchain(clrRuntime.RuntimeMoniker) - : CsProjClassicNetToolchain.From(clrRuntime.MsBuildMoniker); + : CsProjClassicNetToolchain.From(clrRuntime.MsBuildMoniker, clrRuntime.Name); return RoslynToolchain.Instance; diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ExplicitIdsAreDiscovered.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ExplicitIdsAreDiscovered.approved.txt new file mode 100644 index 0000000000..3c088241cb --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ExplicitIdsAreDiscovered.approved.txt @@ -0,0 +1,13 @@ + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + Dry : extra output line + +Job=Dry IterationCount=1 LaunchCount=1 +RunStrategy=ColdStart UnrollFactor=1 WarmupCount=1 + + Method | Mean | Error | +------- |---------:|------:| + Method | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ImplicitIdsAreDiscovered.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ImplicitIdsAreDiscovered.approved.txt new file mode 100644 index 0000000000..bfe110757c --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ImplicitIdsAreDiscovered.approved.txt @@ -0,0 +1,13 @@ + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + Dry : extra output line + +IterationCount=1 LaunchCount=1 RunStrategy=ColdStart +UnrollFactor=1 WarmupCount=1 + + Method | Mean | Error | +------- |---------:|------:| + Method | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.MultipleInputJobsAreDiscovered.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.MultipleInputJobsAreDiscovered.approved.txt new file mode 100644 index 0000000000..fd2768b16f --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.MultipleInputJobsAreDiscovered.approved.txt @@ -0,0 +1,25 @@ + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + Job-rndId0 : extra output line + Job-rndId1 : extra output line + Job-rndId2 : extra output line + Dry : extra output line + Dry-.NET 6.0 : extra output line + Dry-.NET Framework 4.8.1 : extra output line + Dry-NativeAOT 6.0 : extra output line + + + Method | Runtime | IterationCount | LaunchCount | RunStrategy | UnrollFactor | WarmupCount | Mean | Error | Ratio | +------- |--------------------- |--------------- |------------ |------------ |------------- |------------ |---------:|------:|------:| + Method | .NET 7.0 | Default | Default | Default | 16 | Default | 1.000 ns | NA | 1.00 | + Method | .NET Framework 4.8.1 | Default | Default | Default | 16 | Default | 1.000 ns | NA | 1.00 | + Method | NativeAOT 7.0 | Default | Default | Default | 16 | Default | 1.000 ns | NA | 1.00 | + Method | .NET 7.0 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | + Method | .NET Framework 4.8.1 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | + Method | NativeAOT 7.0 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | + Method | .NET 6.0 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | + Method | .NET Framework 4.8.1 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | + Method | NativeAOT 6.0 | 1 | 1 | ColdStart | 1 | 1 | 1.000 ns | NA | 1.00 | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimeIsDiscovered.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimeIsDiscovered.approved.txt new file mode 100644 index 0000000000..f87a0283ae --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimeIsDiscovered.approved.txt @@ -0,0 +1,12 @@ + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + Job-rndId0 : extra output line + +Runtime=.NET 7.0 + + Method | Mean | Error | +------- |---------:|------:| + Method | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimesAreDiscovered.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimesAreDiscovered.approved.txt new file mode 100644 index 0000000000..23a72cac47 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimesAreDiscovered.approved.txt @@ -0,0 +1,13 @@ + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + Job-rndId0 : extra output line + Job-rndId1 : extra output line + + + Method | Runtime | Mean | Error | +------- |--------- |---------:|------:| + Method | .NET 6.0 | 1.000 ns | NA | + Method | .NET 7.0 | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimesWithToolchainsAreDiscovered.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimesWithToolchainsAreDiscovered.approved.txt new file mode 100644 index 0000000000..23a72cac47 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.RuntimesWithToolchainsAreDiscovered.approved.txt @@ -0,0 +1,13 @@ + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + Job-rndId0 : extra output line + Job-rndId1 : extra output line + + + Method | Runtime | Mean | Error | +------- |--------- |---------:|------:| + Method | .NET 6.0 | 1.000 ns | NA | + Method | .NET 7.0 | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ToolchainIsDiscovered.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ToolchainIsDiscovered.approved.txt new file mode 100644 index 0000000000..e4965071f9 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ToolchainIsDiscovered.approved.txt @@ -0,0 +1,12 @@ + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + InProcess : extra output line + +Toolchain=InProcessEmit + + Method | Mean | Error | +------- |---------:|------:| + Method | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ToolchainsAreDiscovered.approved.txt b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ToolchainsAreDiscovered.approved.txt new file mode 100644 index 0000000000..6348d6ff81 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/ApprovedFiles/JobColumnsApprovalTests.ToolchainsAreDiscovered.approved.txt @@ -0,0 +1,13 @@ + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + InProcess : extra output line + + + Method | Toolchain | Mean | Error | +------- |-------------- |---------:|------:| + Method | Default | 1.000 ns | NA | + Method | InProcessEmit | 1.000 ns | NA | diff --git a/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs b/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs new file mode 100644 index 0000000000..ed05d7be65 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/JobColumnsApprovalTests.cs @@ -0,0 +1,173 @@ +using System; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using System.Threading; +using ApprovalTests; +using ApprovalTests.Namers; +using ApprovalTests.Reporters; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Tests.Mocks; +using BenchmarkDotNet.Tests.XUnit; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Xunit; + +namespace BenchmarkDotNet.Tests.Columns +{ + // In case of failed approval tests, use the following reporter: + // [UseReporter(typeof(KDiffReporter))] + [UseReporter(typeof(XUnit2Reporter))] + [UseApprovalSubdirectory("ApprovedFiles")] + [Collection("ApprovalTests")] + public class JobColumnsApprovalTests : IDisposable + { + private readonly CultureInfo initCulture; + + public JobColumnsApprovalTests() => initCulture = Thread.CurrentThread.CurrentCulture; + + public void Dispose() => Thread.CurrentThread.CurrentCulture = initCulture; + + [Fact] + [MethodImpl(MethodImplOptions.NoInlining)] + public void ImplicitIdsAreDiscovered() + { + var config = DefaultConfig.Instance.AddJob(Job.Dry); + + Verify(config); + } + + [Fact] + [MethodImpl(MethodImplOptions.NoInlining)] + public void ExplicitIdsAreDiscovered() + { + var config = DefaultConfig.Instance.AddJob(Job.Dry.WithId("Dry")); + + Verify(config); + } + + [DryJob(RuntimeMoniker.Net60)] + [SimpleJob(RuntimeMoniker.Net60)] + [SimpleJob(RunStrategy.ColdStart, RuntimeMoniker.Net60, 1, 1, 1)] + public class BenchmarkWithAttributeJobs + { + [Benchmark] public void Method() { } + } + + [Fact] + public void ToolchainIsDiscovered() + { + var config = DefaultConfig.Instance.AddJob(Job.InProcess); + + Verify(config); + } + + [Fact] + public void ToolchainsAreDiscovered() + { + var config = DefaultConfig.Instance + .AddJob(Job.Default) + .AddJob(Job.InProcess); + + Verify(config); + } + + [Fact] + public void RuntimeIsDiscovered() + { + var config = DefaultConfig.Instance + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core70)); + + Verify(config); + } + + [Fact] + public void RuntimesAreDiscovered() + { + var config = DefaultConfig.Instance + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core70)) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core60)); + + Verify(config); + } + + [Fact] + public void RuntimesWithToolchainsAreDiscovered() + { + var config = DefaultConfig.Instance + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core70).WithToolchain(InProcessEmitToolchain.Instance)) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core60)); + + Verify(config); + } + + [FactDotNetCoreOnly("In the .Net Framework cmd job uses CsProjClassicNetToolchain while fluent and attribute jobs use RoslynToolchain by default")] + [MethodImpl(MethodImplOptions.NoInlining)] + public void MultipleInputJobsAreDiscovered() + { + var cmdConfig = ConfigParser.Parse( + "--join --runtimes net481 net7.0 nativeaot7.0".Split(), NullLogger.Instance).config; + + var fluentConfig = DefaultConfig.Instance + .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core70)) + .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net481)) + .AddJob(Job.Dry.WithRuntime(NativeAotRuntime.Net70)); + + var config = ManualConfig.Union(cmdConfig, fluentConfig); + + Verify(config); + } + + public class Benchmark + { + [Benchmark] public void Method() { } + } + + [DryJob(RuntimeMoniker.Net60)] + [DryJob(RuntimeMoniker.Net481)] + [DryJob(RuntimeMoniker.NativeAot60)] + public class BenchmarkWithDryJobs + { + [Benchmark] public void Method() { } + } + + private static void Verify(IConfig config) + { + Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; + + var exporter = MarkdownExporter.Mock; + var summary = MockFactory.CreateSummary(config); + + var logger = new AccumulationLogger(); + exporter.ExportToLog(summary, logger); + + var log = ReplaceRandomIDs(logger.GetLog()); + Approvals.Verify(log); + } + + private static string ReplaceRandomIDs(string log) + { + var regex = new Regex(@"Job-\w*"); + + var index = 0; + foreach (Match match in regex.Matches(log)) + { + var randomGeneratedJobName = match.Value; + + // JobIdGenerator.GenerateRandomId() generates Job-ABCDEF + // respect the length for proper table formatting + var persistantName = $"Job-rndId{index}"; + log = log.Replace(randomGeneratedJobName, persistantName); + index++; + } + + return log; + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index e3f32cede2..48e47e0f58 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -370,23 +370,54 @@ public void NetFrameworkMonikerParsedCorrectly(string tfm) } [Theory] - [InlineData("net50")] - [InlineData("net60")] - [InlineData("net70")] - [InlineData("net80")] - public void NetMonikersAreRecognizedAsNetCoreMonikers(string tfm) + [InlineData("net50", "net5.0")] + [InlineData("net60", "net6.0")] + [InlineData("net70", "net7.0")] + [InlineData("net80", "net8.0")] + public void NetMonikersAreRecognizedAsNetCoreMonikers(string input, string expected) { - var config = ConfigParser.Parse(new[] { "-r", tfm }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "-r", input }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); CsProjCoreToolchain toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; Assert.NotNull(toolchain); - Assert.Equal(tfm, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); + Assert.Equal(expected, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); + } + + [Theory] + [InlineData("net481", ".NET Framework 4.8.1")] + [InlineData("net7.0", ".NET 7.0")] + public void NetMonikersHaveNiceNames(string input, string expected) + { + // CoreRuntime. + var config = ConfigParser.Parse(new[] { "-r", input }, new OutputLogger(Output)).config; + + Assert.Single(config.GetJobs()); + var job = config.GetJobs().Single(); + Assert.NotNull(job); + Assert.Equal(expected, job.Environment.Runtime.Name); + Assert.Equal(expected, job.GetToolchain().Name); + } + + [Theory] + [InlineData("nativeaot6.0", "NativeAOT 6.0", "ILCompiler 6.0.0-*")] + [InlineData("nativeaot7.0", "NativeAOT 7.0", "Latest ILCompiler")] + public void NativeAotMonikersHaveNiceNames(string input, string runtimeName, string toolchainName) + { + // CoreRuntime. + var config = ConfigParser.Parse(new[] { "-r", input }, new OutputLogger(Output)).config; + + Assert.Single(config.GetJobs()); + var job = config.GetJobs().Single(); + Assert.NotNull(job); + Assert.Equal(runtimeName, job.Environment.Runtime.Name); + Assert.Equal(toolchainName, job.GetToolchain().Name); } [Theory] [InlineData("net5.0-windows")] [InlineData("net5.0-ios")] + [InlineData("net5.0-ios15.0")] public void PlatformSpecificMonikersAreSupported(string msBuildMoniker) { var config = ConfigParser.Parse(new[] { "-r", msBuildMoniker }, new OutputLogger(Output)).config; @@ -397,6 +428,21 @@ public void PlatformSpecificMonikersAreSupported(string msBuildMoniker) Assert.Equal(msBuildMoniker, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); } + [Theory] + [InlineData("net7.0-ios", ".NET 7.0-ios")] + [InlineData("net7.0-IOS", ".NET 7.0-ios")] + [InlineData("net7.0-ios15.0", ".NET 7.0-ios15.0")] + public void PlatformSpecificMonikersHaveNiceNames(string input, string expected) + { + var config = ConfigParser.Parse(new[] { "-r", input }, new OutputLogger(Output)).config; + + Assert.Single(config.GetJobs()); + var job = config.GetJobs().Single(); + Assert.NotNull(job); + //todo: add postfix to runtime name instead toolchain name + Assert.Equal(expected, job.GetToolchain().Name); + } + [Fact] public void CanCompareFewDifferentRuntimes() { diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.Invariant.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.Invariant.approved.txt index 50556461f8..c8d7d109bf 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.Invariant.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.Invariant.approved.txt @@ -8,8 +8,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 .... [options="header"] |=== @@ -41,8 +40,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -
Job=LongRun  IterationCount=100  LaunchCount=3  
-WarmupCount=15  
+
IterationCount=100  LaunchCount=3  WarmupCount=15  
 
@@ -407,8 +405,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 Method | Mean | Error | StdDev | P67 | ------- |---------:|------:|---------:|---------:| @@ -425,8 +422,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 {noformat} ||Method || Mean ||Error || StdDev || P67 || @@ -442,8 +438,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 | Method | Mean | Error | StdDev | P67 | |------- |---------:|------:|---------:|---------:| @@ -460,8 +455,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 ``` | Method | Mean | Error | StdDev | P67 | @@ -478,8 +472,7 @@ MarkdownExporter-stackoverflow [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line - Job=LongRun IterationCount=100 LaunchCount=3 - WarmupCount=15 + IterationCount=100 LaunchCount=3 WarmupCount=15 Method | Mean | Error | StdDev | P67 | ------- |---------:|------:|---------:|---------:| diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.en-US.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.en-US.approved.txt index 50556461f8..c8d7d109bf 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.en-US.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.en-US.approved.txt @@ -8,8 +8,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 .... [options="header"] |=== @@ -41,8 +40,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -
Job=LongRun  IterationCount=100  LaunchCount=3  
-WarmupCount=15  
+
IterationCount=100  LaunchCount=3  WarmupCount=15  
 
@@ -407,8 +405,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 Method | Mean | Error | StdDev | P67 | ------- |---------:|------:|---------:|---------:| @@ -425,8 +422,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 {noformat} ||Method || Mean ||Error || StdDev || P67 || @@ -442,8 +438,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 | Method | Mean | Error | StdDev | P67 | |------- |---------:|------:|---------:|---------:| @@ -460,8 +455,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 ``` | Method | Mean | Error | StdDev | P67 | @@ -478,8 +472,7 @@ MarkdownExporter-stackoverflow [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line - Job=LongRun IterationCount=100 LaunchCount=3 - WarmupCount=15 + IterationCount=100 LaunchCount=3 WarmupCount=15 Method | Mean | Error | StdDev | P67 | ------- |---------:|------:|---------:|---------:| diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.ru-RU.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.ru-RU.approved.txt index b28fd9dd15..6dea3c88e3 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.ru-RU.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.ru-RU.approved.txt @@ -8,8 +8,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 .... [options="header"] |=== @@ -41,8 +40,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -
Job=LongRun  IterationCount=100  LaunchCount=3  
-WarmupCount=15  
+
IterationCount=100  LaunchCount=3  WarmupCount=15  
 
@@ -407,8 +405,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 Method | Mean | Error | StdDev | P67 | ------- |---------:|------:|---------:|---------:| @@ -425,8 +422,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 {noformat} ||Method || Mean ||Error || StdDev || P67 || @@ -442,8 +438,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 | Method | Mean | Error | StdDev | P67 | |------- |---------:|------:|---------:|---------:| @@ -460,8 +455,7 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line -Job=LongRun IterationCount=100 LaunchCount=3 -WarmupCount=15 +IterationCount=100 LaunchCount=3 WarmupCount=15 ``` | Method | Mean | Error | StdDev | P67 | @@ -478,8 +472,7 @@ MarkdownExporter-stackoverflow [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION LongRun : extra output line - Job=LongRun IterationCount=100 LaunchCount=3 - WarmupCount=15 + IterationCount=100 LaunchCount=3 WarmupCount=15 Method | Mean | Error | StdDev | P67 | ------- |---------:|------:|---------:|---------:| diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs index f1393f8b62..5cb9e695e1 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs @@ -32,9 +32,12 @@ public static Summary CreateSummary(Type benchmarkType) ImmutableArray.Empty); } - public static Summary CreateSummary(IConfig config) => new Summary( + public static Summary CreateSummary(IConfig config) + => CreateSummary(config); + + public static Summary CreateSummary(IConfig config) => new Summary( "MockSummary", - CreateReports(config), + CreateReports(config), new HostEnvironmentInfoBuilder().WithoutDotNetSdkVersion().Build(), string.Empty, string.Empty, @@ -57,8 +60,8 @@ public static Summary CreateSummary(IConfig config, bool hugeSd, Metric[] metric ImmutableArray.Empty, ImmutableArray.Empty); - private static ImmutableArray CreateReports(IConfig config) - => CreateBenchmarks(config).Select(CreateSimpleReport).ToImmutableArray(); + private static ImmutableArray CreateReports(IConfig config) + => CreateBenchmarks(config).Select(CreateSimpleReport).ToImmutableArray(); private static BenchmarkCase[] CreateBenchmarks(IConfig config) => BenchmarkConverter.TypeToBenchmarks(typeof(TBenchmarks), config).BenchmarksCases;