From 0558dc2720c002951fe7308f27f6b9b9de33e1e6 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Tue, 18 Oct 2022 12:13:49 +0300 Subject: [PATCH 1/9] mock factory --- tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs index 813d25f4d1..85eecda7d4 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs @@ -32,6 +32,21 @@ public static Summary CreateSummary(Type benchmarkType) ImmutableArray.Empty); } + public static Summary CreateSummary(IConfig config, params Type[] benchmarkTypes) + { + var runInfos = benchmarkTypes.Select(type => BenchmarkConverter.TypeToBenchmarks(type, config)); + return new Summary( + "MockSummary", + runInfos.SelectMany(i => i.BenchmarksCases.Select((benchmark, index) => CreateReport(benchmark, 5, (index + 1) * 100))).ToImmutableArray(), + new HostEnvironmentInfoBuilder().WithoutDotNetSdkVersion().Build(), + string.Empty, + string.Empty, + TimeSpan.FromMinutes(1), + TestCultureInfo.Instance, + ImmutableArray.Empty, + ImmutableArray.Empty); + } + public static Summary CreateSummary(IConfig config) => new Summary( "MockSummary", CreateReports(config), From 2025e8700554d828802a57cd7ea8b4c850daa580 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Tue, 18 Oct 2022 12:14:14 +0300 Subject: [PATCH 2/9] Add new tests --- ...kdownExporterMultipleClassApprovalTests.cs | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterMultipleClassApprovalTests.cs diff --git a/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterMultipleClassApprovalTests.cs b/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterMultipleClassApprovalTests.cs new file mode 100644 index 0000000000..7bd0358a7f --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterMultipleClassApprovalTests.cs @@ -0,0 +1,123 @@ +using System; +using System.Globalization; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using ApprovalTests; +using ApprovalTests.Namers; +using ApprovalTests.Reporters; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Tests.Mocks; +using BenchmarkDotNet.Validators; +using JetBrains.Annotations; +using Xunit; + +namespace BenchmarkDotNet.Tests.Exporters +{ + [UseReporter(typeof(XUnit2Reporter))] + [UseApprovalSubdirectory("ApprovedFiles")] + [Collection("ApprovalTests")] + public class MarkdownExporterMultipleClassApprovalTests : IDisposable + { + private readonly CultureInfo initCulture; + + public MarkdownExporterMultipleClassApprovalTests() => initCulture = Thread.CurrentThread.CurrentCulture; + + [UsedImplicitly] + public static TheoryData GetLogicalGroupRules() + { + var data = new TheoryData(); + + foreach (var type in typeof(LogicalGroupBenchmarks).GetNestedTypes()) + { + data.Add(type, null); // no logical rule + + foreach (var rule in Enum.GetValues(typeof(BenchmarkLogicalGroupRule))) + data.Add(type, (BenchmarkLogicalGroupRule?)rule); + } + + return data; + } + + [Theory] + [MemberData(nameof(GetLogicalGroupRules))] + [MethodImpl(MethodImplOptions.NoInlining)] + public void LogicalRuleTest(Type benchmarkType, BenchmarkLogicalGroupRule? rule) + { + var config = DefaultConfig.Instance; + if (rule is { } logicalRule) + config = config.AddLogicalGroupRules(logicalRule); + + var fileName = rule == null + ? $"{benchmarkType.Name}.Default" + : $"{benchmarkType.Name}.{rule}"; + + NamerFactory.AdditionalInformation = fileName; + Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; + + var logger = new AccumulationLogger(); + logger.WriteLine($"=== " + fileName + " ==="); + + var exporter = MarkdownExporter.Mock; + var summary = MockFactory.CreateSummary(config, benchmarkType.GetNestedTypes()); + exporter.ExportToLog(summary, logger); + + var validator = BaselineValidator.FailOnError; + var errors = validator.Validate(new ValidationParameters(summary.BenchmarksCases, summary.BenchmarksCases.First().Config)).ToList(); + logger.WriteLine(); + logger.WriteLine("Errors: " + errors.Count); + foreach (var error in errors) + logger.WriteLineError("* " + error.Message); + + Approvals.Verify(logger.GetLog()); + } + + public void Dispose() => Thread.CurrentThread.CurrentCulture = initCulture; + + public static class LogicalGroupBenchmarks + { + public static class NoBaseline + { + [LogicalGroupColumn, CategoriesColumn, BaselineColumn] + [BenchmarkCategory("A")] + public class Bench1 + { + [Params(10, 20)] public int Param; + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [BenchmarkCategory("B")] + public class Bench2 + { + [Params(10, 20)] public int Param; + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + } + + public static class OneBaseline + { + [LogicalGroupColumn, CategoriesColumn, BaselineColumn] + [BenchmarkCategory("A")] + public class Bench1 + { + [Params(10, 20)] public int Param; + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [BenchmarkCategory("B")] + public class Bench2 + { + [Params(10, 20)] public int Param; + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + } + } + } +} \ No newline at end of file From b1d588ee014d27ef327da445cac34cd4c5f5e07d Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Thu, 20 Oct 2022 01:57:08 +0300 Subject: [PATCH 3/9] Add approved files --- ...uleTest.NoBaseline.ByCategory.approved.txt | 22 +++++++++++++++ ...icalRuleTest.NoBaseline.ByJob.approved.txt | 21 ++++++++++++++ ...lRuleTest.NoBaseline.ByMethod.approved.txt | 24 ++++++++++++++++ ...lRuleTest.NoBaseline.ByParams.approved.txt | 22 +++++++++++++++ ...alRuleTest.NoBaseline.Default.approved.txt | 21 ++++++++++++++ ...leTest.OneBaseline.ByCategory.approved.txt | 24 ++++++++++++++++ ...calRuleTest.OneBaseline.ByJob.approved.txt | 22 +++++++++++++++ ...RuleTest.OneBaseline.ByMethod.approved.txt | 28 +++++++++++++++++++ ...RuleTest.OneBaseline.ByParams.approved.txt | 22 +++++++++++++++ ...lRuleTest.OneBaseline.Default.approved.txt | 22 +++++++++++++++ 10 files changed, 228 insertions(+) create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByCategory.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByJob.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByMethod.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByParams.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.Default.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByCategory.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByJob.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByMethod.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByParams.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.Default.approved.txt diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByCategory.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByCategory.approved.txt new file mode 100644 index 0000000000..022d6cb928 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByCategory.approved.txt @@ -0,0 +1,22 @@ +=== NoBaseline.ByCategory === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Type | Method | Categories | Param | Mean | Error | StdDev | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | A | No | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | A | No | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | A | No | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | A | No | + | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | B | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | B | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | B | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | B | No | + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByJob.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByJob.approved.txt new file mode 100644 index 0000000000..8a0e33e0b4 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByJob.approved.txt @@ -0,0 +1,21 @@ +=== NoBaseline.ByJob === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Type | Method | Categories | Param | Mean | Error | StdDev | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByMethod.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByMethod.approved.txt new file mode 100644 index 0000000000..fe1f7cc842 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByMethod.approved.txt @@ -0,0 +1,24 @@ +=== NoBaseline.ByMethod === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Type | Method | Categories | Param | Mean | Error | StdDev | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | Bench1.Foo | No | ^ + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | Bench1.Foo | No | ^ + | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | Bench2.Foo | No | ^ + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | Bench2.Foo | No | ^ + | | | | | | | | | + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | Bench1.Bar | No | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | Bench1.Bar | No | ^ + | | | | | | | | | + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | Bench2.Bar | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | Bench2.Bar | No | ^ + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByParams.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByParams.approved.txt new file mode 100644 index 0000000000..91b1b1da01 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByParams.approved.txt @@ -0,0 +1,22 @@ +=== NoBaseline.ByParams === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Type | Method | Categories | Param | Mean | Error | StdDev | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | [Param=10] | No | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | [Param=10] | No | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | [Param=10] | No | + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | [Param=10] | No | + | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | [Param=20] | No | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | [Param=20] | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | [Param=20] | No | + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | [Param=20] | No | + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.Default.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.Default.approved.txt new file mode 100644 index 0000000000..5addd7aa65 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.Default.approved.txt @@ -0,0 +1,21 @@ +=== NoBaseline.Default === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Type | Method | Categories | Param | Mean | Error | StdDev | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | * | No | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | * | No | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | * | No | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | * | No | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | * | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | * | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | * | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | * | No | + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByCategory.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByCategory.approved.txt new file mode 100644 index 0000000000..83ee167c3c --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByCategory.approved.txt @@ -0,0 +1,24 @@ +=== OneBaseline.ByCategory === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|------------------------ |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | A-[Param=10]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | A-[Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | A-[Param=20]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | A-[Param=20]-DefaultJob | No | + | | | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-[Param=10]-DefaultJob | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-[Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-[Param=20]-DefaultJob | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-[Param=20]-DefaultJob | No | + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByJob.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByJob.approved.txt new file mode 100644 index 0000000000..cf806a3835 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByJob.approved.txt @@ -0,0 +1,22 @@ +=== OneBaseline.ByJob === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|---------------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-[Param=10] | Yes | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | DefaultJob-[Param=10] | No | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-[Param=10] | No | + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | DefaultJob-[Param=10] | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-[Param=20] | Yes | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | DefaultJob-[Param=20] | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-[Param=20] | No | + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | DefaultJob-[Param=20] | No | + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByMethod.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByMethod.approved.txt new file mode 100644 index 0000000000..4771f8b1f9 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByMethod.approved.txt @@ -0,0 +1,28 @@ +=== OneBaseline.ByMethod === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|--------------------------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Bench1.Foo-[Param=10]-DefaultJob | Yes | ^ + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Bench1.Foo-[Param=20]-DefaultJob | Yes | ^ + | | | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2.Foo-[Param=10]-DefaultJob | No | ^ + | | | | | | | | | | | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2.Foo-[Param=20]-DefaultJob | No | ^ + | | | | | | | | | | | + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench1.Bar-[Param=10]-DefaultJob | No | ^ + | | | | | | | | | | | + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench1.Bar-[Param=20]-DefaultJob | No | ^ + | | | | | | | | | | | + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2.Bar-[Param=10]-DefaultJob | No | ^ + | | | | | | | | | | | + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2.Bar-[Param=20]-DefaultJob | No | ^ + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByParams.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByParams.approved.txt new file mode 100644 index 0000000000..5fe4d84070 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByParams.approved.txt @@ -0,0 +1,22 @@ +=== OneBaseline.ByParams === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|---------------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-DefaultJob | No | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-DefaultJob | No | + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-DefaultJob | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-DefaultJob | No | + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-DefaultJob | No | + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.Default.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.Default.approved.txt new file mode 100644 index 0000000000..d94900d436 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.Default.approved.txt @@ -0,0 +1,22 @@ +=== OneBaseline.Default === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|---------------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-DefaultJob | No | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-DefaultJob | No | + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-DefaultJob | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-DefaultJob | No | + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-DefaultJob | No | + +Errors: 0 From 86bb98d80553853d482137ca610f6a59a4c62ee8 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Tue, 18 Oct 2022 17:28:34 +0300 Subject: [PATCH 4/9] Add ByType rule --- .../Configs/BenchmarkLogicalGroupRule.cs | 2 +- src/BenchmarkDotNet/Order/DefaultOrderer.cs | 35 ++++++++++++++----- .../Running/DescriptorComparer.cs | 10 +++--- src/BenchmarkDotNet/Running/TypeComparer.cs | 17 +++++++++ 4 files changed, 49 insertions(+), 15 deletions(-) create mode 100644 src/BenchmarkDotNet/Running/TypeComparer.cs diff --git a/src/BenchmarkDotNet/Configs/BenchmarkLogicalGroupRule.cs b/src/BenchmarkDotNet/Configs/BenchmarkLogicalGroupRule.cs index 3ad78af28b..154604b361 100644 --- a/src/BenchmarkDotNet/Configs/BenchmarkLogicalGroupRule.cs +++ b/src/BenchmarkDotNet/Configs/BenchmarkLogicalGroupRule.cs @@ -2,6 +2,6 @@ { public enum BenchmarkLogicalGroupRule { - ByMethod, ByJob, ByParams, ByCategory + ByMethod, ByJob, ByParams, ByCategory, ByType } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Order/DefaultOrderer.cs b/src/BenchmarkDotNet/Order/DefaultOrderer.cs index 0bca3cf3ec..9cbcaee3e6 100644 --- a/src/BenchmarkDotNet/Order/DefaultOrderer.cs +++ b/src/BenchmarkDotNet/Order/DefaultOrderer.cs @@ -4,6 +4,7 @@ using System.Diagnostics.CodeAnalysis; using System.Linq; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Parameters; using BenchmarkDotNet.Reports; @@ -20,7 +21,8 @@ public class DefaultOrderer : IOrderer private readonly IComparer categoryComparer = CategoryComparer.Instance; private readonly IComparer paramsComparer = ParameterComparer.Instance; private readonly IComparer jobComparer = JobComparer.Instance; - private readonly IComparer targetComparer; + private readonly IComparer typeComparer = TypeComparer.Instance; + private readonly IComparer methodComparer; public SummaryOrderPolicy SummaryOrderPolicy { get; } public MethodOrderPolicy MethodOrderPolicy { get; } @@ -31,7 +33,7 @@ public DefaultOrderer( { SummaryOrderPolicy = summaryOrderPolicy; MethodOrderPolicy = methodOrderPolicy; - targetComparer = new DescriptorComparer(methodOrderPolicy); + methodComparer = new MethodComparer(methodOrderPolicy); } [PublicAPI] @@ -39,7 +41,7 @@ public virtual IEnumerable GetExecutionOrder( ImmutableArray benchmarkCases, IEnumerable order = null) { - var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, targetComparer, order); + var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, typeComparer, methodComparer, order); var list = benchmarkCases.ToList(); list.Sort(benchmarkComparer); return list; @@ -87,8 +89,15 @@ public string GetLogicalGroupKey(ImmutableArray allBenchmarksCase { var explicitRules = benchmarkCase.Config.GetLogicalGroupRules().ToList(); var implicitRules = new List(); + + bool hasMultipleTypes = allBenchmarksCases.DistinctBy(b => b.Descriptor.Type).Count() > 1; bool hasJobBaselines = allBenchmarksCases.Any(b => b.Job.Meta.Baseline); bool hasDescriptorBaselines = allBenchmarksCases.Any(b => b.Descriptor.Baseline); + + if (hasMultipleTypes) + { + implicitRules.Add(BenchmarkLogicalGroupRule.ByType); + } if (hasJobBaselines) { implicitRules.Add(BenchmarkLogicalGroupRule.ByParams); @@ -114,8 +123,11 @@ public string GetLogicalGroupKey(ImmutableArray allBenchmarksCase { switch (rule) { + case BenchmarkLogicalGroupRule.ByType: + keys.Add(benchmarkCase.Descriptor.TypeInfo); + break; case BenchmarkLogicalGroupRule.ByMethod: - keys.Add(benchmarkCase.Descriptor.DisplayInfo); + keys.Add(benchmarkCase.Descriptor.WorkloadMethodDisplayInfo); break; case BenchmarkLogicalGroupRule.ByJob: keys.Add(benchmarkCase.Job.DisplayInfo); @@ -139,7 +151,7 @@ public virtual IEnumerable> GetLogicalGroupOrde IEnumerable> logicalGroups, IEnumerable order = null) { - var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, targetComparer, order); + var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, typeComparer, methodComparer, order); var logicalGroupComparer = new LogicalGroupComparer(benchmarkComparer); var list = logicalGroups.ToList(); list.Sort(logicalGroupComparer); @@ -152,6 +164,7 @@ private class BenchmarkComparer : IComparer { private static readonly BenchmarkLogicalGroupRule[] DefaultOrder = { + BenchmarkLogicalGroupRule.ByType, BenchmarkLogicalGroupRule.ByCategory, BenchmarkLogicalGroupRule.ByParams, BenchmarkLogicalGroupRule.ByJob, @@ -161,18 +174,21 @@ private class BenchmarkComparer : IComparer private readonly IComparer categoryComparer; private readonly IComparer paramsComparer; private readonly IComparer jobComparer; - private readonly IComparer targetComparer; + private readonly IComparer typeComparer; + private readonly IComparer methodComparer; private readonly List order; public BenchmarkComparer( IComparer categoryComparer, IComparer paramsComparer, IComparer jobComparer, - IComparer targetComparer, + IComparer typeComparer, + IComparer methodComparer, IEnumerable order) { this.categoryComparer = categoryComparer; - this.targetComparer = targetComparer; + this.typeComparer = typeComparer; + this.methodComparer = methodComparer; this.jobComparer = jobComparer; this.paramsComparer = paramsComparer; @@ -192,7 +208,8 @@ public int Compare(BenchmarkCase x, BenchmarkCase y) { int compare = rule switch { - BenchmarkLogicalGroupRule.ByMethod => targetComparer?.Compare(x.Descriptor, y.Descriptor) ?? 0, + BenchmarkLogicalGroupRule.ByType => typeComparer?.Compare(x.Descriptor, y.Descriptor) ?? 0, + BenchmarkLogicalGroupRule.ByMethod => methodComparer?.Compare(x.Descriptor, y.Descriptor) ?? 0, BenchmarkLogicalGroupRule.ByJob => jobComparer?.Compare(x.Job, y.Job) ?? 0, BenchmarkLogicalGroupRule.ByParams => paramsComparer?.Compare(x.Parameters, y.Parameters) ?? 0, BenchmarkLogicalGroupRule.ByCategory => categoryComparer?.Compare(x.Descriptor.Categories, y.Descriptor.Categories) ?? 0, diff --git a/src/BenchmarkDotNet/Running/DescriptorComparer.cs b/src/BenchmarkDotNet/Running/DescriptorComparer.cs index 8d35bba769..839878c532 100644 --- a/src/BenchmarkDotNet/Running/DescriptorComparer.cs +++ b/src/BenchmarkDotNet/Running/DescriptorComparer.cs @@ -5,14 +5,14 @@ namespace BenchmarkDotNet.Running { - internal class DescriptorComparer : IComparer + internal class MethodComparer : IComparer { - [PublicAPI] public static readonly IComparer Alphabetical = new DescriptorComparer(MethodOrderPolicy.Alphabetical); - [PublicAPI] public static readonly IComparer Declared = new DescriptorComparer(MethodOrderPolicy.Declared); + [PublicAPI] public static readonly IComparer Alphabetical = new MethodComparer(MethodOrderPolicy.Alphabetical); + [PublicAPI] public static readonly IComparer Declared = new MethodComparer(MethodOrderPolicy.Declared); private readonly MethodOrderPolicy methodOrderPolicy; - public DescriptorComparer(MethodOrderPolicy methodOrderPolicy) + public MethodComparer(MethodOrderPolicy methodOrderPolicy) { this.methodOrderPolicy = methodOrderPolicy; } @@ -25,7 +25,7 @@ public int Compare(Descriptor x, Descriptor y) switch (methodOrderPolicy) { case MethodOrderPolicy.Alphabetical: - return string.CompareOrdinal(x.DisplayInfo, y.DisplayInfo); + return string.CompareOrdinal(x.WorkloadMethodDisplayInfo, y.WorkloadMethodDisplayInfo); case MethodOrderPolicy.Declared: return x.MethodIndex - y.MethodIndex; default: diff --git a/src/BenchmarkDotNet/Running/TypeComparer.cs b/src/BenchmarkDotNet/Running/TypeComparer.cs new file mode 100644 index 0000000000..e32ef174fb --- /dev/null +++ b/src/BenchmarkDotNet/Running/TypeComparer.cs @@ -0,0 +1,17 @@ +using System.Collections.Generic; + +namespace BenchmarkDotNet.Running +{ + internal class TypeComparer : IComparer + { + public static readonly IComparer Instance = new TypeComparer(); + + public int Compare(Descriptor x, Descriptor y) + { + if (x == null && y == null) return 0; + if (x != null && y == null) return 1; + if (x == null) return -1; + return string.CompareOrdinal(x.TypeInfo, y.TypeInfo); + } + } +} \ No newline at end of file From bf9a59170e0f01d7b015ca2e66e930e088f752f5 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Thu, 20 Oct 2022 14:06:44 +0300 Subject: [PATCH 5/9] Update existing approved files --- ...rTest.Invalid_TwoJobBaselines.approved.txt | 18 ++--- ...rTest.JobBaseline_MethodsJobs.approved.txt | 20 +++--- ...JobBaseline_MethodsParamsJobs.approved.txt | 38 +++++------ ..._MethodsParamsJobs_GroupByAll.approved.txt | 66 +++++++++---------- ...thodsParamsJobs_GroupByMethod.approved.txt | 32 ++++----- 5 files changed, 87 insertions(+), 87 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.Invalid_TwoJobBaselines.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.Invalid_TwoJobBaselines.approved.txt index 5bd0d4d330..b3e888e2e2 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.Invalid_TwoJobBaselines.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.Invalid_TwoJobBaselines.approved.txt @@ -8,14 +8,14 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Job2 : extra output line - Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |---------:|--------:|--------:|------:|--------:|-----:|---------------------------- |--------- | - Foo | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Invalid_TwoJobBaselines.Foo | Yes | - Foo | Job2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 2 | Invalid_TwoJobBaselines.Foo | Yes | - | | | | | | | | | | - Bar | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Invalid_TwoJobBaselines.Bar | Yes | - Bar | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | Invalid_TwoJobBaselines.Bar | Yes | + Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | + Foo | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Foo | Yes | + Foo | Job2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 2 | Foo | Yes | + | | | | | | | | | | + Bar | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Bar | Yes | + Bar | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | Bar | Yes | Errors: 2 -* Only 1 job in a group can have "Baseline = true" applied to it, group Invalid_TwoJobBaselines.Foo in class Invalid_TwoJobBaselines has 2 -* Only 1 job in a group can have "Baseline = true" applied to it, group Invalid_TwoJobBaselines.Bar in class Invalid_TwoJobBaselines has 2 +* Only 1 job in a group can have "Baseline = true" applied to it, group Foo in class Invalid_TwoJobBaselines has 2 +* Only 1 job in a group can have "Baseline = true" applied to it, group Bar in class Invalid_TwoJobBaselines has 2 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsJobs.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsJobs.approved.txt index aa8444a3d1..6b192cb15b 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsJobs.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsJobs.approved.txt @@ -8,15 +8,15 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Job2 : extra output line - Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |---------:|--------:|--------:|------:|--------:|-----:|----------------------------- |--------- | - Base | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsJobs.Base | Yes | - Base | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | JobBaseline_MethodsJobs.Base | No | - | | | | | | | | | | - Foo | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsJobs.Foo | Yes | - Foo | Job2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | JobBaseline_MethodsJobs.Foo | No | - | | | | | | | | | | - Bar | Job1 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsJobs.Bar | Yes | - Bar | Job2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | JobBaseline_MethodsJobs.Bar | No | + Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | + Base | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Base | Yes | + Base | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | Base | No | + | | | | | | | | | | + Foo | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Foo | Yes | + Foo | Job2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | Foo | No | + | | | | | | | | | | + Bar | Job1 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Bar | Yes | + Bar | Job2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | Bar | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsParamsJobs.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsParamsJobs.approved.txt index 136b7d77b3..062576fada 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsParamsJobs.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsParamsJobs.approved.txt @@ -8,24 +8,24 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------- |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ - Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | No | - | | | | | | | | | | | - Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | Yes | - Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | No | - | | | | | | | | | | | - Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | Yes | - Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | No | - | | | | | | | | | | | - Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ - Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.43 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | No | - | | | | | | | | | | | - Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | Yes | - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.37 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | No | - | | | | | | | | | | | - Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | Yes | - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | No | + Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------- |--------- | + Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Base | Yes | ^ + Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | [Param=2]-Base | No | + | | | | | | | | | | | + Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Foo | Yes | + Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | [Param=2]-Foo | No | + | | | | | | | | | | | + Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Bar | Yes | + Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | [Param=2]-Bar | No | + | | | | | | | | | | | + Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Base | Yes | ^ + Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.43 | 0.00 | 2 | [Param=10]-Base | No | + | | | | | | | | | | | + Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Foo | Yes | + Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.37 | 0.00 | 2 | [Param=10]-Foo | No | + | | | | | | | | | | | + Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Bar | Yes | + Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | [Param=10]-Bar | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByAll.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByAll.approved.txt index ba6ec2fa2c..4ea33e63e6 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByAll.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByAll.approved.txt @@ -8,38 +8,38 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------------------------- |--------- | - A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=2]-CatA | Yes | ^ - | | | | | | | | | | | - A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=10]-CatA | Yes | ^ - | | | | | | | | | | | - A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=2]-CatA | Yes | ^ - | | | | | | | | | | | - A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=10]-CatA | Yes | ^ - | | | | | | | | | | | - A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=2]-CatA | No | ^ - | | | | | | | | | | | - A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=10]-CatA | No | ^ - | | | | | | | | | | | - A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=2]-CatA | No | ^ - | | | | | | | | | | | - A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=10]-CatA | No | ^ - | | | | | | | | | | | - B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=2]-CatB | Yes | ^ - | | | | | | | | | | | - B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=10]-CatB | Yes | ^ - | | | | | | | | | | | - B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=2]-CatB | Yes | ^ - | | | | | | | | | | | - B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=10]-CatB | Yes | ^ - | | | | | | | | | | | - B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=2]-CatB | No | ^ - | | | | | | | | | | | - B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=10]-CatB | No | ^ - | | | | | | | | | | | - B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=2]-CatB | No | ^ - | | | | | | | | | | | - B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=10]-CatB | No | ^ + Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|------------------------ |--------- | + A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | A1-Job1-[Param=2]-CatA | Yes | ^ + | | | | | | | | | | | + A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | A1-Job1-[Param=10]-CatA | Yes | ^ + | | | | | | | | | | | + A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | A1-Job2-[Param=2]-CatA | Yes | ^ + | | | | | | | | | | | + A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | A1-Job2-[Param=10]-CatA | Yes | ^ + | | | | | | | | | | | + A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | A2-Job1-[Param=2]-CatA | No | ^ + | | | | | | | | | | | + A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | A2-Job1-[Param=10]-CatA | No | ^ + | | | | | | | | | | | + A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | A2-Job2-[Param=2]-CatA | No | ^ + | | | | | | | | | | | + A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | A2-Job2-[Param=10]-CatA | No | ^ + | | | | | | | | | | | + B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | B1-Job1-[Param=2]-CatB | Yes | ^ + | | | | | | | | | | | + B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | B1-Job1-[Param=10]-CatB | Yes | ^ + | | | | | | | | | | | + B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | B1-Job2-[Param=2]-CatB | Yes | ^ + | | | | | | | | | | | + B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | B1-Job2-[Param=10]-CatB | Yes | ^ + | | | | | | | | | | | + B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | B2-Job1-[Param=2]-CatB | No | ^ + | | | | | | | | | | | + B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | B2-Job1-[Param=10]-CatB | No | ^ + | | | | | | | | | | | + B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | B2-Job2-[Param=2]-CatB | No | ^ + | | | | | | | | | | | + B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | B2-Job2-[Param=10]-CatB | No | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByMethod.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByMethod.approved.txt index ca9783e827..90c61c9ce3 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByMethod.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByMethod.approved.txt @@ -8,21 +8,21 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | -------- |----- |------ |-----------:|--------:|--------:|-----:|------------------------------------------------ |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | ^ - Base | Job2 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | - Base | Job1 | 10 | 302.0 ns | 6.09 ns | 1.58 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | ^ - Base | Job2 | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | - | | | | | | | | | - Foo | Job1 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | ^ - Foo | Job2 | 2 | 702.0 ns | 6.09 ns | 1.58 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | - Foo | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | ^ - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | - | | | | | | | | | - Bar | Job1 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | ^ - Bar | Job2 | 2 | 802.0 ns | 6.09 ns | 1.58 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | - Bar | Job1 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | ^ - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | + Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | +------- |----- |------ |-----------:|--------:|--------:|-----:|------------- |--------- | + Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1 | Base | No | ^ + Base | Job2 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 2 | Base | No | + Base | Job1 | 10 | 302.0 ns | 6.09 ns | 1.58 ns | 3 | Base | No | ^ + Base | Job2 | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 4 | Base | No | + | | | | | | | | | + Foo | Job1 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 1 | Foo | No | ^ + Foo | Job2 | 2 | 702.0 ns | 6.09 ns | 1.58 ns | 2 | Foo | No | + Foo | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 3 | Foo | No | ^ + Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 4 | Foo | No | + | | | | | | | | | + Bar | Job1 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1 | Bar | No | ^ + Bar | Job2 | 2 | 802.0 ns | 6.09 ns | 1.58 ns | 2 | Bar | No | + Bar | Job1 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 3 | Bar | No | ^ + Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 4 | Bar | No | Errors: 0 From 16a74d972ef1ee796ec67abc5c6fc38c5b524bc3 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Thu, 20 Oct 2022 14:09:29 +0300 Subject: [PATCH 6/9] Update new approved files --- ...uleTest.NoBaseline.ByCategory.approved.txt | 16 ++++++------ ...icalRuleTest.NoBaseline.ByJob.approved.txt | 21 ++++++++------- ...lRuleTest.NoBaseline.ByMethod.approved.txt | 16 ++++++------ ...lRuleTest.NoBaseline.ByParams.approved.txt | 24 +++++++++-------- ...alRuleTest.NoBaseline.Default.approved.txt | 17 ++++++------ ...leTest.OneBaseline.ByCategory.approved.txt | 26 +++++++++---------- ...calRuleTest.OneBaseline.ByJob.approved.txt | 24 +++++++++-------- ...RuleTest.OneBaseline.ByMethod.approved.txt | 16 ++++++------ ...RuleTest.OneBaseline.ByParams.approved.txt | 24 +++++++++-------- ...lRuleTest.OneBaseline.Default.approved.txt | 24 +++++++++-------- 10 files changed, 109 insertions(+), 99 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByCategory.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByCategory.approved.txt index 022d6cb928..eae3a21b9d 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByCategory.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByCategory.approved.txt @@ -9,14 +9,14 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Type | Method | Categories | Param | Mean | Error | StdDev | LogicalGroup | Baseline | ------- |------- |----------- |------ |---------:|--------:|--------:|------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | A | No | ^ - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | A | No | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | A | No | ^ - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | A | No | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | A-Bench1 | No | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | A-Bench1 | No | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | A-Bench1 | No | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | A-Bench1 | No | | | | | | | | | | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | B | No | ^ - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | B | No | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | B | No | ^ - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | B | No | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | B-Bench2 | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | B-Bench2 | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | B-Bench2 | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | B-Bench2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByJob.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByJob.approved.txt index 8a0e33e0b4..319c9e38cc 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByJob.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByJob.approved.txt @@ -7,15 +7,16 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC DefaultJob : extra output line - Type | Method | Categories | Param | Mean | Error | StdDev | LogicalGroup | Baseline | -------- |------- |----------- |------ |---------:|--------:|--------:|------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | ^ - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | ^ - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | ^ - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | ^ - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | DefaultJob | No | + Type | Method | Categories | Param | Mean | Error | StdDev | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------------------ |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | DefaultJob-Bench1 | No | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | DefaultJob-Bench1 | No | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | DefaultJob-Bench1 | No | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | DefaultJob-Bench1 | No | + | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | DefaultJob-Bench2 | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | DefaultJob-Bench2 | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | DefaultJob-Bench2 | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | DefaultJob-Bench2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByMethod.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByMethod.approved.txt index fe1f7cc842..47dd6012fc 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByMethod.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByMethod.approved.txt @@ -9,16 +9,16 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Type | Method | Categories | Param | Mean | Error | StdDev | LogicalGroup | Baseline | ------- |------- |----------- |------ |---------:|--------:|--------:|------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | Bench1.Foo | No | ^ - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | Bench1.Foo | No | ^ + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | Foo-Bench1 | No | ^ + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | Foo-Bench1 | No | ^ | | | | | | | | | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | Bench2.Foo | No | ^ - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | Bench2.Foo | No | ^ + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | Foo-Bench2 | No | ^ + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | Foo-Bench2 | No | ^ | | | | | | | | | - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | Bench1.Bar | No | ^ - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | Bench1.Bar | No | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | Bar-Bench1 | No | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | Bar-Bench1 | No | ^ | | | | | | | | | - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | Bench2.Bar | No | ^ - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | Bench2.Bar | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | Bar-Bench2 | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | Bar-Bench2 | No | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByParams.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByParams.approved.txt index 91b1b1da01..f592c701c6 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByParams.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByParams.approved.txt @@ -7,16 +7,18 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC DefaultJob : extra output line - Type | Method | Categories | Param | Mean | Error | StdDev | LogicalGroup | Baseline | -------- |------- |----------- |------ |---------:|--------:|--------:|------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | [Param=10] | No | ^ - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | [Param=10] | No | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | [Param=10] | No | - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | [Param=10] | No | - | | | | | | | | | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | [Param=20] | No | ^ - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | [Param=20] | No | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | [Param=20] | No | - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | [Param=20] | No | + Type | Method | Categories | Param | Mean | Error | StdDev | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------------------ |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | [Param=10]-Bench1 | No | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | [Param=10]-Bench1 | No | + | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | [Param=10]-Bench2 | No | + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | [Param=10]-Bench2 | No | + | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | [Param=20]-Bench1 | No | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | [Param=20]-Bench1 | No | + | | | | | | | | | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | [Param=20]-Bench2 | No | + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | [Param=20]-Bench2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.Default.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.Default.approved.txt index 5addd7aa65..2a85cf4fa2 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.Default.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.Default.approved.txt @@ -9,13 +9,14 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Type | Method | Categories | Param | Mean | Error | StdDev | LogicalGroup | Baseline | ------- |------- |----------- |------ |---------:|--------:|--------:|------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | * | No | ^ - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | * | No | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | * | No | ^ - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | * | No | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | * | No | ^ - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | * | No | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | * | No | ^ - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | * | No | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | Bench1 | No | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | Bench1 | No | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | Bench1 | No | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | Bench1 | No | + | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | Bench2 | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | Bench2 | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | Bench2 | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | Bench2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByCategory.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByCategory.approved.txt index 83ee167c3c..4d06c947b9 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByCategory.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByCategory.approved.txt @@ -7,18 +7,18 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC DefaultJob : extra output line - Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | -------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|------------------------ |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | A-[Param=10]-DefaultJob | Yes | ^ - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | A-[Param=10]-DefaultJob | No | - | | | | | | | | | | | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | A-[Param=20]-DefaultJob | Yes | ^ - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | A-[Param=20]-DefaultJob | No | - | | | | | | | | | | | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-[Param=10]-DefaultJob | No | ^ - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-[Param=10]-DefaultJob | No | - | | | | | | | | | | | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-[Param=20]-DefaultJob | No | ^ - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-[Param=20]-DefaultJob | No | + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|------------------------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | A-Bench1-[Param=10]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | A-Bench1-[Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | A-Bench1-[Param=20]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | A-Bench1-[Param=20]-DefaultJob | No | + | | | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-Bench2-[Param=10]-DefaultJob | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-Bench2-[Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-Bench2-[Param=20]-DefaultJob | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-Bench2-[Param=20]-DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByJob.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByJob.approved.txt index cf806a3835..0251f34439 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByJob.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByJob.approved.txt @@ -7,16 +7,18 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC DefaultJob : extra output line - Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | -------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|---------------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-[Param=10] | Yes | ^ - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | DefaultJob-[Param=10] | No | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-[Param=10] | No | - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | DefaultJob-[Param=10] | No | - | | | | | | | | | | | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-[Param=20] | Yes | ^ - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | DefaultJob-[Param=20] | No | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-[Param=20] | No | - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | DefaultJob-[Param=20] | No | + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|----------------------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-Bench1-[Param=10] | Yes | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | DefaultJob-Bench1-[Param=10] | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-Bench1-[Param=20] | Yes | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | DefaultJob-Bench1-[Param=20] | No | + | | | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | DefaultJob-Bench2-[Param=10] | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | DefaultJob-Bench2-[Param=10] | No | + | | | | | | | | | | | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | DefaultJob-Bench2-[Param=20] | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | DefaultJob-Bench2-[Param=20] | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByMethod.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByMethod.approved.txt index 4771f8b1f9..16527f70a7 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByMethod.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByMethod.approved.txt @@ -9,20 +9,20 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | ------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|--------------------------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Bench1.Foo-[Param=10]-DefaultJob | Yes | ^ + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Foo-Bench1-[Param=10]-DefaultJob | Yes | ^ | | | | | | | | | | | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Bench1.Foo-[Param=20]-DefaultJob | Yes | ^ + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Foo-Bench1-[Param=20]-DefaultJob | Yes | ^ | | | | | | | | | | | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2.Foo-[Param=10]-DefaultJob | No | ^ + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | Foo-Bench2-[Param=10]-DefaultJob | No | ^ | | | | | | | | | | | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2.Foo-[Param=20]-DefaultJob | No | ^ + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | Foo-Bench2-[Param=20]-DefaultJob | No | ^ | | | | | | | | | | | - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench1.Bar-[Param=10]-DefaultJob | No | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bar-Bench1-[Param=10]-DefaultJob | No | ^ | | | | | | | | | | | - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench1.Bar-[Param=20]-DefaultJob | No | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bar-Bench1-[Param=20]-DefaultJob | No | ^ | | | | | | | | | | | - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2.Bar-[Param=10]-DefaultJob | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bar-Bench2-[Param=10]-DefaultJob | No | ^ | | | | | | | | | | | - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2.Bar-[Param=20]-DefaultJob | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bar-Bench2-[Param=20]-DefaultJob | No | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByParams.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByParams.approved.txt index 5fe4d84070..b68732373c 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByParams.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByParams.approved.txt @@ -7,16 +7,18 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC DefaultJob : extra output line - Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | -------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|---------------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-DefaultJob | Yes | ^ - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-DefaultJob | No | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-DefaultJob | No | - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-DefaultJob | No | - | | | | | | | | | | | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-DefaultJob | Yes | ^ - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-DefaultJob | No | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-DefaultJob | No | - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-DefaultJob | No | + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|----------------------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-Bench1-DefaultJob | Yes | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-Bench1-DefaultJob | No | + | | | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | [Param=10]-Bench2-DefaultJob | No | + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | [Param=10]-Bench2-DefaultJob | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-Bench1-DefaultJob | Yes | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-Bench1-DefaultJob | No | + | | | | | | | | | | | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | [Param=20]-Bench2-DefaultJob | No | + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | [Param=20]-Bench2-DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.Default.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.Default.approved.txt index d94900d436..b2b69c9eb0 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.Default.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.Default.approved.txt @@ -7,16 +7,18 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC DefaultJob : extra output line - Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | -------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|---------------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-DefaultJob | Yes | ^ - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-DefaultJob | No | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-DefaultJob | No | - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-DefaultJob | No | - | | | | | | | | | | | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-DefaultJob | Yes | ^ - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-DefaultJob | No | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-DefaultJob | No | - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-DefaultJob | No | + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|----------------------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Bench1-[Param=10]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | Bench1-[Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Bench1-[Param=20]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | Bench1-[Param=20]-DefaultJob | No | + | | | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2-[Param=10]-DefaultJob | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2-[Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2-[Param=20]-DefaultJob | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2-[Param=20]-DefaultJob | No | Errors: 0 From 22e88839f27a14dfab1bac640239aa30408d7d7d Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Thu, 20 Oct 2022 14:09:51 +0300 Subject: [PATCH 7/9] Add new approved files (ByType) --- ...calRuleTest.NoBaseline.ByType.approved.txt | 22 +++++++++++++++++ ...alRuleTest.OneBaseline.ByType.approved.txt | 24 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByType.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByType.approved.txt diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByType.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByType.approved.txt new file mode 100644 index 0000000000..c59ba8d399 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.NoBaseline.ByType.approved.txt @@ -0,0 +1,22 @@ +=== NoBaseline.ByType === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Type | Method | Categories | Param | Mean | Error | StdDev | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | Bench1 | No | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | Bench1 | No | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | Bench1 | No | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | Bench1 | No | + | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | Bench2 | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | Bench2 | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | Bench2 | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | Bench2 | No | + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByType.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByType.approved.txt new file mode 100644 index 0000000000..5fb586777b --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByType.approved.txt @@ -0,0 +1,24 @@ +=== OneBaseline.ByType === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|----------------------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Bench1-[Param=10]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | Bench1-[Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Bench1-[Param=20]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | Bench1-[Param=20]-DefaultJob | No | + | | | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2-[Param=10]-DefaultJob | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2-[Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2-[Param=20]-DefaultJob | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2-[Param=20]-DefaultJob | No | + +Errors: 0 From 6571601114ee8ddb24a9a3e103360e1e346b6240 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Thu, 20 Oct 2022 14:11:31 +0300 Subject: [PATCH 8/9] most conservative mode --- src/BenchmarkDotNet/Order/DefaultOrderer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Order/DefaultOrderer.cs b/src/BenchmarkDotNet/Order/DefaultOrderer.cs index 9cbcaee3e6..a2e3d627ea 100644 --- a/src/BenchmarkDotNet/Order/DefaultOrderer.cs +++ b/src/BenchmarkDotNet/Order/DefaultOrderer.cs @@ -94,7 +94,7 @@ public string GetLogicalGroupKey(ImmutableArray allBenchmarksCase bool hasJobBaselines = allBenchmarksCases.Any(b => b.Job.Meta.Baseline); bool hasDescriptorBaselines = allBenchmarksCases.Any(b => b.Descriptor.Baseline); - if (hasMultipleTypes) + if (hasMultipleTypes && !hasJobBaselines && !hasDescriptorBaselines) { implicitRules.Add(BenchmarkLogicalGroupRule.ByType); } From 5eb47d43beab6746cc3e45b5690615328da4b420 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Thu, 20 Oct 2022 14:14:34 +0300 Subject: [PATCH 9/9] Update new approved files --- ...leTest.OneBaseline.ByCategory.approved.txt | 26 ++++++++-------- ...calRuleTest.OneBaseline.ByJob.approved.txt | 24 +++++++-------- ...RuleTest.OneBaseline.ByMethod.approved.txt | 30 ++++++++----------- ...RuleTest.OneBaseline.ByParams.approved.txt | 24 +++++++-------- ...lRuleTest.OneBaseline.Default.approved.txt | 24 +++++++-------- 5 files changed, 59 insertions(+), 69 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByCategory.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByCategory.approved.txt index 4d06c947b9..83ee167c3c 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByCategory.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByCategory.approved.txt @@ -7,18 +7,18 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC DefaultJob : extra output line - Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | -------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|------------------------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | A-Bench1-[Param=10]-DefaultJob | Yes | ^ - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | A-Bench1-[Param=10]-DefaultJob | No | - | | | | | | | | | | | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | A-Bench1-[Param=20]-DefaultJob | Yes | ^ - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | A-Bench1-[Param=20]-DefaultJob | No | - | | | | | | | | | | | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-Bench2-[Param=10]-DefaultJob | No | ^ - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-Bench2-[Param=10]-DefaultJob | No | - | | | | | | | | | | | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-Bench2-[Param=20]-DefaultJob | No | ^ - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-Bench2-[Param=20]-DefaultJob | No | + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|------------------------ |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | A-[Param=10]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | A-[Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | A-[Param=20]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | A-[Param=20]-DefaultJob | No | + | | | | | | | | | | | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-[Param=10]-DefaultJob | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-[Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-[Param=20]-DefaultJob | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | B-[Param=20]-DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByJob.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByJob.approved.txt index 0251f34439..cf806a3835 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByJob.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByJob.approved.txt @@ -7,18 +7,16 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC DefaultJob : extra output line - Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | -------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|----------------------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-Bench1-[Param=10] | Yes | ^ - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | DefaultJob-Bench1-[Param=10] | No | - | | | | | | | | | | | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-Bench1-[Param=20] | Yes | ^ - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | DefaultJob-Bench1-[Param=20] | No | - | | | | | | | | | | | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | DefaultJob-Bench2-[Param=10] | No | ^ - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | DefaultJob-Bench2-[Param=10] | No | - | | | | | | | | | | | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | DefaultJob-Bench2-[Param=20] | No | ^ - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | DefaultJob-Bench2-[Param=20] | No | + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|---------------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-[Param=10] | Yes | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | DefaultJob-[Param=10] | No | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-[Param=10] | No | + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | DefaultJob-[Param=10] | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-[Param=20] | Yes | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | DefaultJob-[Param=20] | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | DefaultJob-[Param=20] | No | + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | DefaultJob-[Param=20] | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByMethod.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByMethod.approved.txt index 16527f70a7..e71c423807 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByMethod.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByMethod.approved.txt @@ -7,22 +7,18 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC DefaultJob : extra output line - Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | -------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|--------------------------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Foo-Bench1-[Param=10]-DefaultJob | Yes | ^ - | | | | | | | | | | | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Foo-Bench1-[Param=20]-DefaultJob | Yes | ^ - | | | | | | | | | | | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | Foo-Bench2-[Param=10]-DefaultJob | No | ^ - | | | | | | | | | | | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | Foo-Bench2-[Param=20]-DefaultJob | No | ^ - | | | | | | | | | | | - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bar-Bench1-[Param=10]-DefaultJob | No | ^ - | | | | | | | | | | | - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bar-Bench1-[Param=20]-DefaultJob | No | ^ - | | | | | | | | | | | - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bar-Bench2-[Param=10]-DefaultJob | No | ^ - | | | | | | | | | | | - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bar-Bench2-[Param=20]-DefaultJob | No | ^ + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|-------------------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Foo-[Param=10]-DefaultJob | Yes | ^ + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Foo-[Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Foo-[Param=20]-DefaultJob | Yes | ^ + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Foo-[Param=20]-DefaultJob | No | + | | | | | | | | | | | + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bar-[Param=10]-DefaultJob | No | ^ + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bar-[Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bar-[Param=20]-DefaultJob | No | ^ + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bar-[Param=20]-DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByParams.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByParams.approved.txt index b68732373c..5fe4d84070 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByParams.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.ByParams.approved.txt @@ -7,18 +7,16 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC DefaultJob : extra output line - Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | -------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|----------------------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-Bench1-DefaultJob | Yes | ^ - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-Bench1-DefaultJob | No | - | | | | | | | | | | | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | [Param=10]-Bench2-DefaultJob | No | - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | [Param=10]-Bench2-DefaultJob | No | - | | | | | | | | | | | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-Bench1-DefaultJob | Yes | ^ - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-Bench1-DefaultJob | No | - | | | | | | | | | | | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | [Param=20]-Bench2-DefaultJob | No | - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | [Param=20]-Bench2-DefaultJob | No | + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|---------------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-DefaultJob | No | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-DefaultJob | No | + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-DefaultJob | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-DefaultJob | No | + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.Default.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.Default.approved.txt index b2b69c9eb0..d94900d436 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.Default.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterMultipleClassApprovalTests.LogicalRuleTest.OneBaseline.Default.approved.txt @@ -7,18 +7,16 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC DefaultJob : extra output line - Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | -------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|----------------------------- |--------- | - Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Bench1-[Param=10]-DefaultJob | Yes | ^ - Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | Bench1-[Param=10]-DefaultJob | No | - | | | | | | | | | | | - Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | Bench1-[Param=20]-DefaultJob | Yes | ^ - Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | Bench1-[Param=20]-DefaultJob | No | - | | | | | | | | | | | - Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2-[Param=10]-DefaultJob | No | ^ - Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2-[Param=10]-DefaultJob | No | - | | | | | | | | | | | - Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2-[Param=20]-DefaultJob | No | ^ - Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | Bench2-[Param=20]-DefaultJob | No | + Type | Method | Categories | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |------- |----------- |------ |---------:|--------:|--------:|------:|--------:|---------------------- |--------- | + Bench1 | Foo | A | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-DefaultJob | No | + Bench2 | Foo | B | 10 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=10]-DefaultJob | No | + Bench2 | Bar | B | 10 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=10]-DefaultJob | No | + | | | | | | | | | | | + Bench1 | Foo | A | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-DefaultJob | Yes | ^ + Bench1 | Bar | A | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-DefaultJob | No | + Bench2 | Foo | B | 20 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=20]-DefaultJob | No | + Bench2 | Bar | B | 20 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | [Param=20]-DefaultJob | No | Errors: 0