ensure the default order of benchmarks is the same as declared in source code (#1907)

Author: adamsitnik

src/BenchmarkDotNet.Annotations/Attributes/BenchmarkAttribute.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Runtime.CompilerServices;
 using JetBrains.Annotations;
 
 namespace BenchmarkDotNet.Attributes
@@ -7,10 +8,20 @@ namespace BenchmarkDotNet.Attributes
     [MeansImplicitUse]
     public class BenchmarkAttribute : Attribute
     {
+        public BenchmarkAttribute([CallerLineNumber] int sourceCodeLineNumber = 0, [CallerFilePath] string sourceCodeFile = "")
+        {
+            SourceCodeLineNumber = sourceCodeLineNumber;
+            SourceCodeFile = sourceCodeFile;
+        }
+
         public string Description { get; set; }
 
         public bool Baseline { get; set; }
 
         public int OperationsPerInvoke { get; set; } = 1;
+
+        public int SourceCodeLineNumber { get; }
+
+        public string SourceCodeFile { get; }
     }
 }

src/BenchmarkDotNet/Running/BenchmarkConverter.cs

@@ -23,13 +23,22 @@ public static BenchmarkRunInfo TypeToBenchmarks(Type type, IConfig config = null
 
             // We should check all methods including private to notify users about private methods with the [Benchmark] attribute
             var bindingFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic;
-            var benchmarkMethods = type.GetMethods(bindingFlags).Where(method => method.HasAttribute<BenchmarkAttribute>()).ToArray();
+            var benchmarkMethods = GetOrderedBenchmarkMethods(type.GetMethods(bindingFlags));
 
             return MethodsToBenchmarksWithFullConfig(type, benchmarkMethods, config);
         }
 
         public static BenchmarkRunInfo MethodsToBenchmarks(Type containingType, MethodInfo[] benchmarkMethods, IConfig config = null)
-            => MethodsToBenchmarksWithFullConfig(containingType, benchmarkMethods, config);
+            => MethodsToBenchmarksWithFullConfig(containingType, GetOrderedBenchmarkMethods(benchmarkMethods), config);
+
+        private static MethodInfo[] GetOrderedBenchmarkMethods(MethodInfo[] methods)
+            => methods
+                .Select(method => (method, attribute: method.ResolveAttribute<BenchmarkAttribute>()))
+                .Where(pair => pair.attribute is not null)
+                .OrderBy(pair => pair.attribute.SourceCodeFile)
+                .ThenBy(pair => pair.attribute.SourceCodeLineNumber)
+                .Select(pair => pair.method)
+                .ToArray();
 
         private static BenchmarkRunInfo MethodsToBenchmarksWithFullConfig(Type type, MethodInfo[] benchmarkMethods, IConfig config)
         {
@@ -108,7 +117,6 @@ private static IEnumerable<Descriptor> GetTargets(
             Tuple<MethodInfo, TargetedAttribute>[] iterationCleanupMethods)
         {
             return targetMethods
-                .Where(m => m.HasAttribute<BenchmarkAttribute>())
                 .Select(methodInfo => CreateDescriptor(type,
                                                    GetTargetedMatchingMethod(methodInfo, globalSetupMethods),
                                                    methodInfo,

src/BenchmarkDotNet/Running/TypeFilter.cs

@@ -9,9 +9,9 @@
 
 namespace BenchmarkDotNet.Running
 {
-    internal static class TypeFilter
+    public static class TypeFilter
     {
-        internal static (bool allTypesValid, IReadOnlyList<Type> runnable) GetTypesWithRunnableBenchmarks(IEnumerable<Type> types, IEnumerable<Assembly> assemblies, ILogger logger)
+        public static (bool allTypesValid, IReadOnlyList<Type> runnable) GetTypesWithRunnableBenchmarks(IEnumerable<Type> types, IEnumerable<Assembly> assemblies, ILogger logger)
         {
             var validRunnableTypes = new List<Type>();
 
@@ -37,7 +37,7 @@ internal static (bool allTypesValid, IReadOnlyList<Type> runnable) GetTypesWithR
             return (true, validRunnableTypes);
         }
 
-        internal static BenchmarkRunInfo[] Filter(IConfig effectiveConfig, IEnumerable<Type> types)
+        public static BenchmarkRunInfo[] Filter(IConfig effectiveConfig, IEnumerable<Type> types)
             => types
                 .Select(type => BenchmarkConverter.TypeToBenchmarks(type, effectiveConfig))
                 .Where(info => info.BenchmarksCases.Any())

tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.BAC_Partial_DifferentFiles.cs

@@ -0,0 +1,12 @@
+using BenchmarkDotNet.Attributes;
+
+namespace BenchmarkDotNet.Tests.Running
+{
+    public partial class BenchmarkConverterTests
+    {
+        public partial class BAC_Partial_DifferentFiles
+        {
+            [Benchmark] public void B() { }
+        }
+    }
+}

tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs

@@ -1,4 +1,5 @@
-using System.Linq;
+using System;
+using System.Linq;
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Environments;
@@ -9,7 +10,7 @@
 
 namespace BenchmarkDotNet.Tests.Running
 {
-    public class BenchmarkConverterTests
+    public partial class BenchmarkConverterTests
     {
         /// <summary>
         /// https://github.com/dotnet/BenchmarkDotNet/issues/495
@@ -198,5 +199,43 @@ public class WithFewMutators
         {
             [Benchmark] public void Method() { }
         }
+
+        [Fact]
+        public void MethodDeclarationOrderIsPreserved()
+        {
+            foreach (Type type in new[] { typeof(BAC), typeof(BAC_Partial), typeof(BAC_Partial_DifferentFiles) })
+            {
+                var info = BenchmarkConverter.TypeToBenchmarks(type);
+
+                Assert.Equal(nameof(BAC.B), info.BenchmarksCases[0].Descriptor.WorkloadMethod.Name);
+                Assert.Equal(nameof(BAC.A), info.BenchmarksCases[1].Descriptor.WorkloadMethod.Name);
+                Assert.Equal(nameof(BAC.C), info.BenchmarksCases[2].Descriptor.WorkloadMethod.Name);
+            }
+        }
+
+        public class BAC
+        {
+            // BAC is not sorted in either desceding or ascending way
+            [Benchmark] public void B() { }
+            [Benchmark] public void A() { }
+            [Benchmark] public void C() { }
+        }
+
+        public partial class BAC_Partial
+        {
+            [Benchmark] public void B() { }
+            [Benchmark] public void A() { }
+        }
+
+        public partial class BAC_Partial
+        {
+            [Benchmark] public void C() { }
+        }
+
+        public partial class BAC_Partial_DifferentFiles
+        {
+            [Benchmark] public void A() { }
+            [Benchmark] public void C() { }
+        }
     }
 }