Support async sources

BenchmarkDotNet.slnx

@@ -2,6 +2,7 @@
   <Folder Name="/build/">
     <File Path="build/common.props" />
     <File Path="build/common.targets" />
+    <File Path="build/roslynBands.props" />
   </Folder>
   <Folder Name="/samples/">
     <File Path="samples/Directory.Build.props" />

build/BenchmarkDotNet.Build/Runners/BuildRunner.cs

@@ -124,12 +124,14 @@ public void BuildProjectSilent(FilePath projectFile)
     public void BuildAnalyzers()
     {
         context.Information("BuildSystemProvider: " + context.BuildSystem().Provider);
-        string[] mccVersions = ["2.8", "3.8", "4.8"];
+        // Each band must restore as well as build: the PackageReference is chosen by MccVersion, so reusing the
+        // default band's project.assets.json silently compiles every band against the default's Roslyn.
+        string[] mccVersions = ["2.8", "3.0", "3.8", "4.4", "4.8", "4.12"];
         foreach (string version in mccVersions)
         {
             context.DotNetBuild(context.AnalyzersProjectFile.FullPath, new DotNetBuildSettings
             {
-                NoRestore = true,
+                NoRestore = false,
                 DiagnosticOutput = true,
                 MSBuildSettings = context.MsBuildSettingsBuild,
                 Configuration = context.BuildConfiguration,
@@ -139,7 +141,7 @@ public void BuildAnalyzers()
 
             context.DotNetBuild(context.CodeFixersProjectFile.FullPath, new DotNetBuildSettings
             {
-                NoRestore = true,
+                NoRestore = false,
                 DiagnosticOutput = true,
                 MSBuildSettings = context.MsBuildSettingsBuild,
                 Configuration = context.BuildConfiguration,

build/roslynBands.props

@@ -0,0 +1,26 @@
+<!-- Shared by BenchmarkDotNet.Analyzers and BenchmarkDotNet.CodeFixers, which each build one assembly per Roslyn version into analyzers/dotnet/roslyn$(MccVersion). -->
+<Project>
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <IsPackable>false</IsPackable>
+    <EnforceExtendedAnalyzerRules>true</EnforceExtendedAnalyzerRules>
+    <NoWarn>$(NoWarn);CS1591</NoWarn>
+    <!-- Default version used by local builds. This is overridden by the build script. -->
+    <MccVersion>4.12</MccVersion>
+    <OutputPath>bin\$(Configuration)\roslyn$(MccVersion)\cs</OutputPath>
+    <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
+    <!-- We multi-target for different compiler versions. https://github.com/dotnet/roslyn/discussions/81256#discussioncomment-14975130, https://github.com/dotnet/roslyn/blob/main/docs/wiki/NuGet-packages.md#versioning -->
+    <!-- C# 7.3/Roslyn 2.8 is our baseline (min version we support) -->
+    <!-- ITypeSymbol.IsRefLikeType -->
+    <DefineConstants Condition="$([MSBuild]::VersionGreaterThanOrEquals('$(MccVersion)', '3.0'))">$(DefineConstants);CODE_ANALYSIS_3_0</DefineConstants>
+    <!-- C# 9 -->
+    <DefineConstants Condition="$([MSBuild]::VersionGreaterThanOrEquals('$(MccVersion)', '3.8'))">$(DefineConstants);CODE_ANALYSIS_3_8</DefineConstants>
+    <!-- C# 11 -->
+    <DefineConstants Condition="$([MSBuild]::VersionGreaterThanOrEquals('$(MccVersion)', '4.4'))">$(DefineConstants);CODE_ANALYSIS_4_4</DefineConstants>
+    <!-- C# 12 -->
+    <DefineConstants Condition="$([MSBuild]::VersionGreaterThanOrEquals('$(MccVersion)', '4.8'))">$(DefineConstants);CODE_ANALYSIS_4_8</DefineConstants>
+    <!-- C# 13 -->
+    <DefineConstants Condition="$([MSBuild]::VersionGreaterThanOrEquals('$(MccVersion)', '4.12'))">$(DefineConstants);CODE_ANALYSIS_4_12</DefineConstants>
+    <MccPackageVersion>$(MccVersion).0</MccPackageVersion>
+  </PropertyGroup>
+</Project>

docs/articles/features/parameterization.md

@@ -21,4 +21,4 @@ name: Benchmark Parameterization
 
 [!include[IntroArrayParam](../samples/IntroArrayParam.md)]
 
-[!include[IntroArguments](../samples/IntroArgumentsPriority.md)]
+[!include[IntroArgumentsPriority](../samples/IntroArgumentsPriority.md)]

docs/articles/samples/IntroArgumentsSource.md

@@ -10,9 +10,18 @@ In case you want to use a lot of values, you should use
 You can mark one or several fields or properties in your class by the
   [`[ArgumentsSource]`](xref:BenchmarkDotNet.Attributes.ArgumentsSourceAttribute) attribute.
 In this attribute, you have to specify the name of public method/property which is going to provide the values
-  (something that implements `IEnumerable`).
+  (something that implements `IEnumerable<T>` or `IAsyncEnumerable<T>`).
+The element type has to be named: a source declared to return only the non-generic `IEnumerable` is rejected,
+  because the generated code has nothing to infer the argument's type from.
 The source may be instance or static. If the source is not in the same type as the benchmark, the type containing the source must be specified in the attribute constructor.
 
+A source returning `IAsyncEnumerable<T>` is awaited while the values are read, so they can be produced
+  asynchronously without resorting to blocking sync-over-async in the source, and such a source method may take
+  an optional [`[EnumeratorCancellation]`](xref:System.Runtime.CompilerServices.EnumeratorCancellationAttribute)
+  `CancellationToken` parameter. Starting the run from a thread that carries a single-threaded
+  `SynchronizationContext` needs the asynchronous entry points - see
+  @BenchmarkDotNet.Samples.IntroParamsSource, where the same applies to `[ParamsSource]`.
+
 ### Source code
 
 [!code-csharp[IntroArgumentsSource.cs](../../../samples/BenchmarkDotNet.Samples/IntroArgumentsSource.cs)]

docs/articles/samples/IntroParamsSource.md

@@ -7,10 +7,29 @@ uid: BenchmarkDotNet.Samples.IntroParamsSource
 In case you want to use a lot of values, you should use
   [`[ParamsSource]`](xref:BenchmarkDotNet.Attributes.ParamsSourceAttribute)
 You can mark one or several fields or properties in your class by the
-  [`[Params]`](xref:BenchmarkDotNet.Attributes.ParamsAttribute) attribute.
+  [`[ParamsSource]`](xref:BenchmarkDotNet.Attributes.ParamsSourceAttribute) attribute.
 In this attribute, you have to specify the name of public method/property which is going to provide the values
-  (something that implements `IEnumerable`).
+  (something that implements `IEnumerable<T>` or `IAsyncEnumerable<T>`).
+The element type has to be named: a source declared to return only the non-generic `IEnumerable` is rejected,
+  because the generated code has nothing to infer the parameter's type from.
 The source may be instance or static. If the source is not in the same type as the benchmark, the type containing the source must be specified in the attribute constructor.
+A static source declared on a base type is used just as one declared on the benchmark type itself.
+
+A source returning `IAsyncEnumerable<T>` is awaited while the values are read, so they can be produced
+  asynchronously - loaded from a database or a remote service, say - without resorting to blocking
+  sync-over-async in the source. Such a source method may take an optional
+  [`[EnumeratorCancellation]`](xref:System.Runtime.CompilerServices.EnumeratorCancellationAttribute)
+  `CancellationToken` parameter, which receives the benchmark's cancellation token while the values are
+  enumerated, so the asynchronous work can be cancelled.
+
+If you start the run from a thread that carries a single-threaded `SynchronizationContext` - a WPF or
+  WinForms UI thread, or legacy ASP.NET - use the asynchronous entry points
+  ([`BenchmarkRunner.RunAsync`](xref:BenchmarkDotNet.Running.BenchmarkRunner) or
+  [`BenchmarkConverter.TypeToBenchmarksAsync`](xref:BenchmarkDotNet.Running.BenchmarkConverter)) and await
+  them. The synchronous ones block the calling thread while the values are read, so an `await` inside your
+  own source captures that context and its continuation cannot run until the call it is blocking returns.
+  Awaiting the asynchronous entry point leaves the thread free to run it. Writing the source's own awaits as
+  `ConfigureAwait(false)` avoids the capture as well.
 
 ### Source code
 
@@ -29,10 +48,7 @@ The source may be instance or static. If the source is not in the same type as t
 
 ### Remarks
 
-**A remark about IParam.**
-
-You don't need to use `IParam` anymore since `0.11.0`.
-Just use complex types as you wish and override `ToString` method to change the display names used in the results.
+Use complex types as you wish and override the `ToString` method to change the display names used in the results.
 
 
 ### Links

samples/BenchmarkDotNet.Samples/IntroArgumentsSource.cs

@@ -1,3 +1,4 @@
+using System.Runtime.CompilerServices;
 using BenchmarkDotNet.Attributes;
 
 namespace BenchmarkDotNet.Samples
@@ -19,6 +20,21 @@ public class IntroArgumentsSource
         [Benchmark]
         [ArgumentsSource(typeof(BenchmarkArguments), nameof(BenchmarkArguments.TimeSpans))] // when the arguments come from a different type, specify that type here
         public void SingleArgument(TimeSpan time) => Thread.Sleep(time);
+
+        [Benchmark]
+        [ArgumentsSource(nameof(NumbersAsync))]
+        public double AsyncSourcedArguments(double x, double y) => Math.Pow(x, y);
+
+        // the source may be an IAsyncEnumerable, which BenchmarkDotNet awaits, so the values can be produced
+        // asynchronously without resorting to blocking sync-over-async in the source. It may take an optional
+        // [EnumeratorCancellation] CancellationToken, which receives the benchmark's cancellation token while
+        // the values are enumerated.
+        public static async IAsyncEnumerable<object[]> NumbersAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
+        {
+            await Task.Delay(10, cancellationToken);
+            yield return new object[] { 1.0, 1.0 };
+            yield return new object[] { 2.0, 2.0 };
+        }
     }
 
     public static class BenchmarkArguments

samples/BenchmarkDotNet.Samples/IntroParamsSource.cs

@@ -1,3 +1,4 @@
+using System.Runtime.CompilerServices;
 using BenchmarkDotNet.Attributes;
 
 namespace BenchmarkDotNet.Samples
@@ -22,8 +23,23 @@ public class IntroParamsSource
         [ParamsSource(typeof(ParamsValues), nameof(ParamsValues.ValuesForC))]
         public int C;
 
+        // public field getting its params from an asynchronous source, which BenchmarkDotNet awaits.
+        // Useful when the values can only be produced asynchronously - loaded from a database or a remote
+        // service - without resorting to blocking sync-over-async in the source.
+        [ParamsSource(nameof(ValuesForD))]
+        public int D;
+
+        // the source method may take an optional [EnumeratorCancellation] CancellationToken. It receives the
+        // benchmark's cancellation token while the values are enumerated, so the async work can be cancelled.
+        public static async IAsyncEnumerable<int> ValuesForD([EnumeratorCancellation] CancellationToken cancellationToken = default)
+        {
+            await Task.Delay(10, cancellationToken);
+            yield return 1;
+            yield return 2;
+        }
+
         [Benchmark]
-        public void Benchmark() => Thread.Sleep(A + B + C + 5);
+        public void Benchmark() => Thread.Sleep(A + B + C + D + 5);
     }
 
     public static class ParamsValues