Add analyzer projects to main sln.

Run analyzer tests in CI.
Fix compile errors.
Ensure analyzers run against test projects.

Author: timcassell

.github/workflows/run-tests.yaml

@@ -45,6 +45,12 @@ jobs:
       - name: Run task 'build'
         shell: cmd
         run: ./build.cmd build
+      - name: Run task 'unit-tests'
+        shell: cmd
+        run: ./build.cmd unit-tests -e
+      - name: Run task 'analyzer-tests'
+        shell: cmd
+        run: ./build.cmd analyzer-tests -e
       - name: Run task 'in-tests-full'
         shell: cmd
         run: ./build.cmd in-tests-full -e
@@ -79,6 +85,8 @@ jobs:
       # Build and Test
       - name: Run task 'build'
         run: ./build.cmd build
+      - name: Run task 'analyzer-tests'
+        run: ./build.cmd analyzer-tests -e
       - name: Run task 'unit-tests'
         run: ./build.cmd unit-tests -e
       - name: Run task 'in-tests-core'
@@ -116,6 +124,8 @@ jobs:
       # Build and Test
       - name: Run task 'build'
         run: ./build.cmd build
+      - name: Run task 'analyzer-tests'
+        run: ./build.cmd analyzer-tests -e
       - name: Run task 'unit-tests'
         run: ./build.cmd unit-tests -e
       - name: Run task 'in-tests-core'

BenchmarkDotNet.Analyzers.sln

@@ -59,6 +59,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.P
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting.Tests", "tests\BenchmarkDotNet.Exporters.Plotting.Tests\BenchmarkDotNet.Exporters.Plotting.Tests.csproj", "{199AC83E-30BD-40CD-87CE-0C838AC0320D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Analyzers", "src\BenchmarkDotNet.Analyzers\BenchmarkDotNet.Analyzers.csproj", "{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Analyzers.Tests", "tests\BenchmarkDotNet.Analyzers.Tests\BenchmarkDotNet.Analyzers.Tests.csproj", "{7DE89F16-2160-42E3-004E-1F5064732121}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -161,6 +165,14 @@ Global
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7DE89F16-2160-42E3-004E-1F5064732121}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7DE89F16-2160-42E3-004E-1F5064732121}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7DE89F16-2160-42E3-004E-1F5064732121}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7DE89F16-2160-42E3-004E-1F5064732121}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -190,6 +202,8 @@ Global
 		{2E2283A3-6DA6-4482-8518-99D6D9F689AB} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 		{B92ECCEF-7C27-4012-9E19-679F3C40A6A6} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D} = {14195214-591A-45B7-851A-19D3BA2413F9}
+		{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
+		{7DE89F16-2160-42E3-004E-1F5064732121} = {14195214-591A-45B7-851A-19D3BA2413F9}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F}

BenchmarkDotNet.sln

@@ -98,6 +98,28 @@ public HelpInfo GetHelp()
     }
 }
 
+[TaskName(Name)]
+[TaskDescription("Run analyzer tests")]
+[IsDependentOn(typeof(BuildTask))]
+public class AnalyzerTestsTask : FrostingTask<BuildContext>, IHelpProvider
+{
+    private const string Name = "analyzer-tests";
+    public override void Run(BuildContext context) => context.UnitTestRunner.RunAnalyzerTests();
+
+    public HelpInfo GetHelp()
+    {
+        return new HelpInfo
+        {
+            Examples =
+            [
+                new Example(Name)
+                    .WithArgument(KnownOptions.Exclusive)
+                    .WithArgument(KnownOptions.Verbosity, "Diagnostic")
+            ]
+        };
+    }
+}
+
 [TaskName(Name)]
 [TaskDescription("Run integration tests using .NET Framework 4.6.2+ (slow)")]
 [IsDependentOn(typeof(BuildTask))]
@@ -123,8 +145,9 @@ public class InTestsCoreTask : FrostingTask<BuildContext>, IHelpProvider
 }
 
 [TaskName(Name)]
-[TaskDescription("Run all unit and integration tests (slow)")]
+[TaskDescription("Run all unit, analyzer, and integration tests (slow)")]
 [IsDependentOn(typeof(UnitTestsTask))]
+[IsDependentOn(typeof(AnalyzerTestsTask))]
 [IsDependentOn(typeof(InTestsFullTask))]
 [IsDependentOn(typeof(InTestsCoreTask))]
 public class AllTestsTask : FrostingTask<BuildContext>, IHelpProvider

build/BenchmarkDotNet.Build/Program.cs

@@ -20,6 +20,11 @@ public class UnitTestRunner(BuildContext context)
         .Combine("BenchmarkDotNet.Exporters.Plotting.Tests")
         .CombineWithFilePath("BenchmarkDotNet.Exporters.Plotting.Tests.csproj");
 
+    private FilePath AnalyzerTestsProjectFile { get; } = context.RootDirectory
+        .Combine("tests")
+        .Combine("BenchmarkDotNet.Analyzers.Tests")
+        .CombineWithFilePath("BenchmarkDotNet.Analyzers.Tests.csproj");
+
     private FilePath IntegrationTestsProjectFile { get; } = context.RootDirectory
         .Combine("tests")
         .Combine("BenchmarkDotNet.IntegrationTests")
@@ -70,5 +75,12 @@ public void RunUnitTests()
             RunUnitTests(targetFramework);
     }
 
+    public void RunAnalyzerTests()
+    {
+        string[] targetFrameworks = context.IsRunningOnWindows() ? ["net462", "net8.0"] : ["net8.0"];
+        foreach (var targetFramework in targetFrameworks)
+            RunTests(AnalyzerTestsProjectFile, "analyzer", targetFramework);
+    }
+
     public void RunInTests(string tfm) => RunTests(IntegrationTestsProjectFile, "integration", tfm);
 }

build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs

@@ -1,6 +1,6 @@
 {
   "sdk": {
-    "version": "8.0.410",
+    "version": "9.0.306",
     "rollForward": "disable"
   }
 }

build/sdk/global.json

@@ -1,24 +1,20 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
+  <Import Project="..\..\build\common.props" />
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
     <IsPackable>false</IsPackable>
-
     <!-- Avoid ID conflicts with the package project. -->
     <PackageId>BenchmarkDotNet.Analyzers</PackageId>
     <EnforceExtendedAnalyzerRules>true</EnforceExtendedAnalyzerRules>
     <NoWarn>$(NoWarn);CS1591</NoWarn>
   </PropertyGroup>
-
   <ItemGroup>
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.14.0" />
   </ItemGroup>
-
   <ItemGroup>
     <AdditionalFiles Include="AnalyzerReleases.Shipped.md" />
     <AdditionalFiles Include="AnalyzerReleases.Unshipped.md" />
   </ItemGroup>
-
   <ItemGroup>
     <Compile Update="BenchmarkDotNetAnalyzerResources.Designer.cs">
       <DesignTime>True</DesignTime>
@@ -30,12 +26,9 @@
       <LastGenOutput>BenchmarkDotNetAnalyzerResources.Designer.cs</LastGenOutput>
     </EmbeddedResource>
   </ItemGroup>
-
   <ItemGroup>
     <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleToAttribute">
       <_Parameter1>BenchmarkDotNet.Analyzers.Tests,PublicKey=00240000048000009400000006020000002400005253413100040000010001002970bbdfca4d129fc74b4845b239973f1b183684f0d7db5e1de7e085917e3656cf94884803cb800d85d5aae5838fb3f8fd1f2829e8208c4f087afcfe970bce44037ba30a66749cd5514b410ca8a35e9c7d6eb86975853c834c9ad25051537f9a05a0c540c5d84f2c7b32ab01619d84367fd424797ba3242f08b0e6ae75f66dad</_Parameter1>
     </AssemblyAttribute>
   </ItemGroup>
-
-  <Import Project="..\..\build\common.props" />
 </Project>

src/BenchmarkDotNet.Analyzers/BenchmarkDotNet.Analyzers.csproj

@@ -16,6 +16,6 @@
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\BenchmarkDotNet.Analyzers\BenchmarkDotNet.Analyzers.csproj" PrivateAssets="all" />
-    <None Include="$(OutputPath)\netstandard2.0\BenchmarkDotNet.Analyzers.dll" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" />
+    <Content Include="..\BenchmarkDotNet.Analyzers\bin\$(Configuration)\netstandard2.0\BenchmarkDotNet.Analyzers.dll" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" />
   </ItemGroup>
 </Project>

src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj

@@ -504,15 +504,15 @@ public async Task Providing_an_unexpected_array_value_type_to_params_attribute_s
 
                                                     public class BenchmarkClass
                                                     {
-                                                        [{{dummyAttributeUsage}}Params({{string.Format(arrayValuesContainerAttributeArgument.Value1, valueAndType.Value1, valueAndType.Value2)}})]
+                                                        [{{dummyAttributeUsage}}Params({{string.Format(arrayValuesContainerAttributeArgument.Value1!, valueAndType.Value1, valueAndType.Value2)}})]
                                                         public {{expectedFieldOrPropertyType}} {{fieldOrPropertyDeclaration}}
                                                     }
                                                     """;
                 TestCode = testCode;
                 ReferenceDummyAttribute();
                 ReferenceDummyEnum();
 
-                AddDefaultExpectedDiagnostic(string.Format(arrayValuesContainerAttributeArgument.Value2,
+                AddDefaultExpectedDiagnostic(string.Format(arrayValuesContainerAttributeArgument.Value2!,
                                                            valueAndType.Value1,
                                                            valueAndType.Value2),
                                              expectedFieldOrPropertyType,

tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ParamsAttributeAnalyzerTests.cs

@@ -1,51 +1,33 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
+  <Import Project="..\..\build\common.props" />
   <PropertyGroup>
-    <TargetFrameworks>net462;net6.0;net8.0</TargetFrameworks>
+    <AssemblyName>BenchmarkDotNet.Analyzers.Tests</AssemblyName>
+    <TargetFrameworks>net462;net8.0</TargetFrameworks>
     <Nullable>enable</Nullable>
-    
     <PreserveCompilationContext>true</PreserveCompilationContext>
     <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
     <GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
     <SuppressTfmSupportBuildErrors>true</SuppressTfmSupportBuildErrors>
+    <NoWarn>$(NoWarn);CS1591</NoWarn>
   </PropertyGroup>
-
   <ItemGroup>
     <None Remove="BenchmarkDotNet.Analyzers.Tests.csproj.DotSettings" />
   </ItemGroup>
-
   <ItemGroup>
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Analyzer.Testing" Version="1.1.2" />
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="4.14.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
     <PackageReference Include="System.Formats.Asn1" Version="9.0.10" />
-    <PackageReference Include="System.Text.RegularExpressions" Version="4.3.1" />
     <PackageReference Include="xunit" Version="2.9.3" />
     <PackageReference Include="Xunit.Combinatorial" Version="1.6.24" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
   </ItemGroup>
-
   <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.0" />
     <ProjectReference Include="..\..\src\BenchmarkDotNet.Analyzers\BenchmarkDotNet.Analyzers.csproj" />
     <ProjectReference Include="..\..\src\BenchmarkDotNet\BenchmarkDotNet.csproj" />
   </ItemGroup>
-
-  <Choose>
-    <When Condition="'$(TargetFramework)' == 'net6.0'">
-      <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.0" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
-
-  <Import Project="..\..\build\common.props" />
-
-</Project>
+</Project>