Organize benchmarks and add optional .NET 11 runtime support

.github/workflows/ci.yml

@@ -36,9 +36,6 @@ jobs:
         run: dotnet restore Lua.slnx
       - name: Test
         run: dotnet test tests/Lua.Tests/Lua.Tests.csproj -c Release --filter "TestCategory!=ExpectedFailure"
-      - name: Verify interpreter benchmark workloads
-        working-directory: sandbox/Benchmark
-        run: dotnet run --project Benchmark.csproj -c Release --no-restore -- --verify
       - name: Test debugger protocol
         run: dotnet test tests/Lua.Debugger.Dap.Tests/Lua.Debugger.Dap.Tests.csproj -c Release --no-restore
       - name: Test debugger controls and hosting

.gitignore

@@ -1,6 +1,8 @@
 .DS_Store
 .claude/settings.local.json
 artifacts/
+BenchmarkDotNet.Artifacts/
+.bdn/
 [Bb]in/
 [Oo]bj/
 

global.json

@@ -0,0 +1,7 @@
+{
+  "sdk": {
+    "version": "10.0.100",
+    "rollForward": "latestFeature",
+    "allowPrerelease": false
+  }
+}

sandbox/Benchmark.NativeAot/Benchmark.NativeAot.csproj

@@ -8,8 +8,20 @@
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" Version="0.15.8" />
     <ProjectReference Include="../../src/Lua/Lua.csproj" />
-    <Compile Include="../Benchmark/ComparisonWorkloads.cs" Link="ComparisonWorkloads.cs" />
-    <Compile Include="../Benchmark/LuaCSharpComparison.cs" Link="LuaCSharpComparison.cs" />
-    <EmbeddedResource Include="../Benchmark/n-body.lua" LogicalName="Comparison.n-body.lua" />
+    <Compile
+      Include="../Benchmark/Workloads/ComparisonWorkloads.cs"
+      Link="ComparisonWorkloads.cs"
+    />
+    <Compile Include="../Benchmark/Workloads/BenchmarkScripts.cs" Link="BenchmarkScripts.cs" />
+    <Compile
+      Include="../Benchmark/Comparisons/LuaCSharpComparison.cs"
+      Link="LuaCSharpComparison.cs"
+    />
+    <EmbeddedResource Include="../Benchmark/Scripts/Comparison/*.lua">
+      <LogicalName>Benchmark.Scripts.Comparison.%(Filename)%(Extension)</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="../Benchmark/Scripts/Compilation/*.lua">
+      <LogicalName>Benchmark.Scripts.Compilation.%(Filename)%(Extension)</LogicalName>
+    </EmbeddedResource>
   </ItemGroup>
 </Project>

sandbox/Benchmark.NativeAot/Program.cs

@@ -1,34 +1,5 @@
 using BenchmarkDotNet.Running;
 
-if (args.Contains("--verify"))
-{
-    foreach (
-        var workload in ComparisonWorkloads
-            .Cases.Concat(ComparisonWorkloads.CompilationCases)
-            .DistinctBy(c => c.Name)
-    )
-        RuntimeVerification.Verify(workload);
-    return;
-}
-
 BenchmarkSwitcher
     .FromTypes([typeof(RuntimeExecutionComparison), typeof(RuntimeCompilationComparison)])
     .Run(args);
-
-static class RuntimeVerification
-{
-    public static void Verify(ComparisonCase workload)
-    {
-        foreach (var bytes in new[] { false, true })
-        {
-            using var runner = new LuaCSharpComparison(workload.Source, bytes);
-            for (var i = 0; i < 2; i++)
-                ComparisonWorkloads.Check(
-                    workload,
-                    bytes ? "Lua-CSharp bytes" : "Lua-CSharp",
-                    runner.Run()
-                );
-        }
-        Console.WriteLine($"PASS {workload.Name}: {workload.Expected:R}");
-    }
-}

sandbox/Benchmark.NativeAot/README.md

@@ -1,7 +1,7 @@
 # .NET 10 JIT and NativeAOT benchmarks
 
 Compare Lua-CSharp source compilation and bytecode execution under .NET 10 JIT
-and NativeAOT, using the sources and checksum checks from `../Benchmark`.
+and NativeAOT, using the Lua sources from `../Benchmark`.
 NativeAOT compiles the C# interpreter; Lua source is still compiled to Lua
 bytecode and interpreted.
 
@@ -10,7 +10,6 @@ Run from this directory:
 ```powershell
 dotnet restore Benchmark.NativeAot.csproj
 dotnet build Benchmark.NativeAot.csproj -c Release --no-restore
-dotnet bin/Release/net10.0/Benchmark.NativeAot.dll --verify
 dotnet bin/Release/net10.0/Benchmark.NativeAot.dll --filter '*RuntimeExecutionComparison*numeric-loop*' --exporters json
 dotnet bin/Release/net10.0/Benchmark.NativeAot.dll --filter '*RuntimeCompilationComparison*numeric-loop*' --exporters json
 ```
@@ -26,9 +25,11 @@ uses speed optimization and the current CPU's native instruction set. Record
 the resolved JIT runtime version too. These native executables target the build
 machine's CPU capabilities.
 
-`--verify` checks all 15 distinct workloads in the managed host, twice in each
-string mode. Each benchmark's setup also runs its checksum checks inside the
-actual JIT or NativeAOT benchmark process before timing. Timed execution uses
+The shared `Scripts/Comparison/` sources include string-keyed map operations
+and Mandelbrot escape iterations. Filter with `*map*` or `*mandelbrot*` to run
+either case.
+
+Timed execution uses
 UTF-16 mode with ASCII inputs, retained state and a compiled chunk. Compilation
 has six source cases, a fresh state outside each batch of 64 loads, and no Lua
 execution during measurement. Both jobs use three launches. Compilation warms

sandbox/Benchmark.NativeAot/RuntimeComparison.cs

@@ -75,7 +75,6 @@ public class RuntimeExecutionComparison
     public void Setup()
     {
         var workload = ComparisonWorkloads.Find(Case);
-        RuntimeVerification.Verify(workload);
         runner = new(workload.Source);
     }
 
@@ -102,7 +101,6 @@ public class RuntimeCompilationComparison
     public void Setup()
     {
         var workload = ComparisonWorkloads.FindCompilation(Case);
-        RuntimeVerification.Verify(workload);
         source = workload.Source;
         var timer = Stopwatch.StartNew();
         do

sandbox/Benchmark/Benchmark.csproj

@@ -1,30 +1,24 @@
-<Project Sdk="Microsoft.NET.Sdk">
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net10.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.15.8" />
-    <PackageReference Include="MoonSharp" Version="2.0.0" />
-    <PackageReference Include="NuLua.Lua52" Version="0.1.0" />
-    <PackageReference Include="NLua" Version="1.7.3" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\src\Lua\Lua.csproj" />
-    <ProjectReference Include="../../src/Lua.Debugger/Lua.Debugger.csproj" />
-    <ProjectReference Include="..\..\src\Lua.Annotations\Lua.Annotations.csproj" />
-    <ProjectReference
-      Include="..\..\src\Lua.SourceGenerator\Lua.SourceGenerator.csproj"
-      OutputItemType="Analyzer"
-      ReferenceOutputAssembly="false"
-    />
-  </ItemGroup>
-
-  <ItemGroup>
-    <EmbeddedResource Include="n-body.lua" LogicalName="Comparison.n-body.lua" />
-  </ItemGroup>
-</Project>
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.16.0-preview.2" />
+    <PackageReference Include="MoonSharp" Version="2.0.0" />
+    <PackageReference Include="NuLua.Lua52" Version="0.2.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Lua\Lua.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <EmbeddedResource Include="Scripts/**/*.lua">
+      <LogicalName>Benchmark.Scripts.$([System.String]::Copy('%(RecursiveDir)').Replace('\', '.').Replace('/', '.'))%(Filename)%(Extension)</LogicalName>
+    </EmbeddedResource>
+  </ItemGroup>
+</Project>

sandbox/Benchmark/Benchmark.sln

@@ -5,12 +5,6 @@ VisualStudioVersion = 17.5.002.0
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Benchmark", "Benchmark.csproj", "{ECA61AB3-09C9-4990-8201-2E83BB9FD48B}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "bin", "bin", "{B33416D1-661B-461E-95CD-C3BCC7DDC23F}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Release", "Release", "{0A15FAB8-2749-4DF1-B1B3-925E6CD29AB1}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "net8.0", "net8.0", "{2308BEED-4D71-4415-B9AC-269272953282}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -25,10 +19,6 @@ Global
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{0A15FAB8-2749-4DF1-B1B3-925E6CD29AB1} = {B33416D1-661B-461E-95CD-C3BCC7DDC23F}
-		{2308BEED-4D71-4415-B9AC-269272953282} = {0A15FAB8-2749-4DF1-B1B3-925E6CD29AB1}
-	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {F4ABD181-CD81-474C-93FF-CD4E6F93197A}
 	EndGlobalSection

sandbox/Benchmark/BenchmarkConfig.cs

@@ -0,0 +1,69 @@
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.ConsoleArguments;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Toolchains.NetCoreApp;
+using CommandLine;
+
+static class BenchmarkConfig
+{
+    public static IConfig Create(ref string[] args)
+    {
+        var defaults = ManualConfig
+            .Create(DefaultConfig.Instance)
+            .AddDiagnoser(MemoryDiagnoser.Default);
+        var (success, parsed, options) = ConfigParser.Parse(args, NullLogger.Instance, defaults);
+        if (
+            !success
+            || !options!.Runtimes.Any(runtime => CoreRuntime.Parse(runtime) == CoreRuntime.Core11_0)
+        )
+            return defaults;
+
+        var effective = ManualConfig.Union(defaults, parsed!).CreateImmutableConfig();
+        var net11 = new Net11RuntimeToolchain(
+            new NetCoreAppSettings
+            {
+                CliPath = options.CliPath,
+                PackagesPath = options.RestorePath,
+                UseFreshPackages = options.UseFreshPackages,
+            }
+        );
+        var jobs = effective
+            .GetJobs()
+            .Select(job =>
+                job.Infrastructure.Runtime == CoreRuntime.Core11_0 ? job.WithToolchain(net11) : job
+            )
+            .ToArray();
+        var first = jobs[0];
+        // Copy settings without job metadata, so the template cannot overwrite baselines.
+        var template = new Job([
+            first.Environment,
+            first.Run,
+            first.Infrastructure,
+            first.Accuracy,
+        ]);
+        template.Infrastructure.Runtime = null;
+
+        // Runtime jobs are already configured; let BenchmarkDotNet handle the remaining CLI normally.
+        options.Runtimes = [];
+        args = Parser
+            .Default.FormatCommandLineArgs(options, settings => settings.SkipDefault = true)
+            .ToArray();
+        return new RuntimeJobs(effective, [.. jobs, template.AsDefault()]);
+    }
+
+    sealed class RuntimeJobs : ManualConfig, IConfig
+    {
+        readonly Job[] jobs;
+
+        public RuntimeJobs(IConfig source, Job[] jobs)
+        {
+            Add(source);
+            this.jobs = jobs;
+        }
+
+        IEnumerable<Job> IConfig.GetJobs() => jobs;
+    }
+}