From 628834abaed4c291881768fc6d84b0029f0b1c20 Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Mon, 23 Feb 2026 19:12:59 +0100 Subject: [PATCH 01/11] Batch CI tests based on namespace --- azure-pipelines-PR.yml | 160 ++++++++++++++++++++++++++++++++++++++-- eng/Build.ps1 | 24 +++++- eng/tests/TestSplit.fsx | 73 ++++++++++++++++++ 3 files changed, 248 insertions(+), 9 deletions(-) create mode 100644 eng/tests/TestSplit.fsx diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index d775d9ed907..fe75f070cf1 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -507,8 +507,8 @@ stages: continueOnError: true condition: failed() - # Windows With Compressed Metadata Desktop - - job: WindowsCompressedMetadata_Desktop + # Windows With Compressed Metadata Desktop (split into 3 batches) + - job: WindowsCompressedMetadata_Desktop_Batch1 variables: - name: XUNIT_LOGS value: $(Build.SourcesDirectory)\artifacts\TestResults\Release @@ -523,7 +523,7 @@ stages: - checkout: self clean: true - - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktop + - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch 1 env: FSharp_CacheEvictionImmediate: true DOTNET_DbgEnableMiniDump: 1 @@ -536,7 +536,7 @@ stages: displayName: Publish Test Results inputs: testResultsFormat: 'VSTest' - testRunTitle: WindowsCompressedMetadata testDesktop + testRunTitle: WindowsCompressedMetadata testDesktop Batch1 mergeTestResults: true testResultsFiles: '*.trx' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' @@ -548,7 +548,7 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows testDesktop binlogs' + ArtifactName: 'Windows testDesktop Batch1 binlogs' ArtifactType: Container parallel: true - task: PublishBuildArtifacts@1 @@ -557,14 +557,14 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows testDesktop process dumps' + ArtifactName: 'Windows testDesktop Batch1 process dumps' ArtifactType: Container parallel: true - task: PublishBuildArtifacts@1 displayName: Publish Test Logs inputs: PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' - ArtifactName: 'Windows testDesktop test logs' + ArtifactName: 'Windows testDesktop Batch1 test logs' publishLocation: Container continueOnError: true condition: always() @@ -575,7 +575,151 @@ stages: displayName: Publish NuGet cache contents inputs: PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Windows testDesktop' + ArtifactName: 'NuGetPackageContents Windows testDesktop Batch1' + publishLocation: Container + continueOnError: true + condition: failed() + + - job: WindowsCompressedMetadata_Desktop_Batch2 + variables: + - name: XUNIT_LOGS + value: $(Build.SourcesDirectory)\artifacts\TestResults\Release + - name: __VSNeverShowWhatsNew + value: 1 + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(_WindowsMachineQueueName) + timeoutInMinutes: 120 + + steps: + - checkout: self + clean: true + + - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch 2 + env: + FSharp_CacheEvictionImmediate: true + DOTNET_DbgEnableMiniDump: 1 + DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. + DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp + NativeToolsOnMachine: true + displayName: Build / Test + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'VSTest' + testRunTitle: WindowsCompressedMetadata testDesktop Batch2 + mergeTestResults: true + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' + continueOnError: true + condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + displayName: Publish BinLog + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + ArtifactName: 'Windows testDesktop Batch2 binlogs' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Dumps + condition: failed() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' + ArtifactName: 'Windows testDesktop Batch2 process dumps' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' + ArtifactName: 'Windows testDesktop Batch2 test logs' + publishLocation: Container + continueOnError: true + condition: always() + - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + - task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents Windows testDesktop Batch2' + publishLocation: Container + continueOnError: true + condition: failed() + + - job: WindowsCompressedMetadata_Desktop_Batch3 + variables: + - name: XUNIT_LOGS + value: $(Build.SourcesDirectory)\artifacts\TestResults\Release + - name: __VSNeverShowWhatsNew + value: 1 + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(_WindowsMachineQueueName) + timeoutInMinutes: 120 + + steps: + - checkout: self + clean: true + + - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch 3 + env: + FSharp_CacheEvictionImmediate: true + DOTNET_DbgEnableMiniDump: 1 + DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. + DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp + NativeToolsOnMachine: true + displayName: Build / Test + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'VSTest' + testRunTitle: WindowsCompressedMetadata testDesktop Batch3 + mergeTestResults: true + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' + continueOnError: true + condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + displayName: Publish BinLog + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + ArtifactName: 'Windows testDesktop Batch3 binlogs' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Dumps + condition: failed() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' + ArtifactName: 'Windows testDesktop Batch3 process dumps' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' + ArtifactName: 'Windows testDesktop Batch3 test logs' + publishLocation: Container + continueOnError: true + condition: always() + - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + - task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents Windows testDesktop Batch3' publishLocation: Container continueOnError: true condition: failed() diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 770e9e88c8a..8a8c4afdab4 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -49,6 +49,7 @@ param ( [switch]$dontUseGlobalNuGetCache = $false, [switch]$warnAsError = $true, [switch][Alias('test')]$testDesktop, + [string]$testDesktopBatch = "", [switch]$testCoreClr, [switch]$testCambridge, [switch]$testCompiler, @@ -121,6 +122,7 @@ function Print-Usage() { Write-Host " -testCompilerService Run FSharpCompilerService unit tests" Write-Host " -testCompilerComponentTests Run FSharpCompilerService component tests" Write-Host " -testDesktop Run tests against full .NET Framework" + Write-Host " -testDesktopBatch <1|2|3> Run a specific batch of the desktop test split (implies -testDesktop)" Write-Host " -testCoreClr Run tests against CoreCLR" Write-Host " -testFSharpCore Run FSharpCore unit tests" Write-Host " -testIntegration Run F# integration tests" @@ -193,6 +195,10 @@ function Process-Arguments() { $script:testEditor = $True } + if ($script:testDesktopBatch -ne "") { + $script:testDesktop = $True + } + if ([System.Boolean]::Parse($script:officialSkipTests)) { $script:testAll = $False $script:testAllButIntegration = $False @@ -605,7 +611,23 @@ try { } if ($testDesktop) { - TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework + if ($testDesktopBatch -ne "") { + $dotnetPath = InitializeDotNetCli + $dotnetExe = Join-Path $dotnetPath "dotnet.exe" + $splitScript = Join-Path $RepoRoot "eng\tests\TestSplit.fsx" + $splitOutput = & $dotnetExe fsi $splitScript $testDesktopBatch + if ($LASTEXITCODE -ne 0) { throw "TestSplit.fsx failed with exit code $LASTEXITCODE" } + foreach ($line in $splitOutput) { + if ($line -match '^dotnet test (\S+) --no-build -c Release\s*(.*)$') { + $proj = $Matches[1] -replace '/', '\' + $projPath = Join-Path $RepoRoot $proj + $settings = $Matches[2].Trim() + TestUsingMSBuild -testProject $projPath -targetFramework $script:desktopTargetFramework -settings $settings + } + } + } else { + TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework + } } if ($testFSharpCore) { diff --git a/eng/tests/TestSplit.fsx b/eng/tests/TestSplit.fsx new file mode 100644 index 00000000000..42980e1f162 --- /dev/null +++ b/eng/tests/TestSplit.fsx @@ -0,0 +1,73 @@ +/// Test split table for parallel CI. +/// Edit the batch assignments below, then run: +/// dotnet fsi eng/tests/TestSplit.fsx +/// to get the dotnet test commands for that batch. + +let totalBatches = 3 +let residualBatch = 2 // uses negation filter; catches unlisted atoms + future namespaces + +// MTP --filter-namespace uses starts-with matching on the test namespace. +// Unlisted atoms go to the residual batch automatically via --filter-not-namespace. + +let componentTestsAtoms = + [// atom batch + "CompilerDirectives", 1 + "CompilerService", 1 + "ErrorMessages", 1 + "FSharpChecker", 1 + "Import", 1 + "Language", 1 + "Miscellaneous", 1 + "XmlComments", 1 + + "Libraries", 2 + "Globalization", 2 + + "EmittedIL", 3 + "Interop", 3 + "InteractiveSession", 3 + "CompilerOptions", 3 + "Conformance", 3 + "Diagnostics", 3 + "Signatures", 3 + "ConstraintSolver", 3 + "Debugger", 3 + "Scripting", 3 + "TypeChecks", 3 + ] + +let otherProjects = + [// project path batch + "tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj", 1 + "tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj", 1 + "tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj", 3 + "tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj", 3 + ] + +// ── filter generation ── + +let batch = + match fsi.CommandLineArgs with + | [| _; n |] -> + let v = int n + if v < 1 || v > totalBatches then failwith $"Batch number must be between 1 and {totalBatches}, got {v}" + v + | _ -> failwith "Usage: dotnet fsi eng/tests/TestSplit.fsx " + +let atomsForBatch b = componentTestsAtoms |> List.filter (fun (_, ba) -> ba = b) |> List.map fst |> List.sort +let otherBatchesAtoms = componentTestsAtoms |> List.filter (fun (_, b) -> b <> batch) |> List.map fst |> List.sort + +let filterArgs = + if batch = residualBatch then + let atoms = otherBatchesAtoms |> String.concat " " + $"--filter-not-namespace {atoms}" + else + let atoms = atomsForBatch batch |> String.concat " " + $"--filter-namespace {atoms}" + +let componentTests = "tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj" + +printfn $"dotnet test {componentTests} --no-build -c Release {filterArgs}" + +for (proj, _) in otherProjects |> List.filter (fun (_, b) -> b = batch) do + printfn $"dotnet test {proj} --no-build -c Release" From 94be73700106652e3a7997493c0bcc5edd7fdafc Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Tue, 24 Feb 2026 07:41:05 +0100 Subject: [PATCH 02/11] Use test matrix for batching --- azure-pipelines-PR.yml | 166 ++++------------------------------------- 1 file changed, 15 insertions(+), 151 deletions(-) diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index fe75f070cf1..b3abd12e12e 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -508,151 +508,15 @@ stages: condition: failed() # Windows With Compressed Metadata Desktop (split into 3 batches) - - job: WindowsCompressedMetadata_Desktop_Batch1 - variables: - - name: XUNIT_LOGS - value: $(Build.SourcesDirectory)\artifacts\TestResults\Release - - name: __VSNeverShowWhatsNew - value: 1 - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(_WindowsMachineQueueName) - timeoutInMinutes: 120 - - steps: - - checkout: self - clean: true - - - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch 1 - env: - FSharp_CacheEvictionImmediate: true - DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. - DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp - NativeToolsOnMachine: true - displayName: Build / Test - - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'VSTest' - testRunTitle: WindowsCompressedMetadata testDesktop Batch1 - mergeTestResults: true - testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' - continueOnError: true - condition: succeededOrFailed() - - - task: PublishBuildArtifacts@1 - displayName: Publish BinLog - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows testDesktop Batch1 binlogs' - ArtifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Publish Dumps - condition: failed() - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows testDesktop Batch1 process dumps' - ArtifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' - ArtifactName: 'Windows testDesktop Batch1 test logs' - publishLocation: Container - continueOnError: true - condition: always() - - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj - displayName: Dump NuGet cache contents - condition: failed() - - task: PublishBuildArtifacts@1 - displayName: Publish NuGet cache contents - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Windows testDesktop Batch1' - publishLocation: Container - continueOnError: true - condition: failed() - - - job: WindowsCompressedMetadata_Desktop_Batch2 - variables: - - name: XUNIT_LOGS - value: $(Build.SourcesDirectory)\artifacts\TestResults\Release - - name: __VSNeverShowWhatsNew - value: 1 - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(_WindowsMachineQueueName) - timeoutInMinutes: 120 - - steps: - - checkout: self - clean: true - - - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch 2 - env: - FSharp_CacheEvictionImmediate: true - DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. - DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp - NativeToolsOnMachine: true - displayName: Build / Test - - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'VSTest' - testRunTitle: WindowsCompressedMetadata testDesktop Batch2 - mergeTestResults: true - testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' - continueOnError: true - condition: succeededOrFailed() - - - task: PublishBuildArtifacts@1 - displayName: Publish BinLog - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows testDesktop Batch2 binlogs' - ArtifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Publish Dumps - condition: failed() - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows testDesktop Batch2 process dumps' - ArtifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' - ArtifactName: 'Windows testDesktop Batch2 test logs' - publishLocation: Container - continueOnError: true - condition: always() - - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj - displayName: Dump NuGet cache contents - condition: failed() - - task: PublishBuildArtifacts@1 - displayName: Publish NuGet cache contents - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Windows testDesktop Batch2' - publishLocation: Container - continueOnError: true - condition: failed() - - - job: WindowsCompressedMetadata_Desktop_Batch3 + - job: WindowsCompressedMetadata_Desktop + strategy: + matrix: + Batch1: + batchNumber: 1 + Batch2: + batchNumber: 2 + Batch3: + batchNumber: 3 variables: - name: XUNIT_LOGS value: $(Build.SourcesDirectory)\artifacts\TestResults\Release @@ -667,7 +531,7 @@ stages: - checkout: self clean: true - - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch 3 + - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch $(batchNumber) env: FSharp_CacheEvictionImmediate: true DOTNET_DbgEnableMiniDump: 1 @@ -680,7 +544,7 @@ stages: displayName: Publish Test Results inputs: testResultsFormat: 'VSTest' - testRunTitle: WindowsCompressedMetadata testDesktop Batch3 + testRunTitle: WindowsCompressedMetadata testDesktop Batch$(batchNumber) mergeTestResults: true testResultsFiles: '*.trx' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' @@ -692,7 +556,7 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows testDesktop Batch3 binlogs' + ArtifactName: 'Windows testDesktop Batch$(batchNumber) binlogs' ArtifactType: Container parallel: true - task: PublishBuildArtifacts@1 @@ -701,14 +565,14 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows testDesktop Batch3 process dumps' + ArtifactName: 'Windows testDesktop Batch$(batchNumber) process dumps' ArtifactType: Container parallel: true - task: PublishBuildArtifacts@1 displayName: Publish Test Logs inputs: PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' - ArtifactName: 'Windows testDesktop Batch3 test logs' + ArtifactName: 'Windows testDesktop Batch$(batchNumber) test logs' publishLocation: Container continueOnError: true condition: always() @@ -719,7 +583,7 @@ stages: displayName: Publish NuGet cache contents inputs: PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Windows testDesktop Batch3' + ArtifactName: 'NuGetPackageContents Windows testDesktop Batch$(batchNumber)' publishLocation: Container continueOnError: true condition: failed() From d80ef7b86551d876a588f91a8652a2843c696427 Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Thu, 26 Feb 2026 14:45:03 +0100 Subject: [PATCH 03/11] Use spekt for test results --- eng/Build.ps1 | 12 ++-- eng/Versions.props | 1 + eng/build.sh | 10 ++-- eng/templates/batched-test-job.yml | 93 ++++++++++++++++++++++++++++++ eng/tests/TestSplit.fsx | 1 + tests/Directory.Build.props | 2 + 6 files changed, 106 insertions(+), 13 deletions(-) create mode 100644 eng/templates/batched-test-job.yml diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 8a8c4afdab4..8f858b21876 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -380,14 +380,10 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str # MTP requires --solution flag for .sln files $testTarget = if ($testProject.EndsWith('.sln')) { "--solution ""$testProject""" } else { "--project ""$testProject""" } - # For solutions, omit --report-xunit-trx-filename so each test assembly generates a unique .trx file. - # With a static filename, all assemblies overwrite the same file and only the last one's results survive. - if ($testProject.EndsWith('.sln')) { - $reportArgs = "--report-xunit-trx" - } else { - $testLogFileName = "${projectName}_${targetFramework}.trx" - $reportArgs = "--report-xunit-trx --report-xunit-trx-filename ""$testLogFileName""" - } + # Xunit XML report via XunitXml.TestLogger with CI-friendly filenames + $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" } + $xunitLogFileName = "{assembly}.{framework}.${jobName}.xml" + $reportArgs = "--report-spekt-xunit --report-spekt-xunit-filename ""$xunitLogFileName""" $test_args = "test $testTarget -c $configuration -f $targetFramework $reportArgs --results-directory ""$testResultsDir"" /bl:$testBinLogPath" # MTP HangDump extension replaces VSTest --blame-hang-timeout diff --git a/eng/Versions.props b/eng/Versions.props index 0032205fef2..8c9852c7620 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -170,6 +170,7 @@ 13.0.3 3.2.2 3.2.2 + 8.0.0 diff --git a/eng/build.sh b/eng/build.sh index 06df1423012..913be4b1690 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -233,17 +233,17 @@ function Test() { testresultsdir="$artifacts_dir/TestResults/$configuration" # MTP requires --solution flag for .sln files - # For solutions, omit --report-xunit-trx-filename so each test assembly generates a unique .trx file. - # With a static filename, all assemblies overwrite the same file and only the last one's results survive. if [[ "$testproject" == *.sln ]]; then testtarget="--solution" - reportargs="--report-xunit-trx" else testtarget="--project" - testlogfilename="${projectname}_${targetframework}.trx" - reportargs="--report-xunit-trx --report-xunit-trx-filename $testlogfilename" fi + # Xunit XML report via XunitXml.TestLogger with CI-friendly filenames + jobname="${SYSTEM_JOBNAME:-local}" + xunitlogfilename="{assembly}.{framework}.${jobname}.xml" + reportargs="--report-spekt-xunit --report-spekt-xunit-filename $xunitlogfilename" + args=(test $testtarget "$testproject" --no-build -c "$configuration" -f "$targetframework" $reportargs --results-directory "$testresultsdir" --hangdump --hangdump-timeout 5m --hangdump-type Full) "$DOTNET_INSTALL_DIR/dotnet" "${args[@]}" || exit $? diff --git a/eng/templates/batched-test-job.yml b/eng/templates/batched-test-job.yml new file mode 100644 index 00000000000..acc2a842624 --- /dev/null +++ b/eng/templates/batched-test-job.yml @@ -0,0 +1,93 @@ +# Template for running test jobs split into parallel batches via strategy:matrix. +# Place this alongside (not inside) the Arcade jobs.yml template reference, +# since Arcade's ${{ each job }} loop cannot expand nested template references. + +parameters: + jobName: '' + pool: {} + timeoutInMinutes: 120 + buildCommand: '' # Full build/test command including $(batchNumber) + buildEnv: {} + batches: [1, 2, 3] + variables: [] + testRunTitlePrefix: '' # e.g. 'WindowsCompressedMetadata testDesktop' + artifactNamePrefix: '' # e.g. 'Windows testDesktop' + configuration: 'Release' + publishBinLog: false + binLogPath: '' + publishDumps: false + +jobs: +- job: ${{ parameters.jobName }} + strategy: + matrix: + ${{ each batch in parameters.batches }}: + Batch${{ batch }}: + batchNumber: ${{ batch }} + pool: ${{ parameters.pool }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + variables: + - ${{ each var in parameters.variables }}: + - name: ${{ var.name }} + value: ${{ var.value }} + steps: + - checkout: self + clean: true + + - script: ${{ parameters.buildCommand }} + env: ${{ parameters.buildEnv }} + displayName: Build / Test + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'VSTest' + testRunTitle: ${{ parameters.testRunTitlePrefix }} Batch$(batchNumber) + mergeTestResults: true + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}' + continueOnError: true + condition: succeededOrFailed() + + - ${{ if parameters.publishBinLog }}: + - task: PublishBuildArtifacts@1 + displayName: Publish BinLog + continueOnError: true + inputs: + PathToPublish: ${{ parameters.binLogPath }} + ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) binlogs' + ArtifactType: Container + parallel: true + + - ${{ if parameters.publishDumps }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Dumps + condition: failed() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/${{ parameters.configuration }}' + ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) process dumps' + ArtifactType: Container + parallel: true + + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}' + ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) test logs' + publishLocation: Container + continueOnError: true + condition: always() + + - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + + - task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents ${{ parameters.artifactNamePrefix }} Batch$(batchNumber)' + publishLocation: Container + continueOnError: true + condition: failed() diff --git a/eng/tests/TestSplit.fsx b/eng/tests/TestSplit.fsx index 42980e1f162..b5389c658fd 100644 --- a/eng/tests/TestSplit.fsx +++ b/eng/tests/TestSplit.fsx @@ -42,6 +42,7 @@ let otherProjects = "tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj", 1 "tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj", 3 "tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj", 3 + "tests/fsharp/FSharpSuite.Tests.fsproj", 3 ] // ── filter generation ── diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index bf1b81f2bef..dc7ed710a3a 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -13,6 +13,8 @@ + + From 969a0991bd624a8808090aacf71c25068aceb8b8 Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Thu, 26 Feb 2026 16:07:56 +0100 Subject: [PATCH 04/11] Add ObjectModel nuget --- tests/Directory.Build.props | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index dc7ed710a3a..ccc7e44ffa3 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -15,6 +15,8 @@ + + From c1b3b00de4fdbf4176e9bcfea354503c9aea41c4 Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Thu, 26 Feb 2026 17:46:40 +0100 Subject: [PATCH 05/11] Fix missing references --- tests/EndToEndBuildTests/Directory.Build.props | 2 ++ .../tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj | 2 ++ vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj | 2 ++ 3 files changed, 6 insertions(+) diff --git a/tests/EndToEndBuildTests/Directory.Build.props b/tests/EndToEndBuildTests/Directory.Build.props index 496de8645f5..f97db4e1684 100644 --- a/tests/EndToEndBuildTests/Directory.Build.props +++ b/tests/EndToEndBuildTests/Directory.Build.props @@ -7,6 +7,8 @@ 3.2.2 3.2.2 2.0.2 + 8.0.0 + 17.14.1 diff --git a/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj b/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj index 284ec6eeb1d..52ba8624a30 100644 --- a/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj +++ b/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj @@ -108,6 +108,8 @@ + + diff --git a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj index 009d4f0a8fa..cf8cc25e837 100644 --- a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj +++ b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj @@ -128,6 +128,8 @@ + + From b36d8a4afa505b5d9f1d9fbf4e71aa91b649436c Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:21:19 +0100 Subject: [PATCH 06/11] Change test result format --- azure-pipelines-PR.yml | 24 ++++++++++++------------ azure-pipelines.yml | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index b3abd12e12e..57dc682614a 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -267,10 +267,10 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: WindowsNoRealsig_testCoreclr mergeTestResults: true - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' condition: succeededOrFailed() @@ -314,10 +314,10 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: WindowsNoRealsig_testDesktop mergeTestResults: true - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' condition: succeededOrFailed() continueOnError: true @@ -462,10 +462,10 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: WindowsCompressedMetadata $(_testKind) $(transparentCompiler) mergeTestResults: true - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)' continueOnError: true condition: succeededOrFailed() @@ -543,10 +543,10 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: WindowsCompressedMetadata testDesktop Batch$(batchNumber) mergeTestResults: true - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' continueOnError: true condition: succeededOrFailed() @@ -616,9 +616,9 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' + testResultsFormat: 'XUnit' testRunTitle: Linux - testResultsFiles: '*.trx' + testResultsFiles: '*.xml' mergeTestResults: true searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' continueOnError: true @@ -661,8 +661,8 @@ stages: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' + testResultsFormat: 'XUnit' + testResultsFiles: '*.xml' testRunTitle: MacOS mergeTestResults: true searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 170d26397e3..e4f0c294362 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -152,8 +152,8 @@ extends: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' + testResultsFormat: 'XUnit' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' continueOnError: true condition: ne(variables['SkipTests'], 'true') From de4717379a65dd469a872507f5929c0831c6fabd Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Tue, 3 Mar 2026 22:49:31 +0100 Subject: [PATCH 07/11] Use steps template --- azure-pipelines-PR.yml | 73 +++++----------------- eng/templates/batched-test-job.yml | 93 ---------------------------- eng/templates/batched-test-steps.yml | 88 ++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 152 deletions(-) delete mode 100644 eng/templates/batched-test-job.yml create mode 100644 eng/templates/batched-test-steps.yml diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index 57dc682614a..6baeba05b1c 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -528,65 +528,20 @@ stages: timeoutInMinutes: 120 steps: - - checkout: self - clean: true - - - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch $(batchNumber) - env: - FSharp_CacheEvictionImmediate: true - DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. - DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp - NativeToolsOnMachine: true - displayName: Build / Test - - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'XUnit' - testRunTitle: WindowsCompressedMetadata testDesktop Batch$(batchNumber) - mergeTestResults: true - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' - continueOnError: true - condition: succeededOrFailed() - - - task: PublishBuildArtifacts@1 - displayName: Publish BinLog - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows testDesktop Batch$(batchNumber) binlogs' - ArtifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Publish Dumps - condition: failed() - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows testDesktop Batch$(batchNumber) process dumps' - ArtifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' - ArtifactName: 'Windows testDesktop Batch$(batchNumber) test logs' - publishLocation: Container - continueOnError: true - condition: always() - - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj - displayName: Dump NuGet cache contents - condition: failed() - - task: PublishBuildArtifacts@1 - displayName: Publish NuGet cache contents - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Windows testDesktop Batch$(batchNumber)' - publishLocation: Container - continueOnError: true - condition: failed() + - template: /eng/templates/batched-test-steps.yml + parameters: + buildCommand: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch $(batchNumber) + buildEnv: + FSharp_CacheEvictionImmediate: true + DOTNET_DbgEnableMiniDump: 1 + DOTNET_DbgMiniDumpType: 3 + DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp + NativeToolsOnMachine: true + testRunTitlePrefix: 'WindowsCompressedMetadata testDesktop' + artifactNamePrefix: 'Windows testDesktop' + publishBinLog: true + binLogPath: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + publishDumps: true # Mock official build - job: MockOfficial diff --git a/eng/templates/batched-test-job.yml b/eng/templates/batched-test-job.yml deleted file mode 100644 index acc2a842624..00000000000 --- a/eng/templates/batched-test-job.yml +++ /dev/null @@ -1,93 +0,0 @@ -# Template for running test jobs split into parallel batches via strategy:matrix. -# Place this alongside (not inside) the Arcade jobs.yml template reference, -# since Arcade's ${{ each job }} loop cannot expand nested template references. - -parameters: - jobName: '' - pool: {} - timeoutInMinutes: 120 - buildCommand: '' # Full build/test command including $(batchNumber) - buildEnv: {} - batches: [1, 2, 3] - variables: [] - testRunTitlePrefix: '' # e.g. 'WindowsCompressedMetadata testDesktop' - artifactNamePrefix: '' # e.g. 'Windows testDesktop' - configuration: 'Release' - publishBinLog: false - binLogPath: '' - publishDumps: false - -jobs: -- job: ${{ parameters.jobName }} - strategy: - matrix: - ${{ each batch in parameters.batches }}: - Batch${{ batch }}: - batchNumber: ${{ batch }} - pool: ${{ parameters.pool }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - variables: - - ${{ each var in parameters.variables }}: - - name: ${{ var.name }} - value: ${{ var.value }} - steps: - - checkout: self - clean: true - - - script: ${{ parameters.buildCommand }} - env: ${{ parameters.buildEnv }} - displayName: Build / Test - - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'VSTest' - testRunTitle: ${{ parameters.testRunTitlePrefix }} Batch$(batchNumber) - mergeTestResults: true - testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}' - continueOnError: true - condition: succeededOrFailed() - - - ${{ if parameters.publishBinLog }}: - - task: PublishBuildArtifacts@1 - displayName: Publish BinLog - continueOnError: true - inputs: - PathToPublish: ${{ parameters.binLogPath }} - ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) binlogs' - ArtifactType: Container - parallel: true - - - ${{ if parameters.publishDumps }}: - - task: PublishBuildArtifacts@1 - displayName: Publish Dumps - condition: failed() - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/${{ parameters.configuration }}' - ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) process dumps' - ArtifactType: Container - parallel: true - - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}' - ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) test logs' - publishLocation: Container - continueOnError: true - condition: always() - - - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj - displayName: Dump NuGet cache contents - condition: failed() - - - task: PublishBuildArtifacts@1 - displayName: Publish NuGet cache contents - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents ${{ parameters.artifactNamePrefix }} Batch$(batchNumber)' - publishLocation: Container - continueOnError: true - condition: failed() diff --git a/eng/templates/batched-test-steps.yml b/eng/templates/batched-test-steps.yml new file mode 100644 index 00000000000..712d5e3219f --- /dev/null +++ b/eng/templates/batched-test-steps.yml @@ -0,0 +1,88 @@ +# Steps template for batched test jobs. +# Use inside a job that has strategy:matrix with a batchNumber variable. +# +# Example usage: +# - job: MyTestJob +# strategy: +# matrix: +# Batch1: { batchNumber: 1 } +# Batch2: { batchNumber: 2 } +# Batch3: { batchNumber: 3 } +# steps: +# - template: /eng/templates/batched-test-steps.yml +# parameters: +# buildCommand: eng\CIBuild.cmd -testDesktopBatch $(batchNumber) +# testRunTitlePrefix: 'MyJob testDesktop' +# artifactNamePrefix: 'Windows testDesktop' + +parameters: + buildCommand: '' + buildEnv: {} + configuration: 'Release' + testRunTitlePrefix: '' + artifactNamePrefix: '' + publishBinLog: false + binLogPath: '' + publishDumps: false + +steps: +- checkout: self + clean: true + +- script: ${{ parameters.buildCommand }} + env: ${{ parameters.buildEnv }} + displayName: Build / Test + +- task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'XUnit' + testRunTitle: ${{ parameters.testRunTitlePrefix }} Batch$(batchNumber) + mergeTestResults: true + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}' + continueOnError: true + condition: succeededOrFailed() + +- ${{ if parameters.publishBinLog }}: + - task: PublishBuildArtifacts@1 + displayName: Publish BinLog + continueOnError: true + inputs: + PathToPublish: ${{ parameters.binLogPath }} + ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) binlogs' + ArtifactType: Container + parallel: true + +- ${{ if parameters.publishDumps }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Dumps + condition: failed() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/${{ parameters.configuration }}' + ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) process dumps' + ArtifactType: Container + parallel: true + +- task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}' + ArtifactName: '${{ parameters.artifactNamePrefix }} Batch$(batchNumber) test logs' + publishLocation: Container + continueOnError: true + condition: always() + +- script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + +- task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents ${{ parameters.artifactNamePrefix }} Batch$(batchNumber)' + publishLocation: Container + continueOnError: true + condition: failed() From 82575a33f33d57e45b5b5f83b8e3f43e1e501a41 Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Wed, 4 Mar 2026 18:56:40 +0100 Subject: [PATCH 08/11] Batch MacOS job --- azure-pipelines-PR.yml | 52 +++++++++++++---------------------------- eng/Build.ps1 | 2 +- eng/build.sh | 36 ++++++++++++++++++++++------ eng/tests/TestSplit.fsx | 31 +++++++++++++++--------- 4 files changed, 66 insertions(+), 55 deletions(-) diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index dfaf1ed3cce..2ebe6fb1f8d 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -601,8 +601,16 @@ stages: continueOnError: true condition: failed() - # MacOS + # MacOS (split into 3 batches) - job: MacOS + strategy: + matrix: + Batch1: + batchNumber: 1 + Batch2: + batchNumber: 2 + Batch3: + batchNumber: 3 pool: vmImage: macos-latest timeoutInMinutes: 120 @@ -610,41 +618,13 @@ stages: - name: _SignType value: Test steps: - - checkout: self - clean: true - - script: ./eng/cibuild.sh --configuration $(_BuildConfig) --testcoreclr - env: - COMPlus_DefaultStackSize: 1000000 - displayName: Build / Test - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'XUnit' - testResultsFiles: '*.xml' - testRunTitle: MacOS - mergeTestResults: true - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - continueOnError: true - condition: succeededOrFailed() - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - ArtifactName: 'MacOS $(_BuildConfig) test logs' - publishLocation: Container - continueOnError: true - condition: failed() - - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj - displayName: Dump NuGet cache contents - condition: failed() - - task: PublishBuildArtifacts@1 - displayName: Publish NuGet cache contents - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Mac' - publishLocation: Container - continueOnError: true - condition: failed() + - template: /eng/templates/batched-test-steps.yml + parameters: + buildCommand: ./eng/cibuild.sh --configuration $(_BuildConfig) --testcoreclrbatch $(batchNumber) + buildEnv: + COMPlus_DefaultStackSize: 1000000 + testRunTitlePrefix: 'MacOS' + artifactNamePrefix: 'MacOS' # End to end build - job: EndToEndBuildTests diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 8f858b21876..5951d2c2989 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -611,7 +611,7 @@ try { $dotnetPath = InitializeDotNetCli $dotnetExe = Join-Path $dotnetPath "dotnet.exe" $splitScript = Join-Path $RepoRoot "eng\tests\TestSplit.fsx" - $splitOutput = & $dotnetExe fsi $splitScript $testDesktopBatch + $splitOutput = & $dotnetExe fsi $splitScript $testDesktopBatch desktop if ($LASTEXITCODE -ne 0) { throw "TestSplit.fsx failed with exit code $LASTEXITCODE" } foreach ($line in $splitOutput) { if ($line -match '^dotnet test (\S+) --no-build -c Release\s*(.*)$') { diff --git a/eng/build.sh b/eng/build.sh index 913be4b1690..eca9450a1ee 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -63,6 +63,7 @@ pack=false publish=false sign=false test_core_clr=false +test_core_clr_batch="" test_compilercomponent_tests=false test_benchmarks=false test_scripting=false @@ -142,6 +143,11 @@ while [[ $# > 0 ]]; do --testcoreclr|--test|-t) test_core_clr=true ;; + --testcoreclrbatch) + test_core_clr=true + test_core_clr_batch=$2 + shift + ;; --testcompilercomponenttests) test_compilercomponent_tests=true ;; @@ -204,6 +210,7 @@ function Test() { BuildMessage="Error running tests" testproject="" targetframework="" + extraargs="" while [[ $# > 0 ]]; do opt="$(echo "$1" | awk '{print tolower($0)}')" case "$opt" in @@ -215,6 +222,10 @@ function Test() { targetframework=$2 shift ;; + --extraargs) + extraargs=$2 + shift + ;; *) echo "Invalid argument: $1" exit 1 @@ -244,7 +255,7 @@ function Test() { xunitlogfilename="{assembly}.{framework}.${jobname}.xml" reportargs="--report-spekt-xunit --report-spekt-xunit-filename $xunitlogfilename" - args=(test $testtarget "$testproject" --no-build -c "$configuration" -f "$targetframework" $reportargs --results-directory "$testresultsdir" --hangdump --hangdump-timeout 5m --hangdump-type Full) + args=(test $testtarget "$testproject" --no-build -c "$configuration" -f "$targetframework" $reportargs --results-directory "$testresultsdir" --hangdump --hangdump-timeout 5m --hangdump-type Full $extraargs) "$DOTNET_INSTALL_DIR/dotnet" "${args[@]}" || exit $? } @@ -362,12 +373,23 @@ BuildSolution if [[ "$test_core_clr" == true ]]; then coreclrtestframework=$tfm - # Note: FSharp.Test.Utilities is a utility library, not a test project. Its tests are disabled due to xUnit3 API incompatibilities. - Test --testproject "$repo_root/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj" --targetframework $coreclrtestframework - Test --testproject "$repo_root/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj" --targetframework $coreclrtestframework - Test --testproject "$repo_root/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj" --targetframework $coreclrtestframework - Test --testproject "$repo_root/tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj" --targetframework $coreclrtestframework - Test --testproject "$repo_root/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj" --targetframework $coreclrtestframework + + if [[ "$test_core_clr_batch" != "" ]]; then + # Run batched: use TestSplit.fsx to get the commands for this batch + while IFS= read -r line; do + # Extract project path and extra filter args from each line + project=$(echo "$line" | sed 's/^dotnet test //' | sed 's/ --no-build.*//') + filterargs=$(echo "$line" | sed 's/^dotnet test [^ ]* --no-build -c Release *//') + Test --testproject "$repo_root/$project" --targetframework $coreclrtestframework --extraargs "$filterargs" + done < <("$DOTNET_INSTALL_DIR/dotnet" fsi "$scriptroot/tests/TestSplit.fsx" "$test_core_clr_batch" coreclr) + else + # Run all tests without batching + Test --testproject "$repo_root/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj" --targetframework $coreclrtestframework + Test --testproject "$repo_root/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj" --targetframework $coreclrtestframework + Test --testproject "$repo_root/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj" --targetframework $coreclrtestframework + Test --testproject "$repo_root/tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj" --targetframework $coreclrtestframework + Test --testproject "$repo_root/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj" --targetframework $coreclrtestframework + fi fi if [[ "$test_compilercomponent_tests" == true ]]; then diff --git a/eng/tests/TestSplit.fsx b/eng/tests/TestSplit.fsx index b5389c658fd..3d88e5f636a 100644 --- a/eng/tests/TestSplit.fsx +++ b/eng/tests/TestSplit.fsx @@ -1,7 +1,8 @@ /// Test split table for parallel CI. /// Edit the batch assignments below, then run: -/// dotnet fsi eng/tests/TestSplit.fsx +/// dotnet fsi eng/tests/TestSplit.fsx [desktop|coreclr] /// to get the dotnet test commands for that batch. +/// The platform argument controls which projects are included (default: all). let totalBatches = 3 let residualBatch = 2 // uses negation filter; catches unlisted atoms + future namespaces @@ -36,24 +37,32 @@ let componentTestsAtoms = "TypeChecks", 3 ] +// Platform tags: "all" = both desktop and coreclr, "desktop" = net472 only let otherProjects = - [// project path batch - "tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj", 1 - "tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj", 1 - "tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj", 3 - "tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj", 3 - "tests/fsharp/FSharpSuite.Tests.fsproj", 3 + [// project path batch platform + "tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj", 1, "all" + "tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj", 1, "all" + "tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj", 3, "all" + "tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj", 3, "all" + "tests/fsharp/FSharpSuite.Tests.fsproj", 3, "desktop" ] // ── filter generation ── -let batch = +let batch, platform = match fsi.CommandLineArgs with | [| _; n |] -> let v = int n if v < 1 || v > totalBatches then failwith $"Batch number must be between 1 and {totalBatches}, got {v}" - v - | _ -> failwith "Usage: dotnet fsi eng/tests/TestSplit.fsx " + v, "all" + | [| _; n; p |] -> + let v = int n + if v < 1 || v > totalBatches then failwith $"Batch number must be between 1 and {totalBatches}, got {v}" + v, p + | _ -> failwith "Usage: dotnet fsi eng/tests/TestSplit.fsx [desktop|coreclr]" + +let matchesPlatform tag = + tag = "all" || tag = platform || platform = "all" let atomsForBatch b = componentTestsAtoms |> List.filter (fun (_, ba) -> ba = b) |> List.map fst |> List.sort let otherBatchesAtoms = componentTestsAtoms |> List.filter (fun (_, b) -> b <> batch) |> List.map fst |> List.sort @@ -70,5 +79,5 @@ let componentTests = "tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.Compo printfn $"dotnet test {componentTests} --no-build -c Release {filterArgs}" -for (proj, _) in otherProjects |> List.filter (fun (_, b) -> b = batch) do +for (proj, _, tag) in otherProjects |> List.filter (fun (_, b, tag) -> b = batch && matchesPlatform tag) do printfn $"dotnet test {proj} --no-build -c Release" From 37ffa03d4be1363d0debc1b0845dd39c678e9e03 Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Thu, 5 Mar 2026 11:18:32 +0100 Subject: [PATCH 09/11] Rebalance the split --- eng/tests/TestSplit.fsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/tests/TestSplit.fsx b/eng/tests/TestSplit.fsx index 3d88e5f636a..a26a93ef7af 100644 --- a/eng/tests/TestSplit.fsx +++ b/eng/tests/TestSplit.fsx @@ -21,12 +21,12 @@ let componentTestsAtoms = "Miscellaneous", 1 "XmlComments", 1 + "EmittedIL", 2 + "Interop", 2 "Libraries", 2 "Globalization", 2 + "InteractiveSession", 2 - "EmittedIL", 3 - "Interop", 3 - "InteractiveSession", 3 "CompilerOptions", 3 "Conformance", 3 "Diagnostics", 3 @@ -41,8 +41,8 @@ let componentTestsAtoms = let otherProjects = [// project path batch platform "tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj", 1, "all" - "tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj", 1, "all" - "tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj", 3, "all" + "tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj", 2, "all" + "tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj", 2, "all" "tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj", 3, "all" "tests/fsharp/FSharpSuite.Tests.fsproj", 3, "desktop" ] From c5d39b8253cb04bddf7464701c729f2bb0db9b83 Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Fri, 6 Mar 2026 17:07:19 +0100 Subject: [PATCH 10/11] Rebalance the split --- eng/tests/TestSplit.fsx | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/eng/tests/TestSplit.fsx b/eng/tests/TestSplit.fsx index a26a93ef7af..c2e2bc884f1 100644 --- a/eng/tests/TestSplit.fsx +++ b/eng/tests/TestSplit.fsx @@ -5,7 +5,7 @@ /// The platform argument controls which projects are included (default: all). let totalBatches = 3 -let residualBatch = 2 // uses negation filter; catches unlisted atoms + future namespaces +let residualBatch = 3 // uses negation filter; catches unlisted atoms + future namespaces // MTP --filter-namespace uses starts-with matching on the test namespace. // Unlisted atoms go to the residual batch automatically via --filter-not-namespace. @@ -64,8 +64,24 @@ let batch, platform = let matchesPlatform tag = tag = "all" || tag = platform || platform = "all" -let atomsForBatch b = componentTestsAtoms |> List.filter (fun (_, ba) -> ba = b) |> List.map fst |> List.sort -let otherBatchesAtoms = componentTestsAtoms |> List.filter (fun (_, b) -> b <> batch) |> List.map fst |> List.sort +let expandAtom atom = + [ atom + $"FSharp.Compiler.ComponentTests.{atom}" + $"ComponentTests.{atom}" ] + +let atomsForBatch b = + componentTestsAtoms + |> List.filter (fun (_, ba) -> ba = b) + |> List.collect (fst >> expandAtom) + |> List.distinct + |> List.sort + +let otherBatchesAtoms = + componentTestsAtoms + |> List.filter (fun (_, b) -> b <> batch) + |> List.collect (fst >> expandAtom) + |> List.distinct + |> List.sort let filterArgs = if batch = residualBatch then From 0ef707b4b0fc4c033f9377705ab732c47dc4696c Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Wed, 11 Mar 2026 19:51:25 +0100 Subject: [PATCH 11/11] Add error handling for TestSplit.fsx --- eng/build.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eng/build.sh b/eng/build.sh index eca9450a1ee..6be36adfa26 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -376,12 +376,17 @@ if [[ "$test_core_clr" == true ]]; then if [[ "$test_core_clr_batch" != "" ]]; then # Run batched: use TestSplit.fsx to get the commands for this batch + splitOutput=$("$DOTNET_INSTALL_DIR/dotnet" fsi "$scriptroot/tests/TestSplit.fsx" "$test_core_clr_batch" coreclr) + if [[ $? -ne 0 ]]; then + echo "TestSplit.fsx failed with exit code $?" + ExitWithExitCode 1 + fi while IFS= read -r line; do # Extract project path and extra filter args from each line project=$(echo "$line" | sed 's/^dotnet test //' | sed 's/ --no-build.*//') filterargs=$(echo "$line" | sed 's/^dotnet test [^ ]* --no-build -c Release *//') Test --testproject "$repo_root/$project" --targetframework $coreclrtestframework --extraargs "$filterargs" - done < <("$DOTNET_INSTALL_DIR/dotnet" fsi "$scriptroot/tests/TestSplit.fsx" "$test_core_clr_batch" coreclr) + done <<< "$splitOutput" else # Run all tests without batching Test --testproject "$repo_root/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj" --targetframework $coreclrtestframework