Enable VS integration tests on CI

azure-pipelines-PR.yml

@@ -448,10 +448,10 @@ stages:
                 _testKind: testCoreclr
                 TEST_TRANSPARENT_COMPILER: 1 # Pipeline variable will map to env var.
                 transparentCompiler: TransparentCompiler
-              # inttests_release:
-              #   _configuration: Release
-              #   _testKind: testIntegration
-              #   setupVsHive: true
+              inttests_release:
+                _configuration: Release
+                _testKind: testIntegration
+                setupVsHive: true
           steps:
           - checkout: self
             clean: true

eng/Build.ps1

@@ -402,6 +402,41 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
     Exec-Console $dotnetExe $test_args
 }
 
+# Runs a test assembly via the xUnit v2 console runner, bypassing `dotnet test` (and therefore
+# the repo-wide Microsoft.Testing.Platform gate declared in global.json). Used only for projects
+# whose harness cannot run under MTP -- today that is FSharp.Editor.IntegrationTests, which
+# depends on Microsoft.VisualStudio.Extensibility.Testing.Xunit (VS-hive launcher, xUnit-v2-locked).
+# The runner is restored via a <PackageDownload Include="xunit.runner.console" .../> on the
+# test project, so it is present in the NuGet global packages folder after a normal slnx restore.
+function TestUsingXUnitConsole([string] $testProject, [string] $targetFramework) {
+
+    $projectName = [System.IO.Path]::GetFileNameWithoutExtension($testProject)
+    $assemblyPath = "$ArtifactsDir\bin\$projectName\$configuration\$targetFramework\$projectName.dll"
+    if (-not (Test-Path $assemblyPath)) {
+        throw "Test assembly not found at $assemblyPath. Was $projectName built before -testIntegration was invoked?"
+    }
+
+    # Get-PackageVersion (eng\build-utils.ps1) reads <XunitRunnerConsoleV2Version> from eng\Versions.props,
+    # and Get-PackageDir resolves the NuGet cache path. Defensive Trim() covers accidental whitespace in the value.
+    $runnerVersion = (Get-PackageVersion "XunitRunnerConsoleV2").Trim()
+    $xunitConsole = Join-Path (Get-PackageDir "xunit.runner.console" $runnerVersion) "tools\net472\xunit.console.exe"
+    if (-not (Test-Path $xunitConsole)) {
+        throw "xunit.console.exe not found at $xunitConsole. Ensure restore of $projectName ran first (PackageDownload of xunit.runner.console v$runnerVersion)."
+    }
+
+    $testResultsDir = "$ArtifactsDir\TestResults\$configuration"
+    Create-Directory $testResultsDir
+    $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" }
+    $resultsXml = Join-Path $testResultsDir "$projectName.$targetFramework.$jobName.xml"
+
+    # -parallel none / -noshadow mirror the project's xunit.runner.json (parallelizeTestCollections=false, shadowCopy=false).
+    $xunit_args = """$assemblyPath"" -xml ""$resultsXml"" -parallel none -noshadow -nologo"
+
+    Write-Host("$xunitConsole $xunit_args")
+
+    Exec-Console $xunitConsole $xunit_args
+}
+
 function Prepare-TempDir() {
     Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.props") $TempDir
     Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.targets") $TempDir
@@ -665,8 +700,8 @@ try {
         TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\UnitTests\VisualFSharp.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework
     }
 
-    if ($testIntegration) {
-        TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework
+    if ($testIntegration -and -not $noVisualStudio) {
+        TestUsingXUnitConsole -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework
     }
 
     if ($testAOT) {

eng/Versions.props

@@ -130,7 +130,7 @@
     <MicrosoftVisualStudioPlatformVSEditorVersion>$(VisualStudioEditorPackagesVersion)</MicrosoftVisualStudioPlatformVSEditorVersion>
     <MicrosoftVisualStudioTextUIWpfVersion>$(VisualStudioEditorPackagesVersion)</MicrosoftVisualStudioTextUIWpfVersion>
     <NuGetVisualStudioVersion>17.14.0</NuGetVisualStudioVersion>
-    <MicrosoftVisualStudioExtensibilityTestingVersion>0.1.800-beta</MicrosoftVisualStudioExtensibilityTestingVersion>
+    <MicrosoftVisualStudioExtensibilityTestingVersion>5.3.0-2.26055.8</MicrosoftVisualStudioExtensibilityTestingVersion>
     <MicrosoftVisualStudioExtensibilityTestingSourceGeneratorVersion>$(MicrosoftVisualStudioExtensibilityTestingVersion)</MicrosoftVisualStudioExtensibilityTestingSourceGeneratorVersion>
 
     <!-- Visual Studio Threading packages -->
@@ -161,6 +161,8 @@
     <NewtonsoftJsonVersion>13.0.4</NewtonsoftJsonVersion>
     <XunitVersion>3.2.2</XunitVersion>
     <XunitRunnerConsoleVersion>3.2.2</XunitRunnerConsoleVersion>
+    <!-- xUnit v2 runner, only consumed by FSharp.Editor.IntegrationTests (VS-hive harness is xUnit-v2-locked). -->
+    <XunitRunnerConsoleV2Version>2.9.3</XunitRunnerConsoleV2Version>
     <XunitXmlTestLoggerVersion>8.0.0</XunitXmlTestLoggerVersion>
     <!-- -->
   </PropertyGroup>

vsintegration/tests/FSharp.Editor.IntegrationTests/AbstractIntegrationTest.cs

@@ -9,7 +9,7 @@
 
 namespace Microsoft.CodeAnalysis.Testing
 {
-    [IdeSettings(MinVersion = VisualStudioVersion.VS2022)]
+    [IdeSettings(MinVersion = VisualStudioVersion.VS18, MaxVersion = VisualStudioVersion.VS18)]
     public abstract class AbstractIntegrationTest : AbstractIdeIntegrationTest
     {
         protected CancellationToken TestToken => HangMitigatingCancellationToken;

vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs

@@ -43,7 +43,7 @@ module Test
 let answer =
 """;
         var expectedBuildSummary = "========== Build: 0 succeeded, 1 failed, 0 up-to-date, 0 skipped ==========";
-        var expectedError = "(Compiler) Library.fs(3, 1): error FS0010: Incomplete structured construct at or before this point in binding";
+        var expectedError = "(Fsc) Library.fs(3, 1): error FS0010: Incomplete structured construct at or before this point in binding";
 
         await SolutionExplorer.CreateSingleProjectSolutionAsync("Library", template, TestToken);
         await SolutionExplorer.RestoreNuGetPackagesAsync(TestToken);

vsintegration/tests/FSharp.Editor.IntegrationTests/DebuggingSequencePointTests.cs

@@ -83,13 +83,23 @@ private async Task PrepareProjectAndBuildAsync()
         await SolutionExplorer.CreateSingleProjectSolutionAsync(ProjectName, SolutionExplorerInProcess.ExistingProjectTemplate, TestToken);
         await SolutionExplorer.SetStartupProjectAsync(ProjectName, TestToken);
 
+        // Write the fixture directly to disk *before* opening it in VS, instead of using
+        // SetTextAsync + "File.SaveAll" command. Reason: the debugger binds breakpoints by
+        // matching the PDB's recorded source-file hash against the hash of Program.fs on disk.
+        // If the edit-buffer is built first and SaveAll lands a moment later, the PDB hash
+        // points at one snapshot while disk holds another -- breakpoints stay unbound
+        // (children=0) and never fire. Writing to disk first keeps the two in lockstep.
+        await SolutionExplorer.WriteFileAsync(ProjectName, "Program.fs", File.ReadAllText(GetFixturePath()), TestToken);
         await SolutionExplorer.OpenFileAsync(ProjectName, "Program.fs", TestToken);
-        await Editor.SetTextAsync(File.ReadAllText(GetFixturePath()), TestToken);
-        await Shell.ExecuteCommandAsync("File.SaveAll", TestToken);
 
         var buildSummary = await SolutionExplorer.BuildSolutionAsync(TestToken);
         Assert.NotNull(buildSummary);
         Assert.Contains("Build: 1 succeeded, 0 failed", string.Join(Environment.NewLine, buildSummary));
+
+        // BuildSolutionAsync leaves the Build Output pane as the active text view; re-open Program.fs
+        // so the subsequent PlaceCaretAsync / ToggleBreakpointAtMarkerAsync operate on the F# source
+        // (rather than searching the build log for "BP_..." markers).
+        await SolutionExplorer.OpenFileAsync(ProjectName, "Program.fs", TestToken);
     }
 
     private static string GetFixturePath()

vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj

@@ -35,8 +35,21 @@
     <PackageReference Include="Nullable" Version="1.3.0" />
   </ItemGroup>
 
+  <ItemGroup>
+    <!-- xUnit v2 console runner: this project is excluded from the repo-wide Microsoft.Testing.Platform
+         gate (ExcludeFromTestPackageReferences above) because Microsoft.VisualStudio.Extensibility.Testing.Xunit
+         is locked to xUnit v2. PackageDownload restores the runner to the NuGet global packages folder
+         so eng\Build.ps1's TestUsingXUnitConsole can locate it, without adding it to this project's
+         compile/runtime closure. PackageDownload requires the bracketed exact-pin syntax. -->
+    <PackageDownload Include="xunit.runner.console" Version="[$(XunitRunnerConsoleV2Version)]" />
+  </ItemGroup>
+
   <Target Name="SyncXunitDesktopDll" AfterTargets="Build" Condition="'$(OutputType)' == 'Exe'">
     <Copy SourceFiles="$(TargetPath)" DestinationFiles="$(TargetDir)$(AssemblyName).dll" SkipUnchangedFiles="true" />
+    <!-- xunit.console.exe loads <dll>.config (not <exe>.config) for the test-AppDomain's binding redirects.
+         Without this copy, the IdeTestFramework's heavy VS-interop dependency graph fails to bind and the
+         custom xUnit test framework silently reports 0 discovered tests. -->
+    <Copy SourceFiles="$(TargetPath).config" DestinationFiles="$(TargetDir)$(AssemblyName).dll.config" SkipUnchangedFiles="true" Condition="Exists('$(TargetPath).config')" />
   </Target>
 
 </Project>

vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using Microsoft.CodeAnalysis.Testing;
+using System;
 using System.Threading.Tasks;
 using Xunit;
 using static Microsoft.VisualStudio.VSConstants;
@@ -11,6 +12,14 @@ namespace FSharp.Editor.IntegrationTests;
 
 public class GoToDefinitionTests : AbstractIntegrationTest
 {
+    // Roslyn's workspace propagates buffer edits to its solution snapshot asynchronously, and the
+    // F# checker schedules its own typecheck on top. WaitForProjectSystemAsync only covers VS
+    // project-system loading -- not these two async hops -- so GoToDefn invoked too quickly after
+    // SetTextAsync / OpenFileAsync sees a stale Document and no-ops. A short delay is the cheapest
+    // pragmatic bridge until a proper Roslyn-style WaitForAsyncOperationsAsync(FeatureAttribute.Workspace)
+    // partial is wired up here.
+    private static readonly TimeSpan FSharpCheckerSettleDelay = TimeSpan.FromSeconds(2);
+
     [IdeFact]
     public async Task GoesToDefinition()
     {
@@ -28,7 +37,8 @@ module Test
         await SolutionExplorer.CreateSingleProjectSolutionAsync("Library", template, TestToken);
         await SolutionExplorer.RestoreNuGetPackagesAsync(TestToken);
         await Editor.SetTextAsync(code, TestToken);
-
+        await Task.Delay(FSharpCheckerSettleDelay, TestToken);
+
         await Editor.PlaceCaretAsync("add 1", TestToken);
         await Shell.ExecuteCommandAsync(VSStd97CmdID.GotoDefn, TestToken);
         var actualText = await Editor.GetCurrentLineTextAsync(TestToken);
@@ -72,6 +82,7 @@ let id (t: SomeType) = t
         await SolutionExplorer.BuildSolutionAsync(TestToken);
 
         await SolutionExplorer.OpenFileAsync("Library", "Module.fsi", TestToken);
+        await Task.Delay(FSharpCheckerSettleDelay, TestToken);
         await Editor.PlaceCaretAsync("SomeType ->", TestToken);
         await Shell.ExecuteCommandAsync(VSStd97CmdID.GotoDefn, TestToken);
         var expectedText = "type SomeType =";
@@ -82,6 +93,7 @@ let id (t: SomeType) = t
         Assert.Equal(expectedWindow, actualWindow);
 
         await SolutionExplorer.OpenFileAsync("Library", "Module.fs", TestToken);
+        await Task.Delay(FSharpCheckerSettleDelay, TestToken);
         await Editor.PlaceCaretAsync("SomeType)", TestToken);
         await Shell.ExecuteCommandAsync(VSStd97CmdID.GotoDefn, TestToken);
         expectedText = "type SomeType =";

vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/EditorInProcess.cs

@@ -75,20 +75,57 @@ public async Task PlaceCaretAsync(string marker, CancellationToken cancellationT
 
     public async Task<IEnumerable<SuggestedActionSet>> InvokeCodeActionListAsync(CancellationToken cancellationToken)
     {
-        await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken);
+        // Poll-and-retry: the lightbulb session can race with F# diagnostics computation in two ways:
+        //   - broker.GetSession(view) returns null when no session is active yet (NRE on cast).
+        //   - The session opens but self-dismisses before reaching Completed/CompletedWithoutData,
+        //     producing TaskCanceledException from LightBulbHelper.WaitForItemsAsync.
+        // Each retry posts a fresh ShowQuickFixes command (creating a new session) and tolerates
+        // both failure modes. Bounded to ~5 seconds; the outer test ct still caps total wait.
+        const int MaxAttempts = 20;
+        var attemptDelay = TimeSpan.FromMilliseconds(250);
 
         var shell = await GetRequiredGlobalServiceAsync<SVsUIShell, IVsUIShell>(cancellationToken);
+        var broker = await GetComponentModelServiceAsync<ILightBulbBroker>(cancellationToken);
         var cmdGroup = typeof(VSConstants.VSStd14CmdID).GUID;
-        var cmdExecOpt = OLECMDEXECOPT.OLECMDEXECOPT_DONTPROMPTUSER;
-
         var cmdID = VSConstants.VSStd14CmdID.ShowQuickFixes;
-        object? obj = null;
-        shell.PostExecCommand(cmdGroup, (uint)cmdID, (uint)cmdExecOpt, ref obj);
 
-        var view = await GetActiveTextViewAsync(cancellationToken);
-        var broker = await GetComponentModelServiceAsync<ILightBulbBroker>(cancellationToken);
+        for (var attempt = 1; attempt <= MaxAttempts; attempt++)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken);
+
+            object? obj = null;
+            shell.PostExecCommand(cmdGroup, (uint)cmdID, (uint)OLECMDEXECOPT.OLECMDEXECOPT_DONTPROMPTUSER, ref obj);
+            var view = await GetActiveTextViewAsync(cancellationToken);
+
+            try
+            {
+                var lightbulbs = await LightBulbHelper.WaitForItemsAsync(broker, view, cancellationToken);
+                if (lightbulbs is not null && System.Linq.Enumerable.Any(lightbulbs))
+                {
+                    return lightbulbs;
+                }
+            }
+            catch (NullReferenceException) when (attempt < MaxAttempts)
+            {
+                // broker.GetSession(view) returned null; F# analyzer hasn't surfaced a session yet.
+            }
+            catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && attempt < MaxAttempts)
+            {
+                // Session opened but was dismissed before producing actions; try again.
+            }
+
+            await Task.Delay(attemptDelay, cancellationToken);
+        }
 
-        var lightbulbs = await LightBulbHelper.WaitForItemsAsync(broker, view, cancellationToken);
-        return lightbulbs;
+        // Final attempt: let the underlying exception propagate so the test failure message
+        // points at the actual root cause (NRE / TaskCanceled) rather than a vague timeout.
+        await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken);
+        {
+            object? obj = null;
+            shell.PostExecCommand(cmdGroup, (uint)cmdID, (uint)OLECMDEXECOPT.OLECMDEXECOPT_DONTPROMPTUSER, ref obj);
+            var view = await GetActiveTextViewAsync(cancellationToken);
+            return await LightBulbHelper.WaitForItemsAsync(broker, view, cancellationToken);
+        }
     }
 }

vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ShellInProcess_Debugging.cs

@@ -84,10 +84,12 @@ public async Task WaitForBreakpointHitAsync(TimeSpan timeout, bool continueOnSte
                 dbg.Go(true);
 
             if (seenRun && mode == EnvDTE.dbgDebugMode.dbgDesignMode)
-                throw new InvalidOperationException("Debug session ended before breakpoint hit.");
+                throw new InvalidOperationException(
+                    $"Debug session ended before breakpoint hit. Breakpoints={BreakpointSummary(dbg)}.");
 
             if (sw.Elapsed > timeout)
-                throw new TimeoutException($"No breakpoint hit after {timeout}. Mode={mode}; Breakpoints={BreakpointSummary(dbg)}.");
+                throw new TimeoutException(
+                    $"No breakpoint hit after {timeout}. Mode={mode}; Breakpoints={BreakpointSummary(dbg)}.");
 
             await Task.Delay(100, cancellationToken);
         }
@@ -122,7 +124,11 @@ private static string BreakpointSummary(EnvDTE.Debugger dbg)
         {
             var children = 0;
             foreach (EnvDTE.Breakpoint _ in bp.Children) children++;
-            items.Add($"{Path.GetFileName(bp.File)}:{bp.FileLine}(children={children})");
+            // CurrentHits + Children counts together let us distinguish unbound (children=0, hits=0)
+            // from "bound but never executed" (children>0, hits=0) from "bound and hit" (hits>0).
+            // Unbound means the PDB has no IL location for this source line; the disk-source hash
+            // didn't match the PDB's recorded hash; or the loaded module doesn't have this code.
+            items.Add($"{Path.GetFileName(bp.File)}:{bp.FileLine}(children={children},hits={bp.CurrentHits})");
         }
         return items.Count == 0 ? "none" : string.Join(",", items);
     }