ci: let rstack-ecosystem-ci run the E2E suites

.github/workflows/ci.yml

@@ -56,13 +56,14 @@ jobs:
       - name: Unit Test
         run: pnpm run test:unit
 
-  # ======== E2E: Windows + macOS, no Linux ========
-  # These are the platforms the extension is actually used on, and the only
-  # ones upstream rstest runs its ported VS Code suites on. Linux is excluded
-  # deliberately: the Extension Host needs xvfb there, and inotify reports a
-  # non-atomic file rewrite as separate truncate/write events, so fixture edits
-  # in the watch-mode suites race the watcher in a way no user hits. Do not add
-  # a Linux E2E job back without also making every fixture edit atomic.
+  # ======== E2E: Windows + macOS ========
+  # Windows and macOS are the platforms users run the extension on. Linux E2E
+  # is deliberately not run here: rstack-ecosystem-ci runs the suites on
+  # ubuntu-latest (rstackjs/rstack-editor#40), and for that `e2e/run.mjs`
+  # re-execs under `xvfb-run` when DISPLAY is unset. The old concern that
+  # non-atomic fixture writes race inotify on Linux was tested and did not
+  # reproduce (rstackjs/rstack-editor#67), so the exclusion is about CI cost,
+  # not correctness.
   e2e:
     name: E2E (${{ matrix.os }})
     runs-on: ${{ matrix.os }}

.gitignore

@@ -158,6 +158,7 @@ packages/vscode/e2e/fixtures/rstack-cli-package-path/infra/pnpm-lock.yaml
 packages/vscode/e2e/lint/fixtures/pnpm-lock.yaml
 packages/vscode/e2e/lint/fixtures/*/pnpm-lock.yaml
 packages/vscode/e2e/rstest/fixtures/*/pnpm-lock.yaml
+packages/vscode/e2e/**/.toolchain/
 
 # Build-time copy of the workspace root LICENSE (see rslib.config.mts)
 packages/vscode/LICENSE

packages/vscode/AGENTS.md

@@ -72,3 +72,6 @@ One extension replacing the standalone `rstack.rslint` and `rstack.rstest` exten
 - E2E fixtures install exact versions of published npm packages (not workspace links): the extension must work against what users actually install. Renovate updates the exact toolchain pins; generated fixture lockfiles and `node_modules` remain disposable and uncommitted.
 - Prefer running the E2E slice that covers the change over the full chain: `pnpm test:e2e <slice ...>` (or the `test:e2e:<slice>` aliases). Slices are declared in the `SLICES` table in `e2e/run.mjs` (name, fixtures, entry) — the package.json scripts are thin forwards and carry no slice knowledge. `RSTACK_LINT_E2E_SUITES=<name,...>` filters lint suites.
 - Run E2E locally as `VSCODE_CLI=1 pnpm test:e2e <slice ...>`. Without it, the launched VS Code overwrites the extension host's `PATH` with a login-shell snapshot; on a machine whose login-shell `node` is below the runtime floor, the User Node preflight (correctly) refuses and every fmt test times out waiting for a server. CI is unaffected — its PATH `node` is new enough either way. The heavier alternative, `--force-disable-user-env` in `e2e/runTest.ts`, was deliberately not taken: it would change env fidelity for every slice.
+- Linux E2E runs only in rstack-ecosystem-ci, which relies on `e2e/run.mjs` re-execing itself under `xvfb-run -a` when `DISPLAY` is unset (the whole chain runs against one X server, so `pnpm test:e2e` is the same command everywhere; `xvfb` is a prerequisite on Linux). This repo's CI runs E2E on Windows and macOS only, so the Linux path is not exercised here.
+- `RSTACK_E2E_TOOLCHAIN_OVERRIDES="<name>=<package dir>,…"` packs each package into `<fixture>/.toolchain/` and adds a `file:` override to every fixture's `pnpm-workspace.yaml`, so rstack-ecosystem-ci can run the suites against unreleased builds; setup fails if the lockfile still holds a registry version. Rslint needs two entries, `@rslint/core` and the host `@rslint/native-<tuple>`: core resolves its binary through `require.resolve('@rslint/native-<tuple>/bin')`, so overriding core alone would leave the published binary in place. The yaml is not restored — the env is for disposable checkouts; locally, `git checkout -- packages/vscode/e2e`. Rationale in the `setupFixtures.mjs` header.
+- `VSCODE_TEST_CACHE_PATH` moves the downloaded VS Code out of `.vscode-test/`; every `runTest.ts` entry forwards it to `@vscode/test-electron`. rstack-ecosystem-ci needs it because it runs `git clean -fdxq` in the checkout before each suite, which would wipe the cache. Unset keeps the default.

packages/vscode/e2e/lint/runTest.ts

@@ -32,6 +32,7 @@ import os from 'node:os';
 import path from 'node:path';
 
 import { downloadAndUnzipVSCode, runTests } from '@vscode/test-electron';
+import { vscodeTestCachePath } from '../vscodeTestCache';
 
 interface TestSuite {
   name: string;
@@ -218,11 +219,14 @@ async function main(): Promise<void> {
   // Resolve the executable once so every suite explicitly uses the same one.
   // `VSCODE_TEST_EXECUTABLE`/`VSCODE_TEST_VERSION` mirror the other runners in
   // this repo; a cached download under `.vscode-test/` is reused.
+  // `VSCODE_TEST_CACHE_PATH` moves that cache outside the checkout
+  // (rstack-ecosystem-ci wipes its consumer with `git clean -fdxq`).
   const vscodeExecutablePath =
     process.env.VSCODE_TEST_EXECUTABLE ||
     (await downloadAndUnzipVSCode({
       version: process.env.VSCODE_TEST_VERSION ?? 'stable',
       timeout: 60_000,
+      cachePath: vscodeTestCachePath,
       extensionDevelopmentPath,
     }));
 

packages/vscode/e2e/rstest/runTest.ts

@@ -28,6 +28,7 @@ import {
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { runTests } from '@vscode/test-electron';
+import { vscodeTestCachePath } from '../vscodeTestCache';
 
 // Repo-relative fixture dirs whose installs this slice needs. workspace-2 has
 // no node_modules of its own per project; the root install serves both nested
@@ -95,6 +96,7 @@ async function main() {
     // Escape hatch for offline / restricted environments: point at an existing
     // VS Code and nothing is downloaded at all.
     vscodeExecutablePath: process.env.VSCODE_TEST_EXECUTABLE || undefined,
+    cachePath: vscodeTestCachePath,
     extensionDevelopmentPath,
     extensionTestsPath,
     // In CI the rstest output channel is unreadable, so the extension mirrors

packages/vscode/e2e/run.mjs

@@ -10,6 +10,10 @@
 // up-to-date fixture, and unknown names throw there), then the entries run
 // sequentially — each `compile`d entry launches its own VS Code via
 // `@vscode/test-electron`.
+//
+// On Linux that VS Code needs an X display, so this file re-execs itself under
+// `xvfb-run` when there is none — running the whole chain, not just the
+// entries, under one server.
 import { spawnSync } from 'node:child_process';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
@@ -75,7 +79,7 @@ const packageRoot = path.resolve(
 /**
  * @param {string} command
  * @param {string[]} args
- * @param {{shell?: boolean}} [opts]
+ * @param {{shell?: boolean, missingMessage?: string}} [opts]
  */
 const run = (command, args, opts = {}) => {
   const result = spawnSync(command, args, {
@@ -89,6 +93,13 @@ const run = (command, args, opts = {}) => {
     shell: opts.shell ?? false,
   });
   if (result.error) {
+    if (
+      opts.missingMessage &&
+      /** @type {NodeJS.ErrnoException} */ (result.error).code === 'ENOENT'
+    ) {
+      console.error(opts.missingMessage);
+      process.exit(1);
+    }
     throw result.error;
   }
   if (result.status !== 0) {
@@ -108,6 +119,28 @@ const selected =
     ? SLICES.filter((slice) => requested.includes(slice.name))
     : SLICES;
 
+// The Extension Host needs an X display. `xvfb-run` exports DISPLAY into the
+// child, so the re-exec'd process fails this check — it cannot recurse. A
+// slice that only runs plain Node (`smoke`) opens no window and skips it.
+if (
+  process.platform === 'linux' &&
+  !process.env.DISPLAY &&
+  selected.some((slice) => slice.compile)
+) {
+  run(
+    'xvfb-run',
+    [
+      '-a',
+      process.execPath,
+      fileURLToPath(import.meta.url),
+      ...process.argv.slice(2),
+    ],
+    { missingMessage: '[e2e] xvfb-run not found; install the xvfb package' },
+  );
+  // `run` exits on failure; the re-exec'd chain already ran every slice.
+  process.exit(0);
+}
+
 if (selected.some((slice) => slice.compile)) {
   console.log('[e2e] compiling slices (tsc -p tsconfig.e2e.json)');
   // On Windows, pnpm is a .cmd shim, and Node refuses to spawn batch files

packages/vscode/e2e/runTest.ts

@@ -10,6 +10,7 @@ import { cpSync, existsSync, mkdtempSync, rmSync, symlinkSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { runTests } from '@vscode/test-electron';
+import { vscodeTestCachePath } from './vscodeTestCache';
 
 const FIXTURE_NAMES = [
   'rslint',
@@ -50,6 +51,7 @@ async function launch({
     // VS Code (`.../Visual Studio Code.app/Contents/MacOS/Electron`, `Code.exe`,
     // `code`) and nothing is downloaded at all.
     vscodeExecutablePath: process.env.VSCODE_TEST_EXECUTABLE || undefined,
+    cachePath: vscodeTestCachePath,
     extensionDevelopmentPath,
     extensionTestsPath,
     launchArgs: [

packages/vscode/e2e/setupFixtures.mjs

@@ -3,7 +3,7 @@
 // The fixtures install **exact published npm versions** of
 // `@rslint/core` / `@rstest/core` / `rstack` — the extension resolves all three
 // from the project, so a fixture that linked this repo's own node_modules would
-// test nothing. `rstack@0.7.2` itself pins `@rslint/core@0.9.0` exactly, so the
+// test nothing. `rstack@0.9.0` itself pins `@rslint/core@0.10.0` exactly, so the
 // Rstack fixture pins its lint core transitively. Each fixture is its own
 // independent install. Exact toolchain pins make installs reproducible
 // without committed lockfiles, and Renovate bumps those pins.
@@ -22,8 +22,28 @@
 //
 // Idempotent: pnpm is a no-op when the fixture is already up to date, so
 // `test:e2e` can always run it.
+//
+// `RSTACK_E2E_TOOLCHAIN_OVERRIDES="@rstest/core=/abs/rstest/packages/core,…"`
+// installs unreleased builds instead, for rstack-ecosystem-ci (#40). Its root
+// `pnpm-workspace.yaml` override cannot reach the fixtures, which are their own
+// workspace roots, so each package is packed into `<fixture>/.toolchain/` and
+// the fixture yaml gets a relative `file:` override: the dependency-recovery
+// suites copy a fixture to tmpdir and reinstall with `--frozen-lockfile`, and
+// pnpm records `file:` relative to the lockfile. After install the lockfile
+// must resolve each overridden package through the tarball, never the
+// registry: build and pin can share a version number, which proves nothing.
+// The yaml is changed in place and never restored: the env is meant for
+// disposable checkouts (ecosystem CI); locally,
+// `git checkout -- packages/vscode/e2e` undoes it. A later run with a different
+// override set replaces the block the earlier run wrote.
 import { spawnSync } from 'node:child_process';
-import { existsSync } from 'node:fs';
+import {
+  copyFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  writeFileSync,
+} from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
@@ -76,19 +96,130 @@ export const FIXTURE_NAMES = Object.keys(FIXTURES);
 
 const pnpmCommand = 'pnpm';
 
-/** @param {string} name */
+/** `name=<absolute package dir>` entries, comma-separated; unset → none. */
+const overrides = (process.env.RSTACK_E2E_TOOLCHAIN_OVERRIDES ?? '')
+  .split(',')
+  .filter((entry) => entry.trim())
+  .map((entry) => {
+    const [pkg = '', dir = ''] = entry
+      .split(/=(.*)/)
+      .map((part) => part.trim());
+    // Without this, a missing `=` packs `packages/vscode` itself.
+    if (
+      !pkg ||
+      !path.isAbsolute(dir) ||
+      !existsSync(path.join(dir, 'package.json'))
+    ) {
+      throw new Error(
+        `RSTACK_E2E_TOOLCHAIN_OVERRIDES: expected name=<absolute package dir>, got ${entry}`,
+      );
+    }
+    return {
+      pkg,
+      dir,
+      file: `${pkg.replace(/^@/, '').replace('/', '-')}.tgz`,
+      // The lookbehind keeps `rstack` from matching `create-rstack@1.0.0`.
+      registry: new RegExp(
+        `(?<![\\w@/.-])${pkg.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&')}@\\d`,
+      ),
+    };
+  });
+
+/**
+ * @param {string[]} args
+ * @param {string} cwd
+ */
+const pnpm = (args, cwd) => {
+  const result = spawnSync(pnpmCommand, args, {
+    cwd,
+    stdio: 'inherit',
+    env: process.env,
+    // On Windows, pnpm is a .cmd shim, and Node refuses to spawn batch
+    // files without a shell (CVE-2024-27980 hardening) — EINVAL otherwise.
+    shell: process.platform === 'win32',
+  });
+  if (result.error) {
+    throw result.error;
+  }
+  if (result.status !== 0) {
+    throw new Error(
+      `pnpm ${args[0]} failed in ${cwd} (exit code ${String(result.status)})`,
+    );
+  }
+};
+
+const OVERRIDE_MARKER = '# RSTACK_E2E_TOOLCHAIN_OVERRIDES';
+
+// One pack per package, shared by every fixture: the tarballs are identical
+// and a native one is ~14 MB. The first fixture that needs a package packs it
+// into its own `.toolchain/`; the rest copy it there.
+const packedTarballs = new Map();
+
+/**
+ * @param {{pkg: string, dir: string, file: string}} entry
+ * @param {string} cwd
+ */
+const packOnce = (entry, cwd) => {
+  const cached = packedTarballs.get(entry.dir);
+  if (cached) {
+    return cached;
+  }
+  const tarball = path.join(cwd, '.toolchain', entry.file);
+  pnpm(['pack', '--out', tarball], entry.dir);
+  packedTarballs.set(entry.dir, tarball);
+  return tarball;
+};
+
+/**
+ * Appends this run's override block to a fixture's `pnpm-workspace.yaml`,
+ * replacing the block an earlier run wrote (a checkout can outlive one
+ * override set). Fixtures are their own workspace roots, so the repo's
+ * overrides cannot reach them; each gets a fixture-relative `file:` spec —
+ * the dependency-recovery suites copy the fixture to tmpdir and reinstall with
+ * `--frozen-lockfile`, and pnpm records `file:` relative to the lockfile.
+ * @param {string} cwd
+ */
+const injectOverrides = (cwd) => {
+  const yamlPath = path.join(cwd, 'pnpm-workspace.yaml');
+  const text = readFileSync(yamlPath, 'utf8');
+  const markerAt = text.indexOf(OVERRIDE_MARKER);
+  const base = markerAt === -1 ? text : text.slice(0, markerAt);
+  // Merging into an overrides map this script did not write needs a YAML
+  // parser; fail loudly instead of appending a duplicate key.
+  if (/^overrides:/m.test(base)) {
+    throw new Error(
+      `${yamlPath} declares its own overrides; cannot inject ${overrides.map(({ pkg }) => pkg).join(', ')}`,
+    );
+  }
+  mkdirSync(path.join(cwd, '.toolchain'), { recursive: true });
+  const lines = [OVERRIDE_MARKER, 'overrides:'];
+  for (const entry of overrides) {
+    const destination = path.join(cwd, '.toolchain', entry.file);
+    const packed = packOnce(entry, cwd);
+    if (packed !== destination) {
+      copyFileSync(packed, destination);
+    }
+    lines.push(`  '${entry.pkg}': 'file:.toolchain/${entry.file}'`);
+  }
+  lines.push('');
+  writeFileSync(yamlPath, `${base.trimEnd()}\n\n${lines.join('\n')}`);
+};
+
+/** @param {string} name @returns {boolean} whether an override was used */
 const install = (name) => {
   const cwd = FIXTURES[name];
   if (!existsSync(path.join(cwd, 'package.json'))) {
     throw new Error(`E2E fixture ${name} has no package.json at ${cwd}`);
   }
+  if (overrides.length > 0) {
+    injectOverrides(cwd);
+  }
   console.log(`[e2e] installing fixture: ${name}`);
   // Keep pnpm's default isolated layout. In the rstack fixture the tool cores
   // are transitive dependencies beside rstack in the virtual store, matching
   // the layout users get rather than masking resolution bugs with public
   // hoisting.
-  const result = spawnSync(
-    pnpmCommand,
+  pnpm(
     [
       'install',
       // Fixtures pin exact toolchain versions rather than committing lockfiles;
@@ -105,35 +236,53 @@ const install = (name) => {
       // build scripts as-is.
       '--config.dangerouslyAllowAllBuilds=true',
     ],
-    {
-      cwd,
-      stdio: 'inherit',
-      env: process.env,
-      // On Windows, pnpm is a .cmd shim, and Node refuses to spawn batch
-      // files without a shell (CVE-2024-27980 hardening) — EINVAL otherwise.
-      shell: process.platform === 'win32',
-    },
+    cwd,
   );
-  if (result.error) {
-    throw result.error;
-  }
-  if (result.status !== 0) {
-    throw new Error(
-      `pnpm install failed for the ${name} fixture (exit code ${String(result.status)})`,
+  // A registry resolution is `name@1.2.3`, the tarball `name@file:…`; the
+  // lockfile also covers transitive ones (the rstack fixture's @rstest/core).
+  const lockfile =
+    overrides.length > 0
+      ? readFileSync(path.join(cwd, 'pnpm-lock.yaml'), 'utf8')
+      : '';
+  let resolved = false;
+  for (const { pkg, registry } of overrides) {
+    if (registry.test(lockfile)) {
+      throw new Error(
+        `\`${pkg}\` still resolves to a registry version in the ${name} fixture`,
+      );
+    }
+    if (!lockfile.includes(`${pkg}@file:`)) {
+      console.log(`[e2e] ${name}: ${pkg} not in graph, override unused`);
+      continue;
+    }
+    resolved = true;
+    console.log(
+      `[e2e] ${name}: ${pkg} resolved through .toolchain (no registry version in pnpm-lock.yaml)`,
     );
   }
+  return resolved;
 };
 
 const main = () => {
   const requested = process.argv.slice(2);
   const names = requested.length > 0 ? requested : FIXTURE_NAMES;
+  // Fixtures that do not depend on an overridden package are legitimate
+  // no-ops; only a run where no fixture used any override is an error.
+  let resolvedThroughTarball = false;
   for (const name of names) {
     if (!FIXTURE_NAMES.includes(name)) {
       throw new Error(
         `unknown E2E fixture: ${name} (known: ${FIXTURE_NAMES.join(', ')})`,
       );
     }
-    install(name);
+    // Not `||=`: that would skip installing every fixture after the first one
+    // that used an override.
+    resolvedThroughTarball = install(name) || resolvedThroughTarball;
+  }
+  if (overrides.length > 0 && !resolvedThroughTarball) {
+    throw new Error(
+      `RSTACK_E2E_TOOLCHAIN_OVERRIDES: no fixture in this run resolved ${overrides.map(({ pkg }) => pkg).join(', ')} through a tarball; the override did nothing`,
+    );
   }
 };