flaky test - wait-for-host-standby: retry puppeteer.launch with diagnostics

packages/realm-server/scripts/wait-for-host-standby.ts

@@ -32,18 +32,90 @@
 // against a genuinely broken vite — in normal dev that ceiling is never
 // reached.
 
-import puppeteer from 'puppeteer';
+import puppeteer, { type Browser } from 'puppeteer';
 
 const PER_ATTEMPT_TIMEOUT_MS = 30_000;
 const MAX_BACKOFF_MS = 5_000;
 const TOTAL_TIMEOUT_MS = 600_000;
+// Chrome startup on a loaded CI runner occasionally takes >30s to print
+// its DevTools WS endpoint to stdout. Puppeteer's default launch timeout
+// is 30s, so a single slow start aborts the whole script before the
+// goto/waitForFunction retry loop ever runs. Give the launch its own
+// generous budget and retry it independently — the page-pool's own
+// BrowserManager also relies on launch succeeding on the first try, but
+// here we're a one-shot startup probe and the cost of a retry is small.
+const LAUNCH_TIMEOUT_MS = 90_000;
+const LAUNCH_MAX_ATTEMPTS = 3;
+const LAUNCH_RETRY_BACKOFF_MS = 2_000;
 
 import { isHttpsLoopback } from '../lib/is-https-loopback';
 
 const log = (msg: string) => console.log(`[wait-for-host-standby] ${msg}`);
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 const elapsedSec = (start: number) => Math.round((Date.now() - start) / 1000);
 
+async function launchBrowserWithRetry({
+  launchArgs,
+  totalDeadline,
+}: {
+  launchArgs: string[];
+  totalDeadline: number;
+}): Promise<Browser> {
+  let executablePath = process.env.PUPPETEER_EXECUTABLE_PATH;
+  let lastError: unknown;
+  for (let attempt = 1; attempt <= LAUNCH_MAX_ATTEMPTS; attempt++) {
+    let remaining = totalDeadline - Date.now();
+    if (remaining <= 0) {
+      break;
+    }
+    let timeout = Math.min(LAUNCH_TIMEOUT_MS, remaining);
+    // On the final attempt, pipe Chrome's own stdout/stderr through node so
+    // that if launch is still failing we capture *why* (sandbox denial,
+    // missing shared library, GPU init crash, etc.) instead of a bare
+    // "Timed out … while waiting for the WS endpoint URL." The earlier
+    // attempts stay quiet on healthy runs.
+    let dumpio =
+      attempt === LAUNCH_MAX_ATTEMPTS ||
+      process.env.WAIT_FOR_HOST_STANDBY_VERBOSE === '1';
+    log(
+      `puppeteer.launch attempt ${attempt}/${LAUNCH_MAX_ATTEMPTS} ` +
+        `(timeout=${timeout}ms, executable=${executablePath ?? 'puppeteer-bundled'}, ` +
+        `args=${JSON.stringify(launchArgs)}, dumpio=${dumpio})`,
+    );
+    let t0 = Date.now();
+    try {
+      let browser = await puppeteer.launch({
+        headless: true,
+        timeout,
+        dumpio,
+        ...(launchArgs.length > 0 ? { args: launchArgs } : {}),
+        ...(executablePath ? { executablePath } : {}),
+      });
+      log(
+        `puppeteer.launch attempt ${attempt} succeeded after ${Date.now() - t0}ms`,
+      );
+      return browser;
+    } catch (e) {
+      lastError = e;
+      let message = e instanceof Error ? e.message : String(e);
+      log(
+        `puppeteer.launch attempt ${attempt} failed after ${Date.now() - t0}ms: ${message}`,
+      );
+      if (attempt === LAUNCH_MAX_ATTEMPTS) {
+        break;
+      }
+      let remainingAfterFailure = totalDeadline - Date.now();
+      if (remainingAfterFailure <= LAUNCH_RETRY_BACKOFF_MS) {
+        break;
+      }
+      await sleep(LAUNCH_RETRY_BACKOFF_MS);
+    }
+  }
+  throw lastError instanceof Error
+    ? lastError
+    : new Error(`puppeteer.launch failed: ${String(lastError)}`);
+}
+
 async function main() {
   // Vite serves HTTPS on localhost:4200 in local dev (the realm-server
   // requires the mkcert leaf and vite reads the same cert). Default
@@ -81,12 +153,9 @@ async function main() {
   log(`probing ${standbyUrl} (max ${TOTAL_TIMEOUT_MS / 1000}s)...`);
   let start = Date.now();
 
-  let browser = await puppeteer.launch({
-    headless: true,
-    ...(launchArgs.length > 0 ? { args: launchArgs } : {}),
-    ...(process.env.PUPPETEER_EXECUTABLE_PATH
-      ? { executablePath: process.env.PUPPETEER_EXECUTABLE_PATH }
-      : {}),
+  let browser = await launchBrowserWithRetry({
+    launchArgs,
+    totalDeadline: start + TOTAL_TIMEOUT_MS,
   });
 
   let attempt = 0;