[core] Warm cold e2e targets before the suite starts runs

.github/scripts/aggregate-e2e-results.js

@@ -248,6 +248,12 @@ function renderInfraSection(infraEvents) {
       `${event.testName} (${event.app})`,
       time ? `at ${time}Z` : null,
       event.runId ? `abandoned \`${event.runId}\`` : null,
+      // cold-start-warmup events carry every stalled probe; the first is
+      // rendered as the abandoned run, the rest as a count.
+      Array.isArray(event.stalledProbeRunIds) &&
+      event.stalledProbeRunIds.length > 1
+        ? `(+${event.stalledProbeRunIds.length - 1} more)`
+        : null,
     ].filter(Boolean);
     console.log(`- ${parts.join(' · ')}`);
   }

packages/core/e2e/e2e.test.ts

@@ -50,6 +50,7 @@ import {
   setupWorld,
   startTracked,
   trackRun,
+  warmDeployment,
   writeDiagnosticsSidecar,
   writeInfraSidecar,
 } from './utils';
@@ -325,9 +326,25 @@ async function startWorkflowViaHttp(
 describe('e2e', () => {
   // Configure the World for the test runner process so that start() and
   // run.returnValue can communicate with the same backend as the workbench app.
+  // Also warm the target before the first test starts a run: a fresh Vercel
+  // deployment picks up runs long after it answers HTTP, and a local dev
+  // server pays its first flow-route compile on the first delivery. Either
+  // cold window otherwise surfaces as pickup-stall infra events on the
+  // suite's first tests (see warmDeployment). rawStart, not start — probes
+  // manage their own stalls without tripping the per-test watchdog.
   beforeAll(async () => {
     setupWorld(deploymentUrl);
-  });
+    await warmDeployment(async () =>
+      rawStart(
+        await getWorkflowMetadata(
+          deploymentUrl,
+          'workflows/99_e2e.ts',
+          'addTenWorkflow'
+        ),
+        [1]
+      )
+    );
+  }, 150_000);
 
   // Enable automatic run diagnostics on test failure
   beforeEach((ctx) => {

packages/core/e2e/utils.test.ts

@@ -1,5 +1,10 @@
-import { afterEach, describe, expect, test } from 'vitest';
-import { hasStepSourceMaps, waitForRunPickup } from './utils';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import {
+  getRecordedInfraEvents,
+  hasStepSourceMaps,
+  waitForRunPickup,
+  warmDeployment,
+} from './utils';
 
 const ORIGINAL_ENV = { ...process.env };
 
@@ -130,3 +135,82 @@ describe('waitForRunPickup', () => {
     await expect(waitForRunPickup(run as any, 5_000)).resolves.toBe(true);
   });
 });
+
+describe('warmDeployment', () => {
+  const makeProbe = (id: string, statuses: string[]) => ({
+    runId: id,
+    get status() {
+      return Promise.resolve(
+        statuses.length > 1 ? statuses.shift() : statuses[0]
+      );
+    },
+    cancel: vi.fn(async () => {}),
+  });
+
+  const eventsBefore = () => getRecordedInfraEvents().length;
+
+  test('a probe picked up first try records nothing', async () => {
+    const before = eventsBefore();
+    const probe = makeProbe('wrun_warm_ok', ['running']);
+    // biome-ignore lint/suspicious/noExplicitAny: minimal Run stand-in
+    const startProbe = vi.fn(async () => probe as any);
+    await warmDeployment(startProbe, {
+      pickupBudgetMs: 300,
+      totalBudgetMs: 2_000,
+    });
+    expect(startProbe).toHaveBeenCalledTimes(1);
+    expect(probe.cancel).not.toHaveBeenCalled();
+    expect(getRecordedInfraEvents().length).toBe(before);
+  });
+
+  test('a stalled probe is abandoned and the warmup recorded once', async () => {
+    const before = eventsBefore();
+    const stalled = makeProbe('wrun_warm_stall', ['pending']);
+    const warm = makeProbe('wrun_warm_pickup', ['running']);
+    const probes = [stalled, warm];
+    // biome-ignore lint/suspicious/noExplicitAny: minimal Run stand-in
+    const startProbe = vi.fn(async () => probes.shift() as any);
+    await warmDeployment(startProbe, {
+      pickupBudgetMs: 300,
+      totalBudgetMs: 10_000,
+    });
+    expect(startProbe).toHaveBeenCalledTimes(2);
+    expect(stalled.cancel).toHaveBeenCalledTimes(1);
+    expect(warm.cancel).not.toHaveBeenCalled();
+
+    const events = getRecordedInfraEvents().slice(before);
+    expect(events).toHaveLength(1);
+    expect(events[0]).toMatchObject({
+      kind: 'cold-start-warmup',
+      testName: 'suite warmup',
+      runId: 'wrun_warm_stall',
+      stalledProbeRunIds: ['wrun_warm_stall'],
+      pickedUpRunId: 'wrun_warm_pickup',
+    });
+  });
+
+  test('an exhausted budget records the warmup with no pickup and returns', async () => {
+    const before = eventsBefore();
+    let n = 0;
+    const startProbe = vi.fn(async () => {
+      n++;
+      // biome-ignore lint/suspicious/noExplicitAny: minimal Run stand-in
+      return makeProbe(`wrun_warm_${n}`, ['pending']) as any;
+    });
+    await warmDeployment(startProbe, {
+      pickupBudgetMs: 200,
+      totalBudgetMs: 500,
+    });
+    expect(startProbe.mock.calls.length).toBeGreaterThanOrEqual(1);
+
+    const events = getRecordedInfraEvents().slice(before);
+    expect(events).toHaveLength(1);
+    expect(events[0]).toMatchObject({
+      kind: 'cold-start-warmup',
+      pickedUpRunId: null,
+    });
+    expect(
+      (events[0] as { stalledProbeRunIds: string[] }).stalledProbeRunIds.length
+    ).toBe(startProbe.mock.calls.length);
+  });
+});

packages/core/e2e/utils.ts

@@ -814,21 +814,54 @@ export function trackRun<T>(
 // cluster of events in one time window reads as the platform blip it is.
 // ---------------------------------------------------------------------------
 
-interface InfraEvent {
-  kind: 'run-pickup-stall';
+interface InfraEventBase {
   testName: string;
+  waitedMs: number;
+  timestamp: string;
+}
+
+/** A mid-suite run the queue never picked up; abandoned and replaced. */
+interface RunPickupStallEvent extends InfraEventBase {
+  kind: 'run-pickup-stall';
   /** The run that was abandoned. */
   runId: string;
   /** The run started in its place. */
   replacementRunId: string;
-  waitedMs: number;
-  timestamp: string;
 }
 
+/**
+ * A fresh deployment needed more than one warmup probe before its queue
+ * consumer picked anything up (see `warmDeployment`). One event per suite,
+ * not per stalled probe.
+ */
+interface ColdStartWarmupEvent extends InfraEventBase {
+  kind: 'cold-start-warmup';
+  /** First abandoned probe (what the aggregation renders). */
+  runId: string;
+  /** Every probe that stalled, in order. */
+  stalledProbeRunIds: string[];
+  /** The probe that was finally picked up, or null if the budget ran out. */
+  pickedUpRunId: string | null;
+}
+
+type InfraEvent = RunPickupStallEvent | ColdStartWarmupEvent;
+
+/** `Omit` that distributes over a union instead of collapsing it. */
+type DistributiveOmit<T, K extends keyof T> = T extends unknown
+  ? Omit<T, K>
+  : never;
+
 const infraEvents: InfraEvent[] = [];
 
+/** Test-only visibility into events recorded so far. */
+export function getRecordedInfraEvents(): readonly InfraEvent[] {
+  return infraEvents;
+}
+
 export function recordInfraEvent(
-  event: Omit<InfraEvent, 'testName' | 'timestamp'> & { testName?: string }
+  event: DistributiveOmit<InfraEvent, 'testName' | 'timestamp'> & {
+    testName?: string;
+  }
 ) {
   infraEvents.push({
     ...event,
@@ -948,6 +981,93 @@ export async function startTracked<T>(
   return replacement;
 }
 
+/**
+ * Total budget for warming a fresh deployment before the suite runs.
+ */
+const WARMUP_BUDGET_MS = Number(
+  process.env.WORKFLOW_E2E_WARMUP_BUDGET_MS ?? '120000'
+);
+
+/**
+ * Warm a cold target before the first test starts a run.
+ *
+ * A target answers HTTP well before its first run is picked up promptly,
+ * for two reasons with one shape: a fresh Vercel deployment's queue
+ * consumer takes a while to start delivering, and a local dev server pays
+ * its first flow-route compile on the first delivery (observed as the
+ * suite's first test recording a pickup stall on local-dev lanes, `waitedMs`
+ * pegged at the full pickup budget). Either way the stalls the watchdog
+ * absorbs cluster on the suite's first tests, which drowns the infra
+ * telemetry in cold-start noise and leaves those tests one stalled
+ * replacement away from failing.
+ *
+ * Probes follow the watchdog's shape: start a throwaway run, and if it is
+ * still `pending` after `WORKFLOW_E2E_PICKUP_BUDGET_MS`, abandon it
+ * (best-effort cancel) and probe again, until a probe is picked up or
+ * `WORKFLOW_E2E_WARMUP_BUDGET_MS` is spent. A picked-up probe is left to
+ * finish on its own — pickup is what proves the pipeline is awake. A warmup
+ * that needed abandoned probes is recorded as a single `cold-start-warmup`
+ * infra event instead of per-test `run-pickup-stall` noise.
+ *
+ * If the budget runs out the suite proceeds anyway: the per-test watchdog
+ * still guards every start, and test failures carry the run diagnostics a
+ * thrown warmup would not.
+ */
+export async function warmDeployment(
+  startProbe: () => Promise<Run<unknown>>,
+  {
+    pickupBudgetMs = PICKUP_BUDGET_MS,
+    totalBudgetMs = WARMUP_BUDGET_MS,
+  }: { pickupBudgetMs?: number; totalBudgetMs?: number } = {}
+): Promise<void> {
+  const startedAt = Date.now();
+  const deadline = startedAt + totalBudgetMs;
+  const stalledProbeRunIds: string[] = [];
+
+  const record = (pickedUpRunId: string | null) => {
+    if (stalledProbeRunIds.length === 0) return;
+    recordInfraEvent({
+      kind: 'cold-start-warmup',
+      testName: 'suite warmup',
+      runId: stalledProbeRunIds[0],
+      stalledProbeRunIds: [...stalledProbeRunIds],
+      pickedUpRunId,
+      waitedMs: Date.now() - startedAt,
+    });
+  };
+
+  for (;;) {
+    const probe = await startProbe();
+    const remaining = deadline - Date.now();
+    if (await waitForRunPickup(probe, Math.min(pickupBudgetMs, remaining))) {
+      record(probe.runId);
+      if (stalledProbeRunIds.length > 0) {
+        console.warn(
+          `[e2e] deployment warmup: ${stalledProbeRunIds.length} probe(s) ` +
+            `stalled before ${probe.runId} was picked up ` +
+            `(${Date.now() - startedAt}ms; infra event, not a test failure)`
+        );
+      }
+      return;
+    }
+
+    stalledProbeRunIds.push(probe.runId);
+    void probe
+      .cancel({ cancelReason: 'e2e: warmup probe stuck pending, abandoned' })
+      .catch(() => {});
+
+    if (deadline - Date.now() <= 0) {
+      record(null);
+      console.warn(
+        `[e2e] deployment warmup: no probe picked up within ` +
+          `${totalBudgetMs}ms (${stalledProbeRunIds.length} abandoned); ` +
+          `proceeding — the per-test pickup watchdog still guards`
+      );
+      return;
+    }
+  }
+}
+
 /**
  * Build a Vercel observability dashboard URL for a workflow run.
  */