From a4bc8b388ecb272749b594d69a21cb593b374c3f Mon Sep 17 00:00:00 2001 From: Alex Langenfeld Date: Mon, 17 Aug 2026 13:07:47 -0500 Subject: [PATCH] [core] Warm cold e2e targets before the suite starts runs A target answers HTTP well before its first run is picked up promptly, for two reasons with one shape: a fresh Vercel deployment's queue consumer takes a while to start delivering, and a local dev server pays its first flow-route compile on the first queue delivery. The run-pickup watchdog's telemetry shows the cost - stalls concentrated on the suite's first test (addTenWorkflow), waitedMs pegged at the full 15s pickup budget, timestamps right at suite start; the sidecar backends identify local-dev lanes as a dominant source alongside fresh Vercel deployments. Each stall burns pickup budget inside a test, drowns the infra telemetry in cold-start noise, and leaves the first tests one stalled replacement away from failing. warmDeployment() runs in the suite's beforeAll: it starts throwaway probe runs, abandoning (best-effort cancelling) any still pending after the pickup budget, until one is picked up or a total budget (WORKFLOW_E2E_WARMUP_BUDGET_MS, default 120s) is spent. A warmup that needed abandoned probes is recorded as a single cold-start-warmup infra event - one per suite instead of per-test run-pickup-stall noise - and an exhausted budget proceeds anyway: the per-test watchdog still guards every start, and test failures carry run diagnostics a thrown warmup would not. Signed-off-by: Alex Langenfeld --- .github/scripts/aggregate-e2e-results.js | 6 ++ packages/core/e2e/e2e.test.ts | 19 +++- packages/core/e2e/utils.test.ts | 88 ++++++++++++++- packages/core/e2e/utils.ts | 130 ++++++++++++++++++++++- 4 files changed, 235 insertions(+), 8 deletions(-) diff --git a/.github/scripts/aggregate-e2e-results.js b/.github/scripts/aggregate-e2e-results.js index 5b5377d2e9..d34726f81d 100644 --- a/.github/scripts/aggregate-e2e-results.js +++ b/.github/scripts/aggregate-e2e-results.js @@ -248,6 +248,12 @@ function renderInfraSection(infraEvents) { `${event.testName} (${event.app})`, time ? `at ${time}Z` : null, event.runId ? `abandoned \`${event.runId}\`` : null, + // cold-start-warmup events carry every stalled probe; the first is + // rendered as the abandoned run, the rest as a count. + Array.isArray(event.stalledProbeRunIds) && + event.stalledProbeRunIds.length > 1 + ? `(+${event.stalledProbeRunIds.length - 1} more)` + : null, ].filter(Boolean); console.log(`- ${parts.join(' ยท ')}`); } diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index bbfb275222..88ce9b9113 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -50,6 +50,7 @@ import { setupWorld, startTracked, trackRun, + warmDeployment, writeDiagnosticsSidecar, writeInfraSidecar, } from './utils'; @@ -325,9 +326,25 @@ async function startWorkflowViaHttp( describe('e2e', () => { // Configure the World for the test runner process so that start() and // run.returnValue can communicate with the same backend as the workbench app. + // Also warm the target before the first test starts a run: a fresh Vercel + // deployment picks up runs long after it answers HTTP, and a local dev + // server pays its first flow-route compile on the first delivery. Either + // cold window otherwise surfaces as pickup-stall infra events on the + // suite's first tests (see warmDeployment). rawStart, not start โ€” probes + // manage their own stalls without tripping the per-test watchdog. beforeAll(async () => { setupWorld(deploymentUrl); - }); + await warmDeployment(async () => + rawStart( + await getWorkflowMetadata( + deploymentUrl, + 'workflows/99_e2e.ts', + 'addTenWorkflow' + ), + [1] + ) + ); + }, 150_000); // Enable automatic run diagnostics on test failure beforeEach((ctx) => { diff --git a/packages/core/e2e/utils.test.ts b/packages/core/e2e/utils.test.ts index 0e56e5d31c..b985ae5c8a 100644 --- a/packages/core/e2e/utils.test.ts +++ b/packages/core/e2e/utils.test.ts @@ -1,5 +1,10 @@ -import { afterEach, describe, expect, test } from 'vitest'; -import { hasStepSourceMaps, waitForRunPickup } from './utils'; +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { + getRecordedInfraEvents, + hasStepSourceMaps, + waitForRunPickup, + warmDeployment, +} from './utils'; const ORIGINAL_ENV = { ...process.env }; @@ -130,3 +135,82 @@ describe('waitForRunPickup', () => { await expect(waitForRunPickup(run as any, 5_000)).resolves.toBe(true); }); }); + +describe('warmDeployment', () => { + const makeProbe = (id: string, statuses: string[]) => ({ + runId: id, + get status() { + return Promise.resolve( + statuses.length > 1 ? statuses.shift() : statuses[0] + ); + }, + cancel: vi.fn(async () => {}), + }); + + const eventsBefore = () => getRecordedInfraEvents().length; + + test('a probe picked up first try records nothing', async () => { + const before = eventsBefore(); + const probe = makeProbe('wrun_warm_ok', ['running']); + // biome-ignore lint/suspicious/noExplicitAny: minimal Run stand-in + const startProbe = vi.fn(async () => probe as any); + await warmDeployment(startProbe, { + pickupBudgetMs: 300, + totalBudgetMs: 2_000, + }); + expect(startProbe).toHaveBeenCalledTimes(1); + expect(probe.cancel).not.toHaveBeenCalled(); + expect(getRecordedInfraEvents().length).toBe(before); + }); + + test('a stalled probe is abandoned and the warmup recorded once', async () => { + const before = eventsBefore(); + const stalled = makeProbe('wrun_warm_stall', ['pending']); + const warm = makeProbe('wrun_warm_pickup', ['running']); + const probes = [stalled, warm]; + // biome-ignore lint/suspicious/noExplicitAny: minimal Run stand-in + const startProbe = vi.fn(async () => probes.shift() as any); + await warmDeployment(startProbe, { + pickupBudgetMs: 300, + totalBudgetMs: 10_000, + }); + expect(startProbe).toHaveBeenCalledTimes(2); + expect(stalled.cancel).toHaveBeenCalledTimes(1); + expect(warm.cancel).not.toHaveBeenCalled(); + + const events = getRecordedInfraEvents().slice(before); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + kind: 'cold-start-warmup', + testName: 'suite warmup', + runId: 'wrun_warm_stall', + stalledProbeRunIds: ['wrun_warm_stall'], + pickedUpRunId: 'wrun_warm_pickup', + }); + }); + + test('an exhausted budget records the warmup with no pickup and returns', async () => { + const before = eventsBefore(); + let n = 0; + const startProbe = vi.fn(async () => { + n++; + // biome-ignore lint/suspicious/noExplicitAny: minimal Run stand-in + return makeProbe(`wrun_warm_${n}`, ['pending']) as any; + }); + await warmDeployment(startProbe, { + pickupBudgetMs: 200, + totalBudgetMs: 500, + }); + expect(startProbe.mock.calls.length).toBeGreaterThanOrEqual(1); + + const events = getRecordedInfraEvents().slice(before); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + kind: 'cold-start-warmup', + pickedUpRunId: null, + }); + expect( + (events[0] as { stalledProbeRunIds: string[] }).stalledProbeRunIds.length + ).toBe(startProbe.mock.calls.length); + }); +}); diff --git a/packages/core/e2e/utils.ts b/packages/core/e2e/utils.ts index d4bf552b40..7c0779ab0a 100644 --- a/packages/core/e2e/utils.ts +++ b/packages/core/e2e/utils.ts @@ -814,21 +814,54 @@ export function trackRun( // cluster of events in one time window reads as the platform blip it is. // --------------------------------------------------------------------------- -interface InfraEvent { - kind: 'run-pickup-stall'; +interface InfraEventBase { testName: string; + waitedMs: number; + timestamp: string; +} + +/** A mid-suite run the queue never picked up; abandoned and replaced. */ +interface RunPickupStallEvent extends InfraEventBase { + kind: 'run-pickup-stall'; /** The run that was abandoned. */ runId: string; /** The run started in its place. */ replacementRunId: string; - waitedMs: number; - timestamp: string; } +/** + * A fresh deployment needed more than one warmup probe before its queue + * consumer picked anything up (see `warmDeployment`). One event per suite, + * not per stalled probe. + */ +interface ColdStartWarmupEvent extends InfraEventBase { + kind: 'cold-start-warmup'; + /** First abandoned probe (what the aggregation renders). */ + runId: string; + /** Every probe that stalled, in order. */ + stalledProbeRunIds: string[]; + /** The probe that was finally picked up, or null if the budget ran out. */ + pickedUpRunId: string | null; +} + +type InfraEvent = RunPickupStallEvent | ColdStartWarmupEvent; + +/** `Omit` that distributes over a union instead of collapsing it. */ +type DistributiveOmit = T extends unknown + ? Omit + : never; + const infraEvents: InfraEvent[] = []; +/** Test-only visibility into events recorded so far. */ +export function getRecordedInfraEvents(): readonly InfraEvent[] { + return infraEvents; +} + export function recordInfraEvent( - event: Omit & { testName?: string } + event: DistributiveOmit & { + testName?: string; + } ) { infraEvents.push({ ...event, @@ -948,6 +981,93 @@ export async function startTracked( return replacement; } +/** + * Total budget for warming a fresh deployment before the suite runs. + */ +const WARMUP_BUDGET_MS = Number( + process.env.WORKFLOW_E2E_WARMUP_BUDGET_MS ?? '120000' +); + +/** + * Warm a cold target before the first test starts a run. + * + * A target answers HTTP well before its first run is picked up promptly, + * for two reasons with one shape: a fresh Vercel deployment's queue + * consumer takes a while to start delivering, and a local dev server pays + * its first flow-route compile on the first delivery (observed as the + * suite's first test recording a pickup stall on local-dev lanes, `waitedMs` + * pegged at the full pickup budget). Either way the stalls the watchdog + * absorbs cluster on the suite's first tests, which drowns the infra + * telemetry in cold-start noise and leaves those tests one stalled + * replacement away from failing. + * + * Probes follow the watchdog's shape: start a throwaway run, and if it is + * still `pending` after `WORKFLOW_E2E_PICKUP_BUDGET_MS`, abandon it + * (best-effort cancel) and probe again, until a probe is picked up or + * `WORKFLOW_E2E_WARMUP_BUDGET_MS` is spent. A picked-up probe is left to + * finish on its own โ€” pickup is what proves the pipeline is awake. A warmup + * that needed abandoned probes is recorded as a single `cold-start-warmup` + * infra event instead of per-test `run-pickup-stall` noise. + * + * If the budget runs out the suite proceeds anyway: the per-test watchdog + * still guards every start, and test failures carry the run diagnostics a + * thrown warmup would not. + */ +export async function warmDeployment( + startProbe: () => Promise>, + { + pickupBudgetMs = PICKUP_BUDGET_MS, + totalBudgetMs = WARMUP_BUDGET_MS, + }: { pickupBudgetMs?: number; totalBudgetMs?: number } = {} +): Promise { + const startedAt = Date.now(); + const deadline = startedAt + totalBudgetMs; + const stalledProbeRunIds: string[] = []; + + const record = (pickedUpRunId: string | null) => { + if (stalledProbeRunIds.length === 0) return; + recordInfraEvent({ + kind: 'cold-start-warmup', + testName: 'suite warmup', + runId: stalledProbeRunIds[0], + stalledProbeRunIds: [...stalledProbeRunIds], + pickedUpRunId, + waitedMs: Date.now() - startedAt, + }); + }; + + for (;;) { + const probe = await startProbe(); + const remaining = deadline - Date.now(); + if (await waitForRunPickup(probe, Math.min(pickupBudgetMs, remaining))) { + record(probe.runId); + if (stalledProbeRunIds.length > 0) { + console.warn( + `[e2e] deployment warmup: ${stalledProbeRunIds.length} probe(s) ` + + `stalled before ${probe.runId} was picked up ` + + `(${Date.now() - startedAt}ms; infra event, not a test failure)` + ); + } + return; + } + + stalledProbeRunIds.push(probe.runId); + void probe + .cancel({ cancelReason: 'e2e: warmup probe stuck pending, abandoned' }) + .catch(() => {}); + + if (deadline - Date.now() <= 0) { + record(null); + console.warn( + `[e2e] deployment warmup: no probe picked up within ` + + `${totalBudgetMs}ms (${stalledProbeRunIds.length} abandoned); ` + + `proceeding โ€” the per-test pickup watchdog still guards` + ); + return; + } + } +} + /** * Build a Vercel observability dashboard URL for a workflow run. */