Skip to content

feat(e2e): lazy app init + playwright sharding for integration tests #8019

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jacekradko wants to merge 10 commits into
base: main
Choose a base branch
from
Open

feat(e2e): lazy app init + playwright sharding for integration tests #8019

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
e0d8337
feat(e2e): add file-based process lock utility
jacekradko Mar 10, 2026
70989c1
feat(e2e): make longRunningApplication.init() lazy and idempotent
jacekradko Mar 10, 2026
92c3c32
feat(e2e): remove eager app init from global setup
jacekradko Mar 10, 2026
c1496d7
feat(e2e): trigger lazy app init from testAgainstRunningApps
jacekradko Mar 10, 2026
4e7c45d
feat(e2e): add PLAYWRIGHT_SHARD env var support to playwright config
jacekradko Mar 10, 2026
cbbf693
ci(repo): add playwright sharding for nextjs integration tests
jacekradko Mar 10, 2026
50a5db7
fix: use correct format placeholder and remove redundant comment
jacekradko Mar 10, 2026
72e025a
fix(e2e): propagate CLERK_FAPI and CLERK_TESTING_TOKEN to adopting wo…
jacekradko Mar 11, 2026
c877949
fix(e2e): guard destroy() and serialize state file writes
jacekradko Mar 11, 2026
7170adb
fix(e2e): guard handshake test teardown when app init fails
jacekradko Mar 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .changeset/lazy-app-init-sharding.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
---
---
30 changes: 28 additions & 2 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ jobs:
integration-tests:
needs: [check-permissions, build-packages]
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
name: Integration Tests (${{ matrix.test-name }}, ${{ matrix.test-project }}${{ matrix.next-version && format(', {0}', matrix.next-version) || '' }})
name: Integration Tests (${{ matrix.test-name }}, ${{ matrix.test-project }}${{ matrix.next-version && format(', {0}', matrix.next-version) || '' }}${{ matrix.shard && format(', shard {0}', matrix.shard) || '' }})
permissions:
contents: read
actions: write # needed for actions/upload-artifact
Expand Down Expand Up @@ -315,9 +315,33 @@ jobs:
- test-name: "nextjs"
test-project: "chrome"
next-version: "15"
shard: "1/3"
shard-label: "1-of-3"
- test-name: "nextjs"
test-project: "chrome"
next-version: "15"
shard: "2/3"
shard-label: "2-of-3"
- test-name: "nextjs"
test-project: "chrome"
next-version: "15"
shard: "3/3"
shard-label: "3-of-3"
- test-name: "nextjs"
test-project: "chrome"
next-version: "16"
shard: "1/3"
shard-label: "1-of-3"
- test-name: "nextjs"
test-project: "chrome"
next-version: "16"
shard: "2/3"
shard-label: "2-of-3"
- test-name: "nextjs"
test-project: "chrome"
next-version: "16"
shard: "3/3"
shard-label: "3-of-3"
- test-name: "quickstart"
test-project: "chrome"
next-version: "15"
Expand Down Expand Up @@ -365,6 +389,7 @@ jobs:
E2E_NEXTJS_VERSION: ${{ matrix.next-version }}
E2E_PROJECT: ${{ matrix.test-project }}
INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }}
PLAYWRIGHT_SHARD: ${{ matrix.shard || '' }}
run: |
# Use turbo's built-in --affected flag to detect changes
# This automatically uses GITHUB_BASE_REF in GitHub Actions
Expand Down Expand Up @@ -449,13 +474,14 @@ jobs:
E2E_CLERK_ENCRYPTION_KEY: ${{ matrix.clerk-encryption-key }}
INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }}
NODE_EXTRA_CA_CERTS: ${{ github.workspace }}/integration/certs/rootCA.pem
PLAYWRIGHT_SHARD: ${{ matrix.shard || '' }}
VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}

- name: Upload test-results
if: ${{ cancelled() || failure() }}
uses: actions/upload-artifact@v4
with:
name: playwright-traces-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.test-name }}${{ matrix.next-version && format('-next{0}', matrix.next-version) || '' }}
name: playwright-traces-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.test-name }}${{ matrix.next-version && format('-next{0}', matrix.next-version) || '' }}${{ matrix.shard-label && format('-shard-{0}', matrix.shard-label) || '' }}
path: integration/test-results
retention-days: 1

Expand Down
266 changes: 191 additions & 75 deletions integration/models/longRunningApplication.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { parsePublishableKey } from '@clerk/shared/keys';
import { clerkSetup } from '@clerk/testing/playwright';

import { awaitableTreekill, fs } from '../scripts';
import { acquireProcessLock, awaitableTreekill, fs } from '../scripts';
import type { Application } from './application';
import type { ApplicationConfig } from './applicationConfig';
import type { EnvironmentConfig } from './environment';
Expand All @@ -16,6 +16,18 @@ const getPort = (_url: string) => {
return Number.parseInt(url.port || (url.protocol === 'https:' ? '443' : '80'));
};

/**
* Check if a server is responding at the given URL.
*/
const isServerReady = async (url: string): Promise<boolean> => {
try {
const res = await fetch(url);
return res.ok;
} catch {
return false;
}
};

export type LongRunningApplication = ReturnType<typeof longRunningApplication>;
export type LongRunningApplicationParams = {
id: string;
Expand All @@ -29,7 +41,8 @@ export type LongRunningApplicationParams = {
* Its interface is the same as the Application and the ApplicationConfig interface,
* making it interchangeable with the Application and ApplicationConfig.
*
* After init() is called, all mutating methods on the config are ignored.
* init() is lazy and idempotent: it checks the state file first, and uses
* file-based locking to ensure only one process initializes each app.
*/
export const longRunningApplication = (params: LongRunningApplicationParams) => {
const { id } = params;
Expand All @@ -54,92 +67,195 @@ export const longRunningApplication = (params: LongRunningApplicationParams) =>
env ||= environmentConfig().fromJson(data.env);
};

/**
* Try to adopt an already-running app from the state file.
* Returns true if the app is running and state was loaded.
*/
const tryAdoptFromStateFile = async (): Promise<boolean> => {
try {
const apps = stateFile.getLongRunningApps();
const data = apps?.[id];
if (!data?.serverUrl) {
return false;
}
const ready = await isServerReady(data.serverUrl);
if (ready) {
port = data.port;
serverUrl = data.serverUrl;
pid = data.pid;
appDir = data.appDir;
env = params.env;
// Propagate testing tokens to this worker process so that
// setupClerkTestingToken() can bypass bot protection.
if (data.clerkFapi) {
process.env.CLERK_FAPI = data.clerkFapi;
}
if (data.clerkTestingToken) {
process.env.CLERK_TESTING_TOKEN = data.clerkTestingToken;
}
return true;
}
return false;
} catch {
// State file may be partially written by another process — not an error
return false;
}
};

/**
* Perform the full app initialization: testing tokens, commit, install, build, serve.
*/
const doFullInit = async () => {
const log = (msg: string) => console.log(`[${name}] ${msg}`);
log('Starting full init...');

try {
const publishableKey = params.env.publicVariables.get('CLERK_PUBLISHABLE_KEY');
const secretKey = params.env.privateVariables.get('CLERK_SECRET_KEY');
const apiUrl = params.env.privateVariables.get('CLERK_API_URL');
const { instanceType, frontendApi: frontendApiUrl } = parsePublishableKey(publishableKey);
Copy link
Contributor

@coderabbitai coderabbitai bot Mar 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

Handle null return from parsePublishableKey to prevent runtime crash.

parsePublishableKey returns PublishableKey | null. If the publishable key is missing or malformed, this line will throw TypeError: Cannot destructure property 'instanceType' of '...' as it is null.

🐛 Proposed fix 
-      const { instanceType, frontendApi: frontendApiUrl } = parsePublishableKey(publishableKey);
+      const parsed = parsePublishableKey(publishableKey);
+      if (!parsed) {
+        throw new Error(`Invalid or missing CLERK_PUBLISHABLE_KEY for ${name}`);
+      }
+      const { instanceType, frontendApi: frontendApiUrl } = parsed;
🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@integration/models/longRunningApplication.ts` at line 116,
parsePublishableKey can return null, so avoid directly destructuring its result;
in the code around the call to parsePublishableKey(publishableKey) (where you
currently extract instanceType and frontendApi), first capture the return value
into a variable (e.g., parsedKey = parsePublishableKey(publishableKey)), check
if parsedKey is null, and handle that case (throw a clear error or return early)
before using parsedKey.instanceType and parsedKey.frontendApi; update any
callers/logic in longRunningApplication.ts that rely on instanceType/frontendApi
accordingly to ensure a safe path when parsePublishableKey returns null.


if (instanceType !== 'development') {
log('Skipping setup of testing tokens for non-development instance');
} else {
log('Setting up testing tokens...');
await clerkSetup({
publishableKey,
frontendApiUrl,
secretKey,
// @ts-expect-error apiUrl is not a typed option for clerkSetup, but it is accepted at runtime.
apiUrl,
dotenv: false,
});
log('Testing tokens setup complete');
}
} catch (error) {
console.error('Error setting up testing tokens:', error);
throw error;
}

try {
log('Committing config...');
app = await config.commit();
log(`Config committed, appDir: ${app.appDir}`);
} catch (error) {
console.error('Error committing config:', error);
throw error;
}

try {
await app.withEnv(params.env);
} catch (error) {
console.error('Error setting up environment:', error);
throw error;
}

try {
log('Running setup (pnpm install)...');
await app.setup();
log('Setup complete');
} catch (error) {
console.error('Error during app setup:', error);
throw error;
}

try {
log('Building app...');
const buildTimeout = new Promise((_, reject) =>
setTimeout(() => reject(new Error(`Build timed out after 120s for ${name}`)), 120_000),
);
await Promise.race([app.build(), buildTimeout]);
log('Build complete');
} catch (error) {
console.error('Error during app build:', error);
throw error;
}

try {
log('Starting serve (detached)...');
const serveResult = await app.serve({ detached: true });
port = serveResult.port;
serverUrl = serveResult.serverUrl;
pid = serveResult.pid;
appDir = app.appDir;
log(`Serve complete: port=${port}, serverUrl=${serverUrl}, pid=${pid}`);
// Serialize state file writes across all apps to prevent concurrent
// read-modify-write from clobbering entries written by other workers.
const releaseStateFileLock = await acquireProcessLock('__state-file__');
try {
stateFile.addLongRunningApp({
port,
serverUrl,
pid,
id,
appDir,
env: params.env.toJson(),
clerkFapi: process.env.CLERK_FAPI,
clerkTestingToken: process.env.CLERK_TESTING_TOKEN,
});
} finally {
releaseStateFileLock();
}
} catch (error) {
console.error('Error during app serve:', error);
throw error;
}
};

const self = new Proxy(
{
// will be called by global.setup.ts and by the test runner
// the first time this is called, the app starts and the state is persisted in the state file
/**
* Lazy, idempotent init. Safe to call from multiple Playwright workers.
* - If the app is already running (found in state file + server responds), reuses it.
* - Otherwise, acquires a file lock and initializes. Other workers wait for the lock.
*/
init: async () => {
const log = (msg: string) => console.log(`[${name}] ${msg}`);
log('Starting init...');
try {
const publishableKey = params.env.publicVariables.get('CLERK_PUBLISHABLE_KEY');
const secretKey = params.env.privateVariables.get('CLERK_SECRET_KEY');
const apiUrl = params.env.privateVariables.get('CLERK_API_URL');
const { instanceType, frontendApi: frontendApiUrl } = parsePublishableKey(publishableKey);

if (instanceType !== 'development') {
log('Skipping setup of testing tokens for non-development instance');
} else {
log('Setting up testing tokens...');
await clerkSetup({
publishableKey,
frontendApiUrl,
secretKey,
// @ts-expect-error apiUrl is not a typed option for clerkSetup, but it is accepted at runtime.
apiUrl,
dotenv: false,
});
log('Testing tokens setup complete');
}
} catch (error) {
console.error('Error setting up testing tokens:', error);
throw error;
}
try {
log('Committing config...');
app = await config.commit();
log(`Config committed, appDir: ${app.appDir}`);
} catch (error) {
console.error('Error committing config:', error);
throw error;
}
try {
await app.withEnv(params.env);
} catch (error) {
console.error('Error setting up environment:', error);
throw error;
}
try {
log('Running setup (pnpm install)...');
await app.setup();
log('Setup complete');
} catch (error) {
console.error('Error during app setup:', error);
throw error;

// Fast path: already initialized in this process
if (serverUrl && (await isServerReady(serverUrl))) {
log('Already initialized in this process');
return;
}
try {
log('Building app...');
const buildTimeout = new Promise((_, reject) =>
setTimeout(() => reject(new Error(`Build timed out after 120s for ${name}`)), 120_000),
);
await Promise.race([app.build(), buildTimeout]);
log('Build complete');
} catch (error) {
console.error('Error during app build:', error);
throw error;

// Check if another process already initialized this app
if (await tryAdoptFromStateFile()) {
log(`Adopted from state file: ${serverUrl}`);
return;
}

// Need to initialize — acquire lock to prevent duplicate work
log('Acquiring init lock...');
const releaseLock = await acquireProcessLock(id);
try {
log('Starting serve (detached)...');
const serveResult = await app.serve({ detached: true });
port = serveResult.port;
serverUrl = serveResult.serverUrl;
pid = serveResult.pid;
appDir = app.appDir;
log(`Serve complete: port=${port}, serverUrl=${serverUrl}, pid=${pid}`);
stateFile.addLongRunningApp({ port, serverUrl, pid, id, appDir, env: params.env.toJson() });
} catch (error) {
console.error('Error during app serve:', error);
throw error;
// Double-check after acquiring lock (another process may have finished while we waited)
if (await tryAdoptFromStateFile()) {
log(`Adopted from state file after lock: ${serverUrl}`);
return;
}

// We hold the lock and the app is not running — do full init
await doFullInit();
} finally {
releaseLock();
}
},
// will be called by global.teardown.ts
destroy: async () => {
readFromStateFile();
if (!pid && !appDir) {
console.log(`Skipping destroy for ${name}: no pid or appDir`);
return;
}
console.log(`Destroying ${serverUrl}`);
await awaitableTreekill(pid, 'SIGKILL');
// TODO: Test whether this is necessary now that we have awaitableTreekill
await new Promise(res => setTimeout(res, 2000));
await fs.rm(appDir, { recursive: true, force: true });
if (pid) {
await awaitableTreekill(pid, 'SIGKILL');
// TODO: Test whether this is necessary now that we have awaitableTreekill
await new Promise(res => setTimeout(res, 2000));
}
if (appDir) {
await fs.rm(appDir, { recursive: true, force: true });
}
},
// read the persisted state and behave like an app
commit: () => {
Expand Down
2 changes: 2 additions & 0 deletions integration/models/stateFile.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ export type AppParams = {
pid?: number;
appDir: string;
env: ReturnType<EnvironmentConfig['toJson']>;
clerkFapi?: string;
clerkTestingToken?: string;
};

type StandaloneAppParams = {
Expand Down
Loading
Loading