ci(previews): run the preview e2e in its own job

.github/workflows/deploy-pr-preview.yml

@@ -63,6 +63,8 @@ jobs:
                         || contains(github.event.pull_request.labels.*.name, 'preview:web')))) }}
         runs-on: ubuntu-latest
         timeout-minutes: 45
+        outputs:
+            smoke: ${{ steps.smoke.outcome }}
         # `pr-preview` holds its own Infisical identity, limited to the preview environment, so
         # PR code can never request `prod`. Required reviewers would also gate teardowns on close;
         # cleanup-preview-orphans.yml reaps any left waiting.
@@ -164,31 +166,6 @@ jobs:
                   WEB_URL: ${{ steps.deploy.outputs.web_url }}
               run: bun scripts/preview-seed-smoke.ts
 
-            # Browser e2e against the fresh preview (apps/web/e2e/README.md). Non-blocking while
-            # it beds in; `agent.*` tests register only once ANTHROPIC_API_KEY is in Infisical.
-            - name: End-to-end tests against the preview
-              id: e2e
-              if: ${{ env.TEARDOWN != 'true' && steps.smoke.outcome == 'success' }}
-              continue-on-error: true
-              working-directory: apps/web/e2e
-              env:
-                  MAPLE_E2E_APP_URL: ${{ steps.deploy.outputs.web_url }}
-                  MAPLE_E2E_API_URL: ${{ steps.deploy.outputs.api_url }}
-                  E2E_TELEMETRY_DISABLED: "1"
-              run: |
-                  bunx e2e-web install chromium --with-deps
-                  bunx e2e run --reporter list,junit
-
-            - name: Upload e2e results
-              if: ${{ !cancelled() && steps.e2e.outcome != 'skipped' }}
-              uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
-              with:
-                  name: e2e-results
-                  path: apps/web/e2e/.e2e/
-                  if-no-files-found: warn
-                  include-hidden-files: true
-                  retention-days: 7
-
             - name: Destroy PR preview stack with Alchemy
               id: destroy
               if: ${{ env.TEARDOWN == 'true' }}
@@ -304,3 +281,94 @@ jobs:
                   else
                     gh pr comment "$PR_NUMBER" --body-file "$COMMENT_OUTPUT"
                   fi
+
+    # Browser e2e against the fresh preview (apps/web/e2e/README.md). Its own job so a slow
+    # run does not hold the deploy, it reruns without redeploying, and it never sees the
+    # deploy's AWS role or the rest of the preview's secrets.
+    e2e:
+        name: End-to-end tests against the preview
+        needs: deploy-pr-preview
+        # `!cancelled()`: the implicit success() fails on the skipped ingest-binary upstream.
+        if: ${{ !cancelled() && needs.deploy-pr-preview.result == 'success' && needs.deploy-pr-preview.outputs.smoke == 'success' }}
+        runs-on: ubuntu-latest
+        timeout-minutes: 20
+        # Checkout and the Infisical OIDC exchange; none of the workflow's write scopes.
+        permissions:
+            contents: read
+            id-token: write
+        # The Infisical identity is scoped to this environment.
+        environment: pr-preview
+        defaults:
+            run:
+                working-directory: apps/web/e2e
+        env:
+            # Not the deploy's URL outputs: GitHub drops an output that contains any masked
+            # value, and short Infisical values occur in these hostnames.
+            MAPLE_E2E_APP_URL: https://app-pr-${{ github.event.pull_request.number }}.maple.dev
+            MAPLE_E2E_API_URL: https://api-pr-${{ github.event.pull_request.number }}.maple.dev
+            E2E_TELEMETRY_DISABLED: "1"
+            # `preview:web` deploys no Electric sync; tests that read synced state skip.
+            MAPLE_E2E_NO_SYNC: ${{ !contains(github.event.pull_request.labels.*.name, 'preview') && '1' || '' }}
+        steps:
+            # The suite runs PR code beside the sign-in key; no later step needs the token.
+            - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
+              with:
+                  persist-credentials: false
+
+            - uses: ./.github/actions/setup
+              with:
+                  install-filters: "@maple/web"
+                  turbo-cache: "false"
+
+            # @e2e-dev/web pins the Playwright build, so its version keys the browser.
+            - name: Read the e2e engine version
+              id: engine
+              run: echo "version=$(bun -e 'console.log(require("@e2e-dev/web/package.json").version)')" >> "$GITHUB_OUTPUT"
+
+            - name: Cache Chromium
+              id: browser-cache
+              uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+              with:
+                  path: ~/.cache/ms-playwright
+                  key: e2e-web-${{ runner.os }}-${{ steps.engine.outputs.version }}-chromium
+
+            # ubuntu-latest already carries Chromium's shared libraries (see ci.yml).
+            - name: Install Chromium
+              if: steps.browser-cache.outputs.cache-hit != 'true'
+              run: bunx e2e-web install chromium
+
+            # By name, so no other preview secret reaches this job (setup above runs PR code).
+            - name: Load the preview sign-in key from Infisical
+              uses: Infisical/secrets-action@d2e351f16c6ca20d17c85e6c992e04bdeb64e87d # v1.0.18
+              with:
+                  method: "oidc"
+                  identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }}
+                  project-slug: ${{ vars.INFISICAL_PROJECT_SLUG }}
+                  env-slug: ${{ vars.PREVIEW_INFISICAL_ENV_SLUG || 'dev' }}
+                  export-type: "env"
+                  secret-name: PREVIEW_SIGNIN_KEY
+
+            # Optional: agent.* tests register only when it is set, and a missing name throws.
+            - name: Load the agent test key from Infisical
+              continue-on-error: true
+              uses: Infisical/secrets-action@d2e351f16c6ca20d17c85e6c992e04bdeb64e87d # v1.0.18
+              with:
+                  method: "oidc"
+                  identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }}
+                  project-slug: ${{ vars.INFISICAL_PROJECT_SLUG }}
+                  env-slug: ${{ vars.PREVIEW_INFISICAL_ENV_SLUG || 'dev' }}
+                  export-type: "env"
+                  secret-name: ANTHROPIC_API_KEY
+
+            - name: Run the e2e suite
+              run: bunx e2e run --reporter list,junit
+
+            - name: Upload e2e results
+              if: ${{ !cancelled() }}
+              uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+              with:
+                  name: e2e-results
+                  path: apps/web/e2e/.e2e/
+                  if-no-files-found: warn
+                  include-hidden-files: true
+                  retention-days: 7

apps/web/e2e/README.md

@@ -20,5 +20,6 @@ E2E_TELEMETRY_DISABLED=1 MAPLE_E2E_APP_URL=https://app-pr-<n>.maple.dev \
   lines); those tests register only when `ANTHROPIC_API_KEY` is set.
 - Tests create their own rules and destinations with a run-unique suffix and delete them after.
 
-CI runs this after the preview's seed and smoke test (`deploy-pr-preview.yml`), non-blocking;
-traces and screenshots land in the `e2e-results` artifact.
+CI runs this as the `e2e` job of `deploy-pr-preview.yml`, after the preview's seed and smoke test.
+It is not a required check, and it reruns without redeploying. Traces and screenshots land in
+the `e2e-results` artifact.

apps/web/e2e/tests/alert-rules.e2e.ts

@@ -23,84 +23,91 @@ interface Rule {
 const toRuleUuid = Schema.decodeSync(AlertRulePublicId)
 
 test.describe("alert rules", () => {
-	// A rule written by the API or MCP, then edited in the UI: the path that dropped tiers.
-	test(
-		"keeps an API-made tiered rule intact through UI edits",
-		{ session: "reviewer" },
-		async ({ app, browser, screen }) => {
-			const appUrl = app.baseUrl ?? ""
-			await app.open("/alerts")
-			const tag = runTag()
-			const destination = (name: string) =>
-				api<{ id: string }>(browser, appUrl, "/v2/alerts/destinations", {
+	// Editing loads the rule over Electric sync, which a `preview:web` preview does not deploy.
+	if (process.env.MAPLE_E2E_NO_SYNC !== "1") {
+		// A rule written by the API or MCP, then edited in the UI: the path that dropped tiers.
+		test(
+			"keeps an API-made tiered rule intact through UI edits",
+			{ session: "reviewer" },
+			async ({ app, browser, screen }) => {
+				const appUrl = app.baseUrl ?? ""
+				await app.open("/alerts")
+				const tag = runTag()
+				const destination = (name: string) =>
+					api<{ id: string }>(browser, appUrl, "/v2/alerts/destinations", {
+						method: "POST",
+						body: {
+							type: "webhook",
+							name: `e2e ${name} ${tag}`,
+							url: "https://example.com/maple-e2e",
+						},
+					})
+				const warn = await destination("warn")
+				const page = await destination("page")
+				const tiers: Tier[] = [
+					{
+						severity: "warning",
+						value: 0.007,
+						upper: null,
+						recovery_value: 0.005,
+						destination_ids: [warn.body.id],
+					},
+					{
+						severity: "critical",
+						value: 0.03,
+						upper: null,
+						recovery_value: null,
+						destination_ids: [page.body.id],
+					},
+				]
+				const created = await api<Rule>(browser, appUrl, "/v2/alerts/rules", {
 					method: "POST",
 					body: {
-						type: "webhook",
-						name: `e2e ${name} ${tag}`,
-						url: "https://example.com/maple-e2e",
+						name: `e2e tiers ${tag}`,
+						signal_type: "error_rate",
+						comparator: "gt",
+						window_minutes: 5,
+						consecutive_breaches_required: 2,
+						breaches_out_of: 4,
+						thresholds: tiers,
 					},
 				})
-			const warn = await destination("warn")
-			const page = await destination("page")
-			const tiers: Tier[] = [
-				{
-					severity: "warning",
-					value: 0.007,
-					upper: null,
-					recovery_value: 0.005,
-					destination_ids: [warn.body.id],
-				},
-				{
-					severity: "critical",
-					value: 0.03,
-					upper: null,
-					recovery_value: null,
-					destination_ids: [page.body.id],
-				},
-			]
-			const created = await api<Rule>(browser, appUrl, "/v2/alerts/rules", {
-				method: "POST",
-				body: {
-					name: `e2e tiers ${tag}`,
-					signal_type: "error_rate",
-					comparator: "gt",
-					window_minutes: 5,
-					consecutive_breaches_required: 2,
-					breaches_out_of: 4,
-					thresholds: tiers,
-				},
-			})
-			expect(created.status).toBe(200)
-			const editUrl = `/alerts/create?ruleId=${toRuleUuid(created.body.id)}`
-			const read = () => api<Rule>(browser, appUrl, `/v2/alerts/rules/${created.body.id}`)
+				expect(created.status).toBe(200)
+				const editUrl = `/alerts/create?ruleId=${toRuleUuid(created.body.id)}`
+				const read = () => api<Rule>(browser, appUrl, `/v2/alerts/rules/${created.body.id}`)
 
-			try {
-				await app.open(editUrl)
-				await expect(browser.locator("#rule-threshold")).toHaveValue("0.7")
-				await expect(screen.getByLabel("Critical threshold")).toHaveValue("3")
-				await expect(screen.getByLabel("Out of the last")).toHaveValue("4")
+				try {
+					await app.open(editUrl)
+					await expect(browser.locator("#rule-threshold")).toHaveValue("0.7")
+					await expect(screen.getByLabel("Critical threshold")).toHaveValue("3")
+					await expect(screen.getByLabel("Out of the last")).toHaveValue("4")
 
-				await screen.getByLabel("Rule name").fill(`e2e tiers renamed ${tag}`)
-				await screen.getByRole("button", "Save changes").tap()
-				await expect(browser).toHaveURL(/\/alerts$/)
-				const renamed = await read()
-				expect(renamed.body.name).toBe(`e2e tiers renamed ${tag}`)
-				expect(renamed.body.thresholds).toEqual(tiers)
-				expect(renamed.body.breaches_out_of).toBe(4)
+					await screen.getByLabel("Rule name").fill(`e2e tiers renamed ${tag}`)
+					await screen.getByRole("button", "Save changes").tap()
+					await expect(browser).toHaveURL(/\/alerts$/)
+					const renamed = await read()
+					expect(renamed.body.name).toBe(`e2e tiers renamed ${tag}`)
+					expect(renamed.body.thresholds).toEqual(tiers)
+					expect(renamed.body.breaches_out_of).toBe(4)
 
-				await app.open(editUrl)
-				await screen.getByLabel("Critical threshold").fill("4")
-				await screen.getByRole("button", "Save changes").tap()
-				await expect(browser).toHaveURL(/\/alerts$/)
-				const retuned = await read()
-				expect(retuned.body.thresholds.map((tier) => tier.value)).toEqual([0.007, 0.04])
-			} finally {
-				await api(browser, appUrl, `/v2/alerts/rules/${created.body.id}`, { method: "DELETE" })
-				await api(browser, appUrl, `/v2/alerts/destinations/${warn.body.id}`, { method: "DELETE" })
-				await api(browser, appUrl, `/v2/alerts/destinations/${page.body.id}`, { method: "DELETE" })
-			}
-		},
-	)
+					await app.open(editUrl)
+					await screen.getByLabel("Critical threshold").fill("4")
+					await screen.getByRole("button", "Save changes").tap()
+					await expect(browser).toHaveURL(/\/alerts$/)
+					const retuned = await read()
+					expect(retuned.body.thresholds.map((tier) => tier.value)).toEqual([0.007, 0.04])
+				} finally {
+					await api(browser, appUrl, `/v2/alerts/rules/${created.body.id}`, { method: "DELETE" })
+					await api(browser, appUrl, `/v2/alerts/destinations/${warn.body.id}`, {
+						method: "DELETE",
+					})
+					await api(browser, appUrl, `/v2/alerts/destinations/${page.body.id}`, {
+						method: "DELETE",
+					})
+				}
+			},
+		)
+	}
 
 	test("names an empty range bound before save", { session: "reviewer" }, async ({ app, screen }) => {
 		await app.open("/alerts/create")