From 45a744840185f1103a948c6d145637fa2e609239 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 23 Oct 2026 01:42:49 +0200 Subject: [PATCH 1/6] ci(previews): run the preview e2e in its own job The browser e2e ran as a step of the deploy job, so it held the deploy (and the preview comment) for its whole run, could only be retried by redeploying, and ran third-party test tooling next to the AWS deploy role and every preview secret. - New `e2e` job after `deploy-pr-preview`, reading the URLs from its outputs. - It loads only PREVIEW_SIGNIN_KEY and ANTHROPIC_API_KEY: Infisical exports to a file, the two keys move to the env masked, the file is deleted. - Chromium is cached by the @e2e-dev/web version instead of installed with apt deps every run. - No longer continue-on-error: a failure shows as its own red, non-required check. --- .github/workflows/deploy-pr-preview.yml | 112 ++++++++++++++++++------ apps/web/e2e/README.md | 5 +- 2 files changed, 90 insertions(+), 27 deletions(-) diff --git a/.github/workflows/deploy-pr-preview.yml b/.github/workflows/deploy-pr-preview.yml index 27ea9c052b..c4b685bda9 100644 --- a/.github/workflows/deploy-pr-preview.yml +++ b/.github/workflows/deploy-pr-preview.yml @@ -63,6 +63,10 @@ jobs: || contains(github.event.pull_request.labels.*.name, 'preview:web')))) }} runs-on: ubuntu-latest timeout-minutes: 45 + outputs: + web_url: ${{ steps.deploy.outputs.web_url }} + api_url: ${{ steps.deploy.outputs.api_url }} + smoke: ${{ steps.smoke.outcome }} # `pr-preview` holds its own Infisical identity, limited to the preview environment, so # PR code can never request `prod`. Required reviewers would also gate teardowns on close; # cleanup-preview-orphans.yml reaps any left waiting. @@ -164,31 +168,6 @@ jobs: WEB_URL: ${{ steps.deploy.outputs.web_url }} run: bun scripts/preview-seed-smoke.ts - # Browser e2e against the fresh preview (apps/web/e2e/README.md). Non-blocking while - # it beds in; `agent.*` tests register only once ANTHROPIC_API_KEY is in Infisical. - - name: End-to-end tests against the preview - id: e2e - if: ${{ env.TEARDOWN != 'true' && steps.smoke.outcome == 'success' }} - continue-on-error: true - working-directory: apps/web/e2e - env: - MAPLE_E2E_APP_URL: ${{ steps.deploy.outputs.web_url }} - MAPLE_E2E_API_URL: ${{ steps.deploy.outputs.api_url }} - E2E_TELEMETRY_DISABLED: "1" - run: | - bunx e2e-web install chromium --with-deps - bunx e2e run --reporter list,junit - - - name: Upload e2e results - if: ${{ !cancelled() && steps.e2e.outcome != 'skipped' }} - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-results - path: apps/web/e2e/.e2e/ - if-no-files-found: warn - include-hidden-files: true - retention-days: 7 - - name: Destroy PR preview stack with Alchemy id: destroy if: ${{ env.TEARDOWN == 'true' }} @@ -304,3 +283,86 @@ jobs: else gh pr comment "$PR_NUMBER" --body-file "$COMMENT_OUTPUT" fi + + # Browser e2e against the fresh preview (apps/web/e2e/README.md). Its own job so a slow + # run does not hold the deploy, it reruns without redeploying, and it never sees the + # deploy's AWS role or the rest of the preview's secrets. + e2e: + name: End-to-end tests against the preview + needs: deploy-pr-preview + if: ${{ needs.deploy-pr-preview.outputs.smoke == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 20 + # The Infisical identity is scoped to this environment. + environment: pr-preview + defaults: + run: + working-directory: apps/web/e2e + env: + MAPLE_E2E_APP_URL: ${{ needs.deploy-pr-preview.outputs.web_url }} + MAPLE_E2E_API_URL: ${{ needs.deploy-pr-preview.outputs.api_url }} + E2E_TELEMETRY_DISABLED: "1" + SECRETS_FILE: /.e2e-secrets.env + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - uses: ./.github/actions/setup + with: + install-filters: "@maple/web" + turbo-cache: "false" + + # @e2e-dev/web pins the Playwright build, so its version keys the browser. + - name: Read the e2e engine version + id: engine + run: echo "version=$(bun -e 'console.log(require("@e2e-dev/web/package.json").version)')" >> "$GITHUB_OUTPUT" + + - name: Cache Chromium + id: browser-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/ms-playwright + key: e2e-web-${{ runner.os }}-${{ steps.engine.outputs.version }}-chromium + + # ubuntu-latest already carries Chromium's shared libraries (see ci.yml). + - name: Install Chromium + if: steps.browser-cache.outputs.cache-hit != 'true' + run: bunx e2e-web install chromium + + # Exported to a file, not the env, so only the two keys the suite reads leave it. + - name: Load deployment secrets from Infisical + uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16 + with: + method: "oidc" + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ vars.INFISICAL_PROJECT_SLUG }} + env-slug: ${{ vars.PREVIEW_INFISICAL_ENV_SLUG || 'dev' }} + export-type: "file" + file-output-path: ${{ env.SECRETS_FILE }} + secret-path: "/" + + # The action writes `KEY='value'` lines under the workspace; agent.* tests register + # only when ANTHROPIC_API_KEY is set. + - name: Keep only the e2e secrets + working-directory: ${{ github.workspace }} + run: | + file="$GITHUB_WORKSPACE$SECRETS_FILE" + for key in PREVIEW_SIGNIN_KEY ANTHROPIC_API_KEY; do + value=$(sed -n "s/^$key='\(.*\)'\$/\1/p" "$file") + [ -n "$value" ] || continue + echo "::add-mask::$value" + echo "$key=$value" >> "$GITHUB_ENV" + done + rm -f "$file" + + - name: Run the e2e suite + run: bunx e2e run --reporter list,junit + + - name: Upload e2e results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: e2e-results + path: apps/web/e2e/.e2e/ + if-no-files-found: warn + include-hidden-files: true + retention-days: 7 diff --git a/apps/web/e2e/README.md b/apps/web/e2e/README.md index c827a08b44..5710054784 100644 --- a/apps/web/e2e/README.md +++ b/apps/web/e2e/README.md @@ -20,5 +20,6 @@ E2E_TELEMETRY_DISABLED=1 MAPLE_E2E_APP_URL=https://app-pr-.maple.dev \ lines); those tests register only when `ANTHROPIC_API_KEY` is set. - Tests create their own rules and destinations with a run-unique suffix and delete them after. -CI runs this after the preview's seed and smoke test (`deploy-pr-preview.yml`), non-blocking; -traces and screenshots land in the `e2e-results` artifact. +CI runs this as the `e2e` job of `deploy-pr-preview.yml`, after the preview's seed and smoke test. +It is not a required check, and it reruns without redeploying. Traces and screenshots land in +the `e2e-results` artifact. From 76b46c0e7bf557c3e92a1c77aef87f540bcfaa5f Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 23 Oct 2026 01:46:23 +0200 Subject: [PATCH 2/6] ci(previews): pin the e2e job's Infisical action to v1.0.18 like deploy-setup --- .github/workflows/deploy-pr-preview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-pr-preview.yml b/.github/workflows/deploy-pr-preview.yml index c4b685bda9..f5093c18a0 100644 --- a/.github/workflows/deploy-pr-preview.yml +++ b/.github/workflows/deploy-pr-preview.yml @@ -330,7 +330,7 @@ jobs: # Exported to a file, not the env, so only the two keys the suite reads leave it. - name: Load deployment secrets from Infisical - uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16 + uses: Infisical/secrets-action@d2e351f16c6ca20d17c85e6c992e04bdeb64e87d # v1.0.18 with: method: "oidc" identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} From 84ad22941f59cbdf24ad936863094c8221cc1fb6 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 23 Oct 2026 01:46:39 +0200 Subject: [PATCH 3/6] ci(previews): grant the e2e job only contents read and id-token write --- .github/workflows/deploy-pr-preview.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/deploy-pr-preview.yml b/.github/workflows/deploy-pr-preview.yml index f5093c18a0..4e8b7c210f 100644 --- a/.github/workflows/deploy-pr-preview.yml +++ b/.github/workflows/deploy-pr-preview.yml @@ -293,6 +293,10 @@ jobs: if: ${{ needs.deploy-pr-preview.outputs.smoke == 'success' }} runs-on: ubuntu-latest timeout-minutes: 20 + # Checkout and the Infisical OIDC exchange; none of the workflow's write scopes. + permissions: + contents: read + id-token: write # The Infisical identity is scoped to this environment. environment: pr-preview defaults: From 70a43ddf815bf9a7cdccbb7012e8ed539736c628 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 23 Oct 2026 01:50:36 +0200 Subject: [PATCH 4/6] ci(previews): run the e2e job when ingest-binary was skipped upstream --- .github/workflows/deploy-pr-preview.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-pr-preview.yml b/.github/workflows/deploy-pr-preview.yml index 4e8b7c210f..cb53a10098 100644 --- a/.github/workflows/deploy-pr-preview.yml +++ b/.github/workflows/deploy-pr-preview.yml @@ -290,7 +290,8 @@ jobs: e2e: name: End-to-end tests against the preview needs: deploy-pr-preview - if: ${{ needs.deploy-pr-preview.outputs.smoke == 'success' }} + # `!cancelled()`: the implicit success() fails on the skipped ingest-binary upstream. + if: ${{ !cancelled() && needs.deploy-pr-preview.result == 'success' && needs.deploy-pr-preview.outputs.smoke == 'success' }} runs-on: ubuntu-latest timeout-minutes: 20 # Checkout and the Infisical OIDC exchange; none of the workflow's write scopes. From 3fbf9f65c3d5d2e1e5fa5ccf849792b17f11b7b9 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 23 Oct 2026 01:56:50 +0200 Subject: [PATCH 5/6] ci(previews): build the e2e URLs from the PR number; GitHub drops the deploy's URL outputs as secret-bearing --- .github/workflows/deploy-pr-preview.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-pr-preview.yml b/.github/workflows/deploy-pr-preview.yml index cb53a10098..6bf5be290d 100644 --- a/.github/workflows/deploy-pr-preview.yml +++ b/.github/workflows/deploy-pr-preview.yml @@ -64,8 +64,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 outputs: - web_url: ${{ steps.deploy.outputs.web_url }} - api_url: ${{ steps.deploy.outputs.api_url }} smoke: ${{ steps.smoke.outcome }} # `pr-preview` holds its own Infisical identity, limited to the preview environment, so # PR code can never request `prod`. Required reviewers would also gate teardowns on close; @@ -304,8 +302,10 @@ jobs: run: working-directory: apps/web/e2e env: - MAPLE_E2E_APP_URL: ${{ needs.deploy-pr-preview.outputs.web_url }} - MAPLE_E2E_API_URL: ${{ needs.deploy-pr-preview.outputs.api_url }} + # Not the deploy's URL outputs: GitHub drops an output that contains any masked + # value, and short Infisical values occur in these hostnames. + MAPLE_E2E_APP_URL: https://app-pr-${{ github.event.pull_request.number }}.maple.dev + MAPLE_E2E_API_URL: https://api-pr-${{ github.event.pull_request.number }}.maple.dev E2E_TELEMETRY_DISABLED: "1" SECRETS_FILE: /.e2e-secrets.env steps: From 38b032174059cecfe6901e27f8cee574cdb05511 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 23 Oct 2026 02:05:34 +0200 Subject: [PATCH 6/6] ci(previews): fetch only the two e2e secrets by name; skip the sync-backed test on preview:web - Infisical `secret-name` fetches PREVIEW_SIGNIN_KEY (required) and ANTHROPIC_API_KEY (optional) into the env, so no other preview secret ever reaches the job. Drops the file export, its parsing and its path mismatch. - The e2e checkout no longer persists the token. - "keeps an API-made tiered rule intact through UI edits" loads the rule over Electric sync, which preview:web does not deploy: it failed on every web-only preview and passed on full ones. It registers only when sync exists (MAPLE_E2E_NO_SYNC). --- .github/workflows/deploy-pr-preview.yml | 39 ++++--- apps/web/e2e/tests/alert-rules.e2e.ts | 149 +++++++++++++----------- 2 files changed, 98 insertions(+), 90 deletions(-) diff --git a/.github/workflows/deploy-pr-preview.yml b/.github/workflows/deploy-pr-preview.yml index 6bf5be290d..98d54387e0 100644 --- a/.github/workflows/deploy-pr-preview.yml +++ b/.github/workflows/deploy-pr-preview.yml @@ -307,9 +307,13 @@ jobs: MAPLE_E2E_APP_URL: https://app-pr-${{ github.event.pull_request.number }}.maple.dev MAPLE_E2E_API_URL: https://api-pr-${{ github.event.pull_request.number }}.maple.dev E2E_TELEMETRY_DISABLED: "1" - SECRETS_FILE: /.e2e-secrets.env + # `preview:web` deploys no Electric sync; tests that read synced state skip. + MAPLE_E2E_NO_SYNC: ${{ !contains(github.event.pull_request.labels.*.name, 'preview') && '1' || '' }} steps: + # The suite runs PR code beside the sign-in key; no later step needs the token. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + with: + persist-credentials: false - uses: ./.github/actions/setup with: @@ -333,31 +337,28 @@ jobs: if: steps.browser-cache.outputs.cache-hit != 'true' run: bunx e2e-web install chromium - # Exported to a file, not the env, so only the two keys the suite reads leave it. - - name: Load deployment secrets from Infisical + # By name, so no other preview secret reaches this job (setup above runs PR code). + - name: Load the preview sign-in key from Infisical uses: Infisical/secrets-action@d2e351f16c6ca20d17c85e6c992e04bdeb64e87d # v1.0.18 with: method: "oidc" identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} project-slug: ${{ vars.INFISICAL_PROJECT_SLUG }} env-slug: ${{ vars.PREVIEW_INFISICAL_ENV_SLUG || 'dev' }} - export-type: "file" - file-output-path: ${{ env.SECRETS_FILE }} - secret-path: "/" + export-type: "env" + secret-name: PREVIEW_SIGNIN_KEY - # The action writes `KEY='value'` lines under the workspace; agent.* tests register - # only when ANTHROPIC_API_KEY is set. - - name: Keep only the e2e secrets - working-directory: ${{ github.workspace }} - run: | - file="$GITHUB_WORKSPACE$SECRETS_FILE" - for key in PREVIEW_SIGNIN_KEY ANTHROPIC_API_KEY; do - value=$(sed -n "s/^$key='\(.*\)'\$/\1/p" "$file") - [ -n "$value" ] || continue - echo "::add-mask::$value" - echo "$key=$value" >> "$GITHUB_ENV" - done - rm -f "$file" + # Optional: agent.* tests register only when it is set, and a missing name throws. + - name: Load the agent test key from Infisical + continue-on-error: true + uses: Infisical/secrets-action@d2e351f16c6ca20d17c85e6c992e04bdeb64e87d # v1.0.18 + with: + method: "oidc" + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ vars.INFISICAL_PROJECT_SLUG }} + env-slug: ${{ vars.PREVIEW_INFISICAL_ENV_SLUG || 'dev' }} + export-type: "env" + secret-name: ANTHROPIC_API_KEY - name: Run the e2e suite run: bunx e2e run --reporter list,junit diff --git a/apps/web/e2e/tests/alert-rules.e2e.ts b/apps/web/e2e/tests/alert-rules.e2e.ts index 1169dd783d..83646759d4 100644 --- a/apps/web/e2e/tests/alert-rules.e2e.ts +++ b/apps/web/e2e/tests/alert-rules.e2e.ts @@ -23,84 +23,91 @@ interface Rule { const toRuleUuid = Schema.decodeSync(AlertRulePublicId) test.describe("alert rules", () => { - // A rule written by the API or MCP, then edited in the UI: the path that dropped tiers. - test( - "keeps an API-made tiered rule intact through UI edits", - { session: "reviewer" }, - async ({ app, browser, screen }) => { - const appUrl = app.baseUrl ?? "" - await app.open("/alerts") - const tag = runTag() - const destination = (name: string) => - api<{ id: string }>(browser, appUrl, "/v2/alerts/destinations", { + // Editing loads the rule over Electric sync, which a `preview:web` preview does not deploy. + if (process.env.MAPLE_E2E_NO_SYNC !== "1") { + // A rule written by the API or MCP, then edited in the UI: the path that dropped tiers. + test( + "keeps an API-made tiered rule intact through UI edits", + { session: "reviewer" }, + async ({ app, browser, screen }) => { + const appUrl = app.baseUrl ?? "" + await app.open("/alerts") + const tag = runTag() + const destination = (name: string) => + api<{ id: string }>(browser, appUrl, "/v2/alerts/destinations", { + method: "POST", + body: { + type: "webhook", + name: `e2e ${name} ${tag}`, + url: "https://example.com/maple-e2e", + }, + }) + const warn = await destination("warn") + const page = await destination("page") + const tiers: Tier[] = [ + { + severity: "warning", + value: 0.007, + upper: null, + recovery_value: 0.005, + destination_ids: [warn.body.id], + }, + { + severity: "critical", + value: 0.03, + upper: null, + recovery_value: null, + destination_ids: [page.body.id], + }, + ] + const created = await api(browser, appUrl, "/v2/alerts/rules", { method: "POST", body: { - type: "webhook", - name: `e2e ${name} ${tag}`, - url: "https://example.com/maple-e2e", + name: `e2e tiers ${tag}`, + signal_type: "error_rate", + comparator: "gt", + window_minutes: 5, + consecutive_breaches_required: 2, + breaches_out_of: 4, + thresholds: tiers, }, }) - const warn = await destination("warn") - const page = await destination("page") - const tiers: Tier[] = [ - { - severity: "warning", - value: 0.007, - upper: null, - recovery_value: 0.005, - destination_ids: [warn.body.id], - }, - { - severity: "critical", - value: 0.03, - upper: null, - recovery_value: null, - destination_ids: [page.body.id], - }, - ] - const created = await api(browser, appUrl, "/v2/alerts/rules", { - method: "POST", - body: { - name: `e2e tiers ${tag}`, - signal_type: "error_rate", - comparator: "gt", - window_minutes: 5, - consecutive_breaches_required: 2, - breaches_out_of: 4, - thresholds: tiers, - }, - }) - expect(created.status).toBe(200) - const editUrl = `/alerts/create?ruleId=${toRuleUuid(created.body.id)}` - const read = () => api(browser, appUrl, `/v2/alerts/rules/${created.body.id}`) + expect(created.status).toBe(200) + const editUrl = `/alerts/create?ruleId=${toRuleUuid(created.body.id)}` + const read = () => api(browser, appUrl, `/v2/alerts/rules/${created.body.id}`) - try { - await app.open(editUrl) - await expect(browser.locator("#rule-threshold")).toHaveValue("0.7") - await expect(screen.getByLabel("Critical threshold")).toHaveValue("3") - await expect(screen.getByLabel("Out of the last")).toHaveValue("4") + try { + await app.open(editUrl) + await expect(browser.locator("#rule-threshold")).toHaveValue("0.7") + await expect(screen.getByLabel("Critical threshold")).toHaveValue("3") + await expect(screen.getByLabel("Out of the last")).toHaveValue("4") - await screen.getByLabel("Rule name").fill(`e2e tiers renamed ${tag}`) - await screen.getByRole("button", "Save changes").tap() - await expect(browser).toHaveURL(/\/alerts$/) - const renamed = await read() - expect(renamed.body.name).toBe(`e2e tiers renamed ${tag}`) - expect(renamed.body.thresholds).toEqual(tiers) - expect(renamed.body.breaches_out_of).toBe(4) + await screen.getByLabel("Rule name").fill(`e2e tiers renamed ${tag}`) + await screen.getByRole("button", "Save changes").tap() + await expect(browser).toHaveURL(/\/alerts$/) + const renamed = await read() + expect(renamed.body.name).toBe(`e2e tiers renamed ${tag}`) + expect(renamed.body.thresholds).toEqual(tiers) + expect(renamed.body.breaches_out_of).toBe(4) - await app.open(editUrl) - await screen.getByLabel("Critical threshold").fill("4") - await screen.getByRole("button", "Save changes").tap() - await expect(browser).toHaveURL(/\/alerts$/) - const retuned = await read() - expect(retuned.body.thresholds.map((tier) => tier.value)).toEqual([0.007, 0.04]) - } finally { - await api(browser, appUrl, `/v2/alerts/rules/${created.body.id}`, { method: "DELETE" }) - await api(browser, appUrl, `/v2/alerts/destinations/${warn.body.id}`, { method: "DELETE" }) - await api(browser, appUrl, `/v2/alerts/destinations/${page.body.id}`, { method: "DELETE" }) - } - }, - ) + await app.open(editUrl) + await screen.getByLabel("Critical threshold").fill("4") + await screen.getByRole("button", "Save changes").tap() + await expect(browser).toHaveURL(/\/alerts$/) + const retuned = await read() + expect(retuned.body.thresholds.map((tier) => tier.value)).toEqual([0.007, 0.04]) + } finally { + await api(browser, appUrl, `/v2/alerts/rules/${created.body.id}`, { method: "DELETE" }) + await api(browser, appUrl, `/v2/alerts/destinations/${warn.body.id}`, { + method: "DELETE", + }) + await api(browser, appUrl, `/v2/alerts/destinations/${page.body.id}`, { + method: "DELETE", + }) + } + }, + ) + } test("names an empty range bound before save", { session: "reviewer" }, async ({ app, screen }) => { await app.open("/alerts/create")