diff --git a/.github/workflows/deploy-pr-preview.yml b/.github/workflows/deploy-pr-preview.yml index 27ea9c052..98d54387e 100644 --- a/.github/workflows/deploy-pr-preview.yml +++ b/.github/workflows/deploy-pr-preview.yml @@ -63,6 +63,8 @@ jobs: || contains(github.event.pull_request.labels.*.name, 'preview:web')))) }} runs-on: ubuntu-latest timeout-minutes: 45 + outputs: + smoke: ${{ steps.smoke.outcome }} # `pr-preview` holds its own Infisical identity, limited to the preview environment, so # PR code can never request `prod`. Required reviewers would also gate teardowns on close; # cleanup-preview-orphans.yml reaps any left waiting. @@ -164,31 +166,6 @@ jobs: WEB_URL: ${{ steps.deploy.outputs.web_url }} run: bun scripts/preview-seed-smoke.ts - # Browser e2e against the fresh preview (apps/web/e2e/README.md). Non-blocking while - # it beds in; `agent.*` tests register only once ANTHROPIC_API_KEY is in Infisical. - - name: End-to-end tests against the preview - id: e2e - if: ${{ env.TEARDOWN != 'true' && steps.smoke.outcome == 'success' }} - continue-on-error: true - working-directory: apps/web/e2e - env: - MAPLE_E2E_APP_URL: ${{ steps.deploy.outputs.web_url }} - MAPLE_E2E_API_URL: ${{ steps.deploy.outputs.api_url }} - E2E_TELEMETRY_DISABLED: "1" - run: | - bunx e2e-web install chromium --with-deps - bunx e2e run --reporter list,junit - - - name: Upload e2e results - if: ${{ !cancelled() && steps.e2e.outcome != 'skipped' }} - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-results - path: apps/web/e2e/.e2e/ - if-no-files-found: warn - include-hidden-files: true - retention-days: 7 - - name: Destroy PR preview stack with Alchemy id: destroy if: ${{ env.TEARDOWN == 'true' }} @@ -304,3 +281,94 @@ jobs: else gh pr comment "$PR_NUMBER" --body-file "$COMMENT_OUTPUT" fi + + # Browser e2e against the fresh preview (apps/web/e2e/README.md). Its own job so a slow + # run does not hold the deploy, it reruns without redeploying, and it never sees the + # deploy's AWS role or the rest of the preview's secrets. + e2e: + name: End-to-end tests against the preview + needs: deploy-pr-preview + # `!cancelled()`: the implicit success() fails on the skipped ingest-binary upstream. + if: ${{ !cancelled() && needs.deploy-pr-preview.result == 'success' && needs.deploy-pr-preview.outputs.smoke == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 20 + # Checkout and the Infisical OIDC exchange; none of the workflow's write scopes. + permissions: + contents: read + id-token: write + # The Infisical identity is scoped to this environment. + environment: pr-preview + defaults: + run: + working-directory: apps/web/e2e + env: + # Not the deploy's URL outputs: GitHub drops an output that contains any masked + # value, and short Infisical values occur in these hostnames. + MAPLE_E2E_APP_URL: https://app-pr-${{ github.event.pull_request.number }}.maple.dev + MAPLE_E2E_API_URL: https://api-pr-${{ github.event.pull_request.number }}.maple.dev + E2E_TELEMETRY_DISABLED: "1" + # `preview:web` deploys no Electric sync; tests that read synced state skip. + MAPLE_E2E_NO_SYNC: ${{ !contains(github.event.pull_request.labels.*.name, 'preview') && '1' || '' }} + steps: + # The suite runs PR code beside the sign-in key; no later step needs the token. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + with: + persist-credentials: false + + - uses: ./.github/actions/setup + with: + install-filters: "@maple/web" + turbo-cache: "false" + + # @e2e-dev/web pins the Playwright build, so its version keys the browser. + - name: Read the e2e engine version + id: engine + run: echo "version=$(bun -e 'console.log(require("@e2e-dev/web/package.json").version)')" >> "$GITHUB_OUTPUT" + + - name: Cache Chromium + id: browser-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/ms-playwright + key: e2e-web-${{ runner.os }}-${{ steps.engine.outputs.version }}-chromium + + # ubuntu-latest already carries Chromium's shared libraries (see ci.yml). + - name: Install Chromium + if: steps.browser-cache.outputs.cache-hit != 'true' + run: bunx e2e-web install chromium + + # By name, so no other preview secret reaches this job (setup above runs PR code). + - name: Load the preview sign-in key from Infisical + uses: Infisical/secrets-action@d2e351f16c6ca20d17c85e6c992e04bdeb64e87d # v1.0.18 + with: + method: "oidc" + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ vars.INFISICAL_PROJECT_SLUG }} + env-slug: ${{ vars.PREVIEW_INFISICAL_ENV_SLUG || 'dev' }} + export-type: "env" + secret-name: PREVIEW_SIGNIN_KEY + + # Optional: agent.* tests register only when it is set, and a missing name throws. + - name: Load the agent test key from Infisical + continue-on-error: true + uses: Infisical/secrets-action@d2e351f16c6ca20d17c85e6c992e04bdeb64e87d # v1.0.18 + with: + method: "oidc" + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ vars.INFISICAL_PROJECT_SLUG }} + env-slug: ${{ vars.PREVIEW_INFISICAL_ENV_SLUG || 'dev' }} + export-type: "env" + secret-name: ANTHROPIC_API_KEY + + - name: Run the e2e suite + run: bunx e2e run --reporter list,junit + + - name: Upload e2e results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: e2e-results + path: apps/web/e2e/.e2e/ + if-no-files-found: warn + include-hidden-files: true + retention-days: 7 diff --git a/apps/web/e2e/README.md b/apps/web/e2e/README.md index c827a08b4..571005478 100644 --- a/apps/web/e2e/README.md +++ b/apps/web/e2e/README.md @@ -20,5 +20,6 @@ E2E_TELEMETRY_DISABLED=1 MAPLE_E2E_APP_URL=https://app-pr-.maple.dev \ lines); those tests register only when `ANTHROPIC_API_KEY` is set. - Tests create their own rules and destinations with a run-unique suffix and delete them after. -CI runs this after the preview's seed and smoke test (`deploy-pr-preview.yml`), non-blocking; -traces and screenshots land in the `e2e-results` artifact. +CI runs this as the `e2e` job of `deploy-pr-preview.yml`, after the preview's seed and smoke test. +It is not a required check, and it reruns without redeploying. Traces and screenshots land in +the `e2e-results` artifact. diff --git a/apps/web/e2e/tests/alert-rules.e2e.ts b/apps/web/e2e/tests/alert-rules.e2e.ts index 1169dd783..83646759d 100644 --- a/apps/web/e2e/tests/alert-rules.e2e.ts +++ b/apps/web/e2e/tests/alert-rules.e2e.ts @@ -23,84 +23,91 @@ interface Rule { const toRuleUuid = Schema.decodeSync(AlertRulePublicId) test.describe("alert rules", () => { - // A rule written by the API or MCP, then edited in the UI: the path that dropped tiers. - test( - "keeps an API-made tiered rule intact through UI edits", - { session: "reviewer" }, - async ({ app, browser, screen }) => { - const appUrl = app.baseUrl ?? "" - await app.open("/alerts") - const tag = runTag() - const destination = (name: string) => - api<{ id: string }>(browser, appUrl, "/v2/alerts/destinations", { + // Editing loads the rule over Electric sync, which a `preview:web` preview does not deploy. + if (process.env.MAPLE_E2E_NO_SYNC !== "1") { + // A rule written by the API or MCP, then edited in the UI: the path that dropped tiers. + test( + "keeps an API-made tiered rule intact through UI edits", + { session: "reviewer" }, + async ({ app, browser, screen }) => { + const appUrl = app.baseUrl ?? "" + await app.open("/alerts") + const tag = runTag() + const destination = (name: string) => + api<{ id: string }>(browser, appUrl, "/v2/alerts/destinations", { + method: "POST", + body: { + type: "webhook", + name: `e2e ${name} ${tag}`, + url: "https://example.com/maple-e2e", + }, + }) + const warn = await destination("warn") + const page = await destination("page") + const tiers: Tier[] = [ + { + severity: "warning", + value: 0.007, + upper: null, + recovery_value: 0.005, + destination_ids: [warn.body.id], + }, + { + severity: "critical", + value: 0.03, + upper: null, + recovery_value: null, + destination_ids: [page.body.id], + }, + ] + const created = await api(browser, appUrl, "/v2/alerts/rules", { method: "POST", body: { - type: "webhook", - name: `e2e ${name} ${tag}`, - url: "https://example.com/maple-e2e", + name: `e2e tiers ${tag}`, + signal_type: "error_rate", + comparator: "gt", + window_minutes: 5, + consecutive_breaches_required: 2, + breaches_out_of: 4, + thresholds: tiers, }, }) - const warn = await destination("warn") - const page = await destination("page") - const tiers: Tier[] = [ - { - severity: "warning", - value: 0.007, - upper: null, - recovery_value: 0.005, - destination_ids: [warn.body.id], - }, - { - severity: "critical", - value: 0.03, - upper: null, - recovery_value: null, - destination_ids: [page.body.id], - }, - ] - const created = await api(browser, appUrl, "/v2/alerts/rules", { - method: "POST", - body: { - name: `e2e tiers ${tag}`, - signal_type: "error_rate", - comparator: "gt", - window_minutes: 5, - consecutive_breaches_required: 2, - breaches_out_of: 4, - thresholds: tiers, - }, - }) - expect(created.status).toBe(200) - const editUrl = `/alerts/create?ruleId=${toRuleUuid(created.body.id)}` - const read = () => api(browser, appUrl, `/v2/alerts/rules/${created.body.id}`) + expect(created.status).toBe(200) + const editUrl = `/alerts/create?ruleId=${toRuleUuid(created.body.id)}` + const read = () => api(browser, appUrl, `/v2/alerts/rules/${created.body.id}`) - try { - await app.open(editUrl) - await expect(browser.locator("#rule-threshold")).toHaveValue("0.7") - await expect(screen.getByLabel("Critical threshold")).toHaveValue("3") - await expect(screen.getByLabel("Out of the last")).toHaveValue("4") + try { + await app.open(editUrl) + await expect(browser.locator("#rule-threshold")).toHaveValue("0.7") + await expect(screen.getByLabel("Critical threshold")).toHaveValue("3") + await expect(screen.getByLabel("Out of the last")).toHaveValue("4") - await screen.getByLabel("Rule name").fill(`e2e tiers renamed ${tag}`) - await screen.getByRole("button", "Save changes").tap() - await expect(browser).toHaveURL(/\/alerts$/) - const renamed = await read() - expect(renamed.body.name).toBe(`e2e tiers renamed ${tag}`) - expect(renamed.body.thresholds).toEqual(tiers) - expect(renamed.body.breaches_out_of).toBe(4) + await screen.getByLabel("Rule name").fill(`e2e tiers renamed ${tag}`) + await screen.getByRole("button", "Save changes").tap() + await expect(browser).toHaveURL(/\/alerts$/) + const renamed = await read() + expect(renamed.body.name).toBe(`e2e tiers renamed ${tag}`) + expect(renamed.body.thresholds).toEqual(tiers) + expect(renamed.body.breaches_out_of).toBe(4) - await app.open(editUrl) - await screen.getByLabel("Critical threshold").fill("4") - await screen.getByRole("button", "Save changes").tap() - await expect(browser).toHaveURL(/\/alerts$/) - const retuned = await read() - expect(retuned.body.thresholds.map((tier) => tier.value)).toEqual([0.007, 0.04]) - } finally { - await api(browser, appUrl, `/v2/alerts/rules/${created.body.id}`, { method: "DELETE" }) - await api(browser, appUrl, `/v2/alerts/destinations/${warn.body.id}`, { method: "DELETE" }) - await api(browser, appUrl, `/v2/alerts/destinations/${page.body.id}`, { method: "DELETE" }) - } - }, - ) + await app.open(editUrl) + await screen.getByLabel("Critical threshold").fill("4") + await screen.getByRole("button", "Save changes").tap() + await expect(browser).toHaveURL(/\/alerts$/) + const retuned = await read() + expect(retuned.body.thresholds.map((tier) => tier.value)).toEqual([0.007, 0.04]) + } finally { + await api(browser, appUrl, `/v2/alerts/rules/${created.body.id}`, { method: "DELETE" }) + await api(browser, appUrl, `/v2/alerts/destinations/${warn.body.id}`, { + method: "DELETE", + }) + await api(browser, appUrl, `/v2/alerts/destinations/${page.body.id}`, { + method: "DELETE", + }) + } + }, + ) + } test("names an empty range bound before save", { session: "reviewer" }, async ({ app, screen }) => { await app.open("/alerts/create")