diff --git a/.github/workflows/e2e-auth.yml b/.github/workflows/e2e-auth.yml index 84f27951b6f9..a0051e2b7cf8 100644 --- a/.github/workflows/e2e-auth.yml +++ b/.github/workflows/e2e-auth.yml @@ -4,6 +4,16 @@ on: pull_request: push: branches: [main] + workflow_dispatch: + inputs: + repeat_each: + description: "Run each test N times (passed to both runners as --repeat-each)" + required: false + default: "1" + retries: + description: "Override the config's retries (empty keeps the config value)" + required: false + default: "" concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -11,8 +21,18 @@ concurrency: jobs: auth-e2e: - timeout-minutes: 15 + name: Auth E2E (${{ matrix.runner }}) + timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 45 || 15 }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # The same suite on plain Playwright and on Endform, so every run + # yields a side-by-side comparison on the same commit. + runner: [playwright, endform] + permissions: + contents: read + pull-requests: read env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} @@ -21,19 +41,20 @@ jobs: - uses: ./.github/actions/detect-code-changes id: changes + if: github.event_name != 'workflow_dispatch' - uses: ./.github/actions/setup - if: steps.changes.outputs.has_changes == 'true' + if: github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true' - name: Store Playwright's Version - if: steps.changes.outputs.has_changes == 'true' + if: matrix.runner == 'playwright' && (github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true') id: playwright-version run: | PLAYWRIGHT_VERSION=$(pnpm exec playwright --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) echo "version=${PLAYWRIGHT_VERSION}" >> "$GITHUB_OUTPUT" - name: Cache Playwright Browsers - if: steps.changes.outputs.has_changes == 'true' + if: matrix.runner == 'playwright' && (github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true') id: cache-playwright-browsers uses: actions/cache@v6 with: @@ -41,15 +62,40 @@ jobs: key: ${{ steps.playwright-version.outputs.version }}-playwright-browsers-chromium - name: Install Playwright Browsers - if: steps.changes.outputs.has_changes == 'true' && steps.cache-playwright-browsers.outputs.cache-hit != 'true' + if: matrix.runner == 'playwright' && (github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true') && steps.cache-playwright-browsers.outputs.cache-hit != 'true' run: pnpm exec playwright install --with-deps chromium - name: Build packages and auth-test-studio - if: steps.changes.outputs.has_changes == 'true' + if: github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true' run: pnpm build --filter=auth-test-studio... - - name: Run auth E2E tests - if: steps.changes.outputs.has_changes == 'true' - run: pnpm --filter e2e test:auth + - name: Run auth E2E tests (Playwright) + if: matrix.runner == 'playwright' && (github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true') + working-directory: e2e + run: | + args=(--repeat-each "${REPEAT_EACH:-1}") + if [ -n "${RETRIES}" ]; then args+=(--retries "${RETRIES}"); fi + pnpm exec playwright test --config=playwright.auth.config.ts "${args[@]}" + env: + CI: "true" + REPEAT_EACH: ${{ inputs.repeat_each }} + RETRIES: ${{ inputs.retries }} + + # Runs every test on its own Endform machine. The preview server from the + # config's webServer keeps running on this job, and Endform routes the + # runners' localhost traffic back to it. The runners have no studio + # node_modules, so the DOM prop leak guard's vocabulary is written here + # first and shipped to them (see e2e/endform.config.ts). + - name: Run auth E2E tests (Endform) + if: matrix.runner == 'endform' && (github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true') + working-directory: e2e + run: | + pnpm run dom-prop-vocabulary + args=(--repeat-each "${REPEAT_EACH:-1}") + if [ -n "${RETRIES}" ]; then args+=(--retries "${RETRIES}"); fi + npx --yes endform@0.81.5 test --organization-id OQXolTtG --config=playwright.auth.config.ts "${args[@]}" env: CI: "true" + ENDFORM_API_KEY: ${{ secrets.ENDFORM_API_KEY }} + REPEAT_EACH: ${{ inputs.repeat_each }} + RETRIES: ${{ inputs.retries }} diff --git a/e2e/.gitignore b/e2e/.gitignore index df13193211b4..ddb76620dd6c 100644 --- a/e2e/.gitignore +++ b/e2e/.gitignore @@ -4,3 +4,4 @@ /results-auth /blob-report /diagnostics +/helpers/domPropLeaks/vocabulary.generated.json diff --git a/e2e/endform.config.ts b/e2e/endform.config.ts new file mode 100644 index 000000000000..4905197578f9 --- /dev/null +++ b/e2e/endform.config.ts @@ -0,0 +1,7 @@ +// Endform sends each runner the files the tests import. Files they read at runtime without +// importing them are listed here. +export default { + // Written by `pnpm --filter e2e dom-prop-vocabulary` before the run, see + // `helpers/domPropLeaks/vocabulary.ts`. + additionalFiles: ['helpers/domPropLeaks/vocabulary.generated.json'], +} diff --git a/e2e/helpers/domPropLeaks/vocabulary.ts b/e2e/helpers/domPropLeaks/vocabulary.ts index 6cd5c2f846d8..3825f2c5ee30 100644 --- a/e2e/helpers/domPropLeaks/vocabulary.ts +++ b/e2e/helpers/domPropLeaks/vocabulary.ts @@ -1,6 +1,7 @@ import {readFileSync} from 'node:fs' import {createRequire} from 'node:module' import path from 'node:path' +import {fileURLToPath} from 'node:url' import {loadReactDomInternals, STUDIO_PACKAGE_JSON} from './reactDomInternals' @@ -125,9 +126,25 @@ function toSource(regExp: RegExp): RegExpSource { let cachedVocabulary: DomPropVocabulary | undefined +/** + * The vocabulary as written ahead of the run by `scripts/writeDomPropVocabulary.ts`. Endform runs + * each test on a machine that has the e2e package but not the studio's `node_modules`, so the + * runner reads this file (shipped through `endform.config.ts`) instead of the studio's `react-dom`. + */ +export const PREBUILT_VOCABULARY_PATH = fileURLToPath( + new URL('./vocabulary.generated.json', import.meta.url), +) + export function loadDomPropVocabulary(): DomPropVocabulary { if (cachedVocabulary) return cachedVocabulary + if (process.env.ENDFORM === 'true') { + cachedVocabulary = JSON.parse( + readFileSync(PREBUILT_VOCABULARY_PATH, 'utf8'), + ) as DomPropVocabulary + return cachedVocabulary + } + const react = loadReactDomInternals() const isPropValid = loadIsPropValid() cachedVocabulary = { diff --git a/e2e/package.json b/e2e/package.json index 44987d0a8051..a69ec2faa72e 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -10,6 +10,7 @@ "cleanup": "tsx -r dotenv-flow/config scripts/cleanupDatasets", "codegen": "node -r dotenv-flow/config ./node_modules/.bin/sanity-test codegen", "dev": "pnpm --filter studio-e2e-testing dev", + "dom-prop-vocabulary": "tsx scripts/writeDomPropVocabulary.ts", "extract-diagnostics": "tsx scripts/flakeReport/extractDiagnostics.ts", "flake-report": "tsx scripts/flakeReport/index.ts", "preview": "pnpm e2e:build && pnpm --filter studio-e2e-testing preview", diff --git a/e2e/playwright.auth.config.ts b/e2e/playwright.auth.config.ts index 0b7a0ff7a1f4..d4b6018f9eca 100644 --- a/e2e/playwright.auth.config.ts +++ b/e2e/playwright.auth.config.ts @@ -31,7 +31,8 @@ export default defineConfig({ reporter: [['list']], use: { actionTimeout: 10_000, - trace: 'on-first-retry', + // Endform shows the trace of every failed attempt in its dashboard + trace: process.env.ENDFORM === 'true' ? 'retain-on-failure' : 'on-first-retry', viewport: {width: 1728, height: 1000}, video: 'retain-on-failure', baseURL: `http://localhost:${PORT}`, diff --git a/e2e/scripts/writeDomPropVocabulary.ts b/e2e/scripts/writeDomPropVocabulary.ts new file mode 100644 index 000000000000..5fe072cdfbde --- /dev/null +++ b/e2e/scripts/writeDomPropVocabulary.ts @@ -0,0 +1,9 @@ +/** + * Writes the DOM prop leak guard's vocabulary to a JSON file, for test runners that cannot read + * the studio's `react-dom` themselves (Endform). Run it after `pnpm install` and before the suite. + */ +import {writeFileSync} from 'node:fs' + +import {loadDomPropVocabulary, PREBUILT_VOCABULARY_PATH} from '../helpers/domPropLeaks/vocabulary' + +writeFileSync(PREBUILT_VOCABULARY_PATH, `${JSON.stringify(loadDomPropVocabulary(), null, 2)}\n`)