diff --git a/.changeset/expo-verify-rename-e2e-tests.md b/.changeset/expo-verify-rename-e2e-tests.md new file mode 100644 index 00000000000..a845151cc84 --- /dev/null +++ b/.changeset/expo-verify-rename-e2e-tests.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.claude/skills/verify-clerk-expo/SKILL.md b/.claude/skills/verify-clerk-expo/SKILL.md index 75a85314417..8b0cebbda83 100644 --- a/.claude/skills/verify-clerk-expo/SKILL.md +++ b/.claude/skills/verify-clerk-expo/SKILL.md @@ -1,6 +1,6 @@ --- name: verify-clerk-expo -description: Drive @clerk/expo and @clerk/expo-native-components in the expo-native test app (native AuthView, UserButton, UserProfileView, custom useSignIn and useSignUp flows, token cache, the Google and biometrics native modules) on an iOS simulator or Android emulator against a real Clerk development instance that the session creates and deletes, and capture video, screenshots, and the app log as evidence. The device runs on this Mac, or on a CI runner when the machine cannot run it. Use it to prove any change to packages/expo or the test app works before calling it done, to reproduce a UI bug, or to run the golden regression specs. +description: Drive @clerk/expo and @clerk/expo-native-components in the expo-native test app (native AuthView, UserButton, UserProfileView, custom useSignIn and useSignUp flows, token cache, the Google and biometrics native modules) on an iOS simulator or Android emulator against a real Clerk development instance that the session creates and deletes, and capture video, screenshots, and the app log as evidence. The device runs on this Mac, or on a CI runner when the machine cannot run it. Use it to prove any change to packages/expo or the test app works before calling it done, to reproduce a UI bug, or to run the e2e tests. --- # verify-clerk-expo @@ -103,14 +103,14 @@ With the remote backend, plain `doctor` starts nothing and pushes nothing, and ` Input reaches the app only through specs. A spec is a TypeScript file that uses the `host` fixture from `specs/fixtures.ts` and e2e's `screen` locators. ```console -$ integration/expo-native/bin/control-clerk-expo run native-auth-view --platform ios # one feature: every spec in specs/golden/native-auth-view/ -$ integration/expo-native/bin/control-clerk-expo run custom-flow-sign-in/complete # one golden spec, on iOS +$ integration/expo-native/bin/control-clerk-expo run native-auth-view --platform ios # one feature: every spec in specs/features/native-auth-view/ +$ integration/expo-native/bin/control-clerk-expo run custom-flow-sign-in/complete # one spec, on iOS $ integration/expo-native/bin/control-clerk-expo run specs/explored/.e2e.ts # a spec you wrote $ integration/expo-native/bin/control-clerk-expo run native-auth-view native-js-sync # several targets in one run, with one video $ integration/expo-native/bin/control-clerk-expo screen --platform ios # the UI tree that is on screen now; --png adds a screenshot ``` -`run --all` runs every golden spec. `run` also takes `--grep `, `--retries `, `--no-video`, and `--wait `. `--retries ` runs a failed test again, up to `n` more times. The default is 0, so a run of your own change shows exactly what happened. The wait covers a free lane and another `run` in this worktree that holds the device. A spec limited to one platform with `test(title, { platforms: ['ios'] }, fn)` reports as skipped on the other. +`run --all` runs every committed spec. `run` also takes `--grep `, `--retries `, `--no-video`, and `--wait `. `--retries ` runs a failed test again, up to `n` more times. The default is 0, so a run of your own change shows exactly what happened. The wait covers a free lane and another `run` in this worktree that holds the device. A spec limited to one platform with `test(title, { platforms: ['ios'] }, fn)` reports as skipped on the other. For a JS change on a local device, edit the source and `run` the spec, with no `up` in between. For a change to a native input, the same `run` rebuilds the dev client first. On a remote device, commit and push, then `run`. @@ -158,13 +158,13 @@ A spec file that needs other settings than the standard ones has a settings file ### Check your work -Golden specs under `specs/golden//` are committed and cover the feature map in `features/`, which starts at `features/README.md`. Prove your own change, on both platforms when the change is not specific to one, and leave the rest of the golden specs to PR CI (`.github/workflows/verify-e2e.yml`), which runs every one of them once the pull request is ready for review. When your change is to behavior a golden spec already covers, that spec is your proof: run it. A spec covers a behavior only when it asserts on it. A spec that passes through the screen you changed without checking what you changed is not your proof. Run another feature's specs yourself only when you changed code that feature shares and want to know before CI does. For new work: +The specs under `specs/features//` are committed and cover the feature map in `features/`, which starts at `features/README.md`. Prove your own change, on both platforms when the change is not specific to one, and leave the rest of the committed specs to PR CI (`.github/workflows/verify-e2e.yml`), which runs every one of them once the pull request is ready for review. When your change is to behavior a committed spec already covers, that spec is your proof: run it. A spec covers a behavior only when it asserts on it. A spec that passes through the screen you changed without checking what you changed is not your proof. Run another feature's specs yourself only when you changed code that feature shares and want to know before CI does. For new work: 1. Write a spec under `specs/explored/`, which is gitignored. It imports the fixture as `'../fixtures.ts'`. 2. Run it by path. 3. Run `screen` after the run, pass or fail. It prints the tree that the app is on now, with each node's role, label, and testID. 4. Fix the locators from that output until the spec passes. -5. When the change adds or changes what `@clerk/expo` does, move the spec to `specs/golden//`, change its import to `'../../fixtures.ts'`, `git add` it, and update the feature's page and its line in `features/README.md`. A change to wording or appearance alone does not earn a golden spec, because a spec that pins a label fails on the next change of wording: leave that spec in `specs/explored/`. The run directory keeps a copy of every spec the run used. +5. When the change adds or changes what `@clerk/expo` does, move the spec to `specs/features//`, change its import to `'../../fixtures.ts'`, `git add` it, and update the feature's page and its line in `features/README.md`. A change to wording or appearance alone does not earn a committed spec, because a spec that pins a label fails on the next change of wording: leave that spec in `specs/explored/`. The run directory keeps a copy of every spec the run used. A failing spec prints `FAIL`, the first assertion message, and the path of its failure page, and `run` exits 1. The failure page (`.verify/runs//e2e/failures/*.md`) lists every step, the screen tree at the failure, and a screenshot. A run in which no test executed, such as a `--grep` that matches nothing, fails with `NO_SPECS` and exits 3, because it proved nothing. A run in which every selected spec is limited to the other platform is the one exception. It exits 0, prints `skip` for each spec, and its `next` line says `nothing ran: every selected spec was left out by platform, so the run proves nothing to post`. Read that line, because the exit code alone looks like a pass. @@ -172,7 +172,7 @@ With `--retries`, a test that fails and then passes prints `flaky` and the error ### Agent steps -e2e has a built-in agent, which a test drives with `agent.act(...)` and `agent.assert(...)`. The CLI configures it only when the machine has a Vercel AI Gateway key. The model is `anthropic/claude-haiku-5.5`, and `openai/gpt-6-luna-fast` is the gateway's backup. Supply the key in `AI_GATEWAY_API_KEY_FILE`, a file that only you can read, or in `AI_GATEWAY_API_KEY`. The `agent` line of `doctor` shows the model and where the key came from. Committed golden specs do not use agent steps. +e2e has a built-in agent, which a test drives with `agent.act(...)` and `agent.assert(...)`. The CLI configures it only when the machine has a Vercel AI Gateway key. The model is `anthropic/claude-haiku-5.5`, and `openai/gpt-6-luna-fast` is the gateway's backup. Supply the key in `AI_GATEWAY_API_KEY_FILE`, a file that only you can read, or in `AI_GATEWAY_API_KEY`. The `agent` line of `doctor` shows the model and where the key came from. Committed specs do not use agent steps. ## Evidence @@ -229,7 +229,7 @@ Work in this order, with one `up` for both runs: make the branch and `git push` On a local device `attach` writes the description itself, and no run needs a push. A remote session builds only the commit it was started on and later commits of the branch, so its before run comes before the push of the change. Its evidence reaches the description about a minute after `down`. -Attach the run of your own change. Run your new or changed spec on its own and attach that run, so the screenshots or the video show only what the change is about. You do not owe a regression run: PR CI (`.github/workflows/verify-e2e.yml`) runs every golden spec on the pull request and reports them there. If you ran other golden specs anyway, cite that run's id in the PR and leave its video in `.verify/runs/`. +Attach the run of your own change. Run your new or changed spec on its own and attach that run, so the screenshots or the video show only what the change is about. You do not owe a regression run: PR CI (`.github/workflows/verify-e2e.yml`) runs every committed spec on the pull request and reports them there. If you ran other committed specs anyway, cite that run's id in the PR and leave its video in `.verify/runs/`. ## Cleanup diff --git a/.claude/skills/verify-clerk-expo/features/README.md b/.claude/skills/verify-clerk-expo/features/README.md index 29b18196b3e..87f3f4d71f8 100644 --- a/.claude/skills/verify-clerk-expo/features/README.md +++ b/.claude/skills/verify-clerk-expo/features/README.md @@ -1,6 +1,6 @@ # @clerk/expo verification map -This directory is the maintained source for verifying the user-facing behavior of `@clerk/expo` through the `expo-native` test app in `integration/templates/expo-native`. On a local device the test app runs as a Debug dev client that loads its JS, and `packages/expo` with it, from a local Metro server. On a remote device it runs as a standalone Release build of a pushed commit, with the JS embedded. Read this index before driving the app, then use the matching feature file as the recipe. Every recipe runs through `integration/expo-native/bin/control-clerk-expo` and the golden specs under `specs/golden//`. +This directory is the maintained source for verifying the user-facing behavior of `@clerk/expo` through the `expo-native` test app in `integration/templates/expo-native`. On a local device the test app runs as a Debug dev client that loads its JS, and `packages/expo` with it, from a local Metro server. On a remote device it runs as a standalone Release build of a pushed commit, with the JS embedded. Read this index before driving the app, then use the matching feature file as the recipe. Every recipe runs through `integration/expo-native/bin/control-clerk-expo` and the specs under `specs/features//`. ## Test users and sign-in @@ -39,7 +39,7 @@ The iOS identifiers come from `Sources/ClerkKitUI/Components/Auth/ClerkAccessibi ## Feature entry contract -Each feature file starts with an H1 title and one paragraph describing the user-visible behavior, then exactly four H2 sections in this order: `Sub-features`, `How to get to it (user POV)`, `Driving it with verify`, and `Gotchas`. `Driving it with verify` starts with `Preconditions:` and names the golden specs that prove each sub-feature. A feature file lists only entry points that a spec drives. +Each feature file starts with an H1 title and one paragraph describing the user-visible behavior, then exactly four H2 sections in this order: `Sub-features`, `How to get to it (user POV)`, `Driving it with verify`, and `Gotchas`. `Driving it with verify` starts with `Preconditions:` and names the specs that prove each sub-feature. A feature file lists only entry points that a spec drives. ## Features diff --git a/.claude/skills/verify-clerk-expo/features/native-modules.md b/.claude/skills/verify-clerk-expo/features/native-modules.md index 500f155efcb..67b33f9c2cb 100644 --- a/.claude/skills/verify-clerk-expo/features/native-modules.md +++ b/.claude/skills/verify-clerk-expo/features/native-modules.md @@ -18,7 +18,7 @@ An app that uses `useSignInWithGoogle` from `@clerk/expo/google` or `useBiometri Preconditions: - `up` has built the test app. The build links `@clerk/expo-google-signin` and `@clerk/expo-biometrics` from the workspace. The test app's `app.json` lists the `@clerk/expo-google-signin` config plugin, beside those of `@clerk/expo` and `@clerk/expo-native-components`, and holds three placeholder values under `extra`: a web client ID, an iOS client ID, and the iOS URL scheme that the plugin registers. They belong to no Google project. -- `specs/golden/native-modules/biometric-availability.settings.json` turns biometric sign-in on for the instance (`auth_biometric.used_for_sign_in`), and `run` applies it before the test of that spec file. The Google spec file runs on the standard settings, so a run of both has two settings groups. +- `specs/features/native-modules/biometric-availability.settings.json` turns biometric sign-in on for the instance (`auth_biometric.used_for_sign_in`), and `run` applies it before the test of that spec file. The Google spec file runs on the standard settings, so a run of both has two settings groups. - **Google sign-in.** Run `integration/expo-native/bin/control-clerk-expo run native-modules/google-sign-in --platform ios`, and again with `--platform android`. The test taps `Native modules` on the home and taps `google-sign-in-button`. On iOS it waits for the system prompt that names `accounts.google.com` and dismisses it with `device.alert('dismiss')`. It then expects `google-result` to say `Google sign-in was cancelled`. On Android it waits for a node whose id begins `com.google.android.gms:id/`, which is a page of Google Play services, and ends there. Screenshots `google-system-prompt` and `google-cancelled`, both on iOS; on Android the video shows Google's page. - **Biometric availability.** Run `run native-modules/biometric-availability` on each platform. The test opens the same screen, taps `biometric-availability-button`, and expects `biometric-availability-result` to say `biometric availability: biometric_authentication_unavailable` on iOS and `biometric availability: no_local_credential` on Android. Screenshot `biometric-availability`. diff --git a/.claude/skills/verify-clerk-expo/references/instances.md b/.claude/skills/verify-clerk-expo/references/instances.md index 2f24c8d77c0..2ddf7df133d 100644 --- a/.claude/skills/verify-clerk-expo/references/instances.md +++ b/.claude/skills/verify-clerk-expo/references/instances.md @@ -15,7 +15,7 @@ The CLI checks those 62 leaves and the leaves a settings file declares, and no o ## Declare other settings -A spec file that runs on the standard settings declares nothing. One golden spec of this repository declares settings, `specs/golden/native-modules/biometric-availability.e2e.ts`, which turns biometric sign-in on. A spec file that needs other settings has a settings file beside it, with the same name and `.settings.json` in place of `.e2e.ts`. For `specs/golden//.e2e.ts`, that file is `specs/golden//.settings.json`: +A spec file that runs on the standard settings declares nothing. One committed spec of this repository declares settings, `specs/features/native-modules/biometric-availability.e2e.ts`, which turns biometric sign-in on. A spec file that needs other settings has a settings file beside it, with the same name and `.settings.json` in place of `.e2e.ts`. For `specs/features//.e2e.ts`, that file is `specs/features//.settings.json`: ```json { diff --git a/.github/workflows/verify-e2e.yml b/.github/workflows/verify-e2e.yml index f6252686105..79abdf9d707 100644 --- a/.github/workflows/verify-e2e.yml +++ b/.github/workflows/verify-e2e.yml @@ -233,7 +233,7 @@ jobs: run: | "$CLI" screen --platform ios > /dev/null || echo "::warning::the first launch failed here; the first spec pays for the cold start" - - name: Run the golden specs + - name: Run the end-to-end tests timeout-minutes: 30 env: CLERK_PLATFORM_API_KEY: ${{ secrets.MOBILE_VERIFICATION_PLATFORM_API_KEY }} diff --git a/integration/expo-native/README.md b/integration/expo-native/README.md index d26d37882ce..518cfdf241b 100644 --- a/integration/expo-native/README.md +++ b/integration/expo-native/README.md @@ -20,7 +20,7 @@ export CLERK_E2E_DEVICE= export CLERK_E2E_APP_PATH= export CLERK_PUBLISHABLE_KEY=pk_test_... export CLERK_SECRET_KEY=sk_test_... -npx e2e run specs/golden/custom-flow-sign-in/complete.e2e.ts +npx e2e run specs/features/custom-flow-sign-in/complete.e2e.ts ``` `node src/fixture.ts ios` runs `pnpm install`, builds the packages the test app uses, and builds the test app in `integration/templates/expo-native` as a standalone app. A standalone app carries its JavaScript, so the run needs no Metro server. `node src/fixture.ts android` builds the APK for an emulator. @@ -50,7 +50,7 @@ Every run creates users with addresses like `1412107c1ek3x9+clerk_test@example.c This test file has a `.settings.json` file beside it, and passes only on an instance with those settings: -- `specs/golden/native-modules/biometric-availability.e2e.ts` +- `specs/features/native-modules/biometric-availability.e2e.ts` The CLI's `run` puts the instance on the declared settings before the file starts, and e2e alone does not. Run them through the CLI, for example `integration/expo-native/bin/control-clerk-expo run native-modules` from the repo root. @@ -65,7 +65,7 @@ The CLI's `run` puts the instance on the declared settings before the file start ## Layout -- `e2e.config.ts` and `specs/` are the e2e project. `specs/golden/` has the tests, `specs/fixtures.ts` has the `host` fixture they use, `specs/app.ts` names the test app, and `specs/support/` is the tests' support code. `specs/support/inputs.ts` is the only file that reads the settings above. +- `e2e.config.ts` and `specs/` are the e2e project. `specs/features/` has the tests, `specs/fixtures.ts` has the `host` fixture they use, `specs/app.ts` names the test app, and `specs/support/` is the tests' support code. `specs/support/inputs.ts` is the only file that reads the settings above. - `bin/` and `src/` are the CLI. Under the CLI the test process gets the same settings, with one difference: in place of the secret key it gets the address and token of a stand-in that the CLI runs on this machine. The stand-in forwards the three Backend API calls the tests make, for this run's test users only, so the test process never holds the key. - `test/` and `testing/` are the unit tests of both. `npm test` runs them, with no device, key, or network. - `src/core/`, `specs/support/`, `specs/fixtures.ts`, and `e2e.config.ts` are the same files, byte for byte, in clerk-ios, clerk-android, and clerk/javascript. `src/core/MANIFEST` lists them, and the CLI's `doctor` fails when they differ from it. diff --git a/integration/expo-native/specs/golden/custom-flow-sign-in/complete.e2e.ts b/integration/expo-native/specs/features/custom-flow-sign-in/complete.e2e.ts similarity index 100% rename from integration/expo-native/specs/golden/custom-flow-sign-in/complete.e2e.ts rename to integration/expo-native/specs/features/custom-flow-sign-in/complete.e2e.ts diff --git a/integration/expo-native/specs/golden/custom-flow-sign-up/complete.e2e.ts b/integration/expo-native/specs/features/custom-flow-sign-up/complete.e2e.ts similarity index 100% rename from integration/expo-native/specs/golden/custom-flow-sign-up/complete.e2e.ts rename to integration/expo-native/specs/features/custom-flow-sign-up/complete.e2e.ts diff --git a/integration/expo-native/specs/golden/native-auth-view/complete.e2e.ts b/integration/expo-native/specs/features/native-auth-view/complete.e2e.ts similarity index 100% rename from integration/expo-native/specs/golden/native-auth-view/complete.e2e.ts rename to integration/expo-native/specs/features/native-auth-view/complete.e2e.ts diff --git a/integration/expo-native/specs/golden/native-auth-view/logo.e2e.ts b/integration/expo-native/specs/features/native-auth-view/logo.e2e.ts similarity index 100% rename from integration/expo-native/specs/golden/native-auth-view/logo.e2e.ts rename to integration/expo-native/specs/features/native-auth-view/logo.e2e.ts diff --git a/integration/expo-native/specs/golden/native-auth-view/opens.e2e.ts b/integration/expo-native/specs/features/native-auth-view/opens.e2e.ts similarity index 100% rename from integration/expo-native/specs/golden/native-auth-view/opens.e2e.ts rename to integration/expo-native/specs/features/native-auth-view/opens.e2e.ts diff --git a/integration/expo-native/specs/golden/native-js-sync/sign-in-from-native.e2e.ts b/integration/expo-native/specs/features/native-js-sync/sign-in-from-native.e2e.ts similarity index 100% rename from integration/expo-native/specs/golden/native-js-sync/sign-in-from-native.e2e.ts rename to integration/expo-native/specs/features/native-js-sync/sign-in-from-native.e2e.ts diff --git a/integration/expo-native/specs/golden/native-js-sync/sign-out-from-native.e2e.ts b/integration/expo-native/specs/features/native-js-sync/sign-out-from-native.e2e.ts similarity index 100% rename from integration/expo-native/specs/golden/native-js-sync/sign-out-from-native.e2e.ts rename to integration/expo-native/specs/features/native-js-sync/sign-out-from-native.e2e.ts diff --git a/integration/expo-native/specs/golden/native-modules/biometric-availability.e2e.ts b/integration/expo-native/specs/features/native-modules/biometric-availability.e2e.ts similarity index 100% rename from integration/expo-native/specs/golden/native-modules/biometric-availability.e2e.ts rename to integration/expo-native/specs/features/native-modules/biometric-availability.e2e.ts diff --git a/integration/expo-native/specs/golden/native-modules/biometric-availability.settings.json b/integration/expo-native/specs/features/native-modules/biometric-availability.settings.json similarity index 100% rename from integration/expo-native/specs/golden/native-modules/biometric-availability.settings.json rename to integration/expo-native/specs/features/native-modules/biometric-availability.settings.json diff --git a/integration/expo-native/specs/golden/native-modules/google-sign-in.e2e.ts b/integration/expo-native/specs/features/native-modules/google-sign-in.e2e.ts similarity index 100% rename from integration/expo-native/specs/golden/native-modules/google-sign-in.e2e.ts rename to integration/expo-native/specs/features/native-modules/google-sign-in.e2e.ts diff --git a/integration/expo-native/specs/golden/token-cache-persistence/relaunch.e2e.ts b/integration/expo-native/specs/features/token-cache-persistence/relaunch.e2e.ts similarity index 100% rename from integration/expo-native/specs/golden/token-cache-persistence/relaunch.e2e.ts rename to integration/expo-native/specs/features/token-cache-persistence/relaunch.e2e.ts diff --git a/integration/expo-native/specs/golden/user-button-and-profile/embedded-profile.e2e.ts b/integration/expo-native/specs/features/user-button-and-profile/embedded-profile.e2e.ts similarity index 100% rename from integration/expo-native/specs/golden/user-button-and-profile/embedded-profile.e2e.ts rename to integration/expo-native/specs/features/user-button-and-profile/embedded-profile.e2e.ts diff --git a/integration/expo-native/specs/golden/user-button-and-profile/profile.e2e.ts b/integration/expo-native/specs/features/user-button-and-profile/profile.e2e.ts similarity index 100% rename from integration/expo-native/specs/golden/user-button-and-profile/profile.e2e.ts rename to integration/expo-native/specs/features/user-button-and-profile/profile.e2e.ts diff --git a/integration/expo-native/src/core/MANIFEST b/integration/expo-native/src/core/MANIFEST index 7b42cc7d377..b78165fc7a5 100644 --- a/integration/expo-native/src/core/MANIFEST +++ b/integration/expo-native/src/core/MANIFEST @@ -17,10 +17,10 @@ df955a42a2bd4ca1780f56837ebb9c832c68e97f74a35764d42a32e44b28286b specs/support/ d56c29efc8061adfd65b89f0f02465af3234ecd52289ef6d9f338483afeadeb5 src/core/broker.ts 7c741449c755524bba80f1ce314554293003a34f7f7c30696015ba1c0c3dc1ab src/core/claims.ts 1fc59d13e7197e6c7099c0467744b978d498d1fe409c88254d44e2f99913a7b5 src/core/clerk.ts -4b8f5d87af69b85cd0bd5e729de238ccdb39f42e54d2e38af9ca43c2f172a433 src/core/cli.ts +6eef5f3989f8e29771fb47ec1b3721699249ea272e5009ccf8e66ece9981c4e6 src/core/cli.ts 4aeacea985f5ef80001e5695feec53182abb9695eb07130898568743755fc8af src/core/devices.ts 3ae461d6aa75a2cefac0392a9ad124aec4105ac25b9320371743e2ea90fe2202 src/core/driver.ts -b59071aaefeec02dea89bd25615f8ada0da0a51e805a2523011b8b591e7435d3 src/core/e2e.ts +efc4cdabcd1b00e903bfcfeb077016f1654c07ef4926b6389d248227c12c0af5 src/core/e2e.ts c9a29e27eb835d9c3fc9b0bd394ff55b5f1fcc2b2fc0d96da51197e5e7bca003 src/core/evidence.ts 27bfd4ce59937aa8c97d9fd6849cf52f3da05626fbc9388385f4b5e684577d36 src/core/exec.ts e2075f17f52ded8dea64d8a2473820396ca5ac1a22db59f21a092da28e55911f src/core/github-report.ts @@ -47,6 +47,6 @@ d614ee5c98a85d15037b744a67404496273f62985fd7d6a049cacfd5ab335503 src/core/remot e734336ea3b345057b69193790e2efce9a662c67ec5b4a426c811f999b45ccdb src/core/remote/tunnel.ts 685eabf010c7f63d076cfd3da1a681b5cc8024720538c1bf3a99792b563d8b60 src/core/slot.ts a8838aaf1fa9e4a6bc4e350ea674079201da76c5962656039b96db2505591aed src/core/state.ts -c9900be2f1097ed0395018531bed554064440f57962afac1daa794040b1b2c28 src/core/types.ts +25501a3117a814fb9e0cd4a00a4ecaf33a28e589b0a0e22679ab660736846b15 src/core/types.ts 79b822b663c1ac1748cf3c8ec390ef9204378f4c97531f4da9b0bf5736a20d36 src/core/verbs.ts 9be1816773216373548ec9933978b53ace3d3bfdba17b72c1e5ed6d8aef80f62 src/core/workspace.ts diff --git a/integration/expo-native/src/core/cli.ts b/integration/expo-native/src/core/cli.ts index 61c0dba0007..3a03155f903 100644 --- a/integration/expo-native/src/core/cli.ts +++ b/integration/expo-native/src/core/cli.ts @@ -202,10 +202,10 @@ function rel(packageDir: string, path: string): string { function renderRun(result: RunResult, packageDir: string): string[] { const r = result.record; const lines: string[] = []; - const width = Math.max(0, ...r.results.map((x) => x.spec.path.replace(/^specs\/(golden\/)?/, '').length)); + const width = Math.max(0, ...r.results.map((x) => x.spec.path.replace(/^specs\/(features\/)?/, '').length)); for (const x of r.results) { const label = { passed: 'pass', failed: 'FAIL', skipped: 'skip', flaky: 'flaky', interrupted: 'INTR' }[x.status]; - const name = x.spec.path.replace(/^specs\/(golden\/)?/, ''); + const name = x.spec.path.replace(/^specs\/(features\/)?/, ''); const tail = x.status === 'skipped' ? (x.skipReason ?? '') : `${x.seconds}s`; lines.push(` ${pad(label, 5)} ${pad(name, width)} ${x.title} ${tail}`); if (x.error !== null) lines.push(` ${x.status === 'flaky' ? `passed on attempt ${x.attempts}; the attempt before it failed: ` : ''}${x.error}`); diff --git a/integration/expo-native/src/core/e2e.ts b/integration/expo-native/src/core/e2e.ts index e2364e3f8d5..c473c6b9aa7 100644 --- a/integration/expo-native/src/core/e2e.ts +++ b/integration/expo-native/src/core/e2e.ts @@ -33,20 +33,20 @@ const toPosix = (path: string) => path.split(sep).join('/'); function specRef(packageDir: string, absolute: string): SpecRef | null { const path = toPosix(relative(packageDir, absolute)); - const golden = /^specs\/golden\/([^/]+)\/.+\.e2e\.ts$/.exec(path); - if (golden !== null) return { kind: 'golden', path, feature: golden[1] as FeatureName }; + const committed = /^specs\/features\/([^/]+)\/.+\.e2e\.ts$/.exec(path); + if (committed !== null) return { kind: 'committed', path, feature: committed[1] as FeatureName }; if (/^specs\/explored\/.+\.e2e\.ts$/.test(path)) return { kind: 'explored', path, feature: null }; return null; } export function resolveSpecs(packageDir: string, selection: SpecSelection, cwd: string = process.cwd()): readonly SpecRef[] { - const goldenDir = join(packageDir, 'specs', 'golden'); - const features = existsSync(goldenDir) ? readdirSync(goldenDir).filter((name) => statSync(join(goldenDir, name)).isDirectory()).sort() : []; + const featuresDir = join(packageDir, 'specs', 'features'); + const features = existsSync(featuresDir) ? readdirSync(featuresDir).filter((name) => statSync(join(featuresDir, name)).isDirectory()).sort() : []; const toRefs = (paths: readonly string[]) => paths.flatMap((p) => specRef(packageDir, p) ?? []).sort((a, b) => a.path.localeCompare(b.path)); if ('all' in selection) { - const refs = toRefs(walk(goldenDir)); - if (refs.length === 0) throw new VerifyFailure('NO_SPECS', 'there are no golden specs under specs/golden/', 'write one under specs/golden//, or run an explored spec by path'); + const refs = toRefs(walk(featuresDir)); + if (refs.length === 0) throw new VerifyFailure('NO_SPECS', 'there are no specs under specs/features/', 'write one under specs/features//, or run an explored spec by path'); return refs; } @@ -58,15 +58,15 @@ export function resolveSpecs(packageDir: string, selection: SpecSelection, cwd: const file = candidates.find((c) => existsSync(c)); if (file !== undefined) { const ref = specRef(packageDir, file); - if (ref === null) throw new VerifyFailure('NO_SPECS', `${selector} is not under specs/golden// or specs/explored/`, 'move the spec under specs/explored/ and run it by that path'); + if (ref === null) throw new VerifyFailure('NO_SPECS', `${selector} is not under specs/features// or specs/explored/`, 'move the spec under specs/explored/ and run it by that path'); refs = [ref]; } } else if (selector.includes('/')) { const [feature, spec] = selector.split('/', 2); - const file = join(goldenDir, feature ?? '', `${(spec ?? '').replace(/\.e2e\.ts$/, '')}${SPEC_SUFFIX}`); + const file = join(featuresDir, feature ?? '', `${(spec ?? '').replace(/\.e2e\.ts$/, '')}${SPEC_SUFFIX}`); if (existsSync(file)) refs = toRefs([file]); } else if (features.includes(selector)) { - refs = toRefs(walk(join(goldenDir, selector))); + refs = toRefs(walk(join(featuresDir, selector))); } if (refs.length === 0) { throw new VerifyFailure( diff --git a/integration/expo-native/src/core/types.ts b/integration/expo-native/src/core/types.ts index f9beb03efe5..20a97f7c862 100644 --- a/integration/expo-native/src/core/types.ts +++ b/integration/expo-native/src/core/types.ts @@ -242,7 +242,7 @@ export interface DeletionTarget { export type VerbResult = DoctorReport | UpResult | RunResult | ScreenResult | AttachResult | DownResult; -export type SpecKind = 'golden' | 'explored'; +export type SpecKind = 'committed' | 'explored'; export interface SpecRef { readonly kind: SpecKind; readonly path: string; diff --git a/integration/expo-native/test/cli.test.ts b/integration/expo-native/test/cli.test.ts index 0ef5bc06d8e..a3606f4723d 100644 --- a/integration/expo-native/test/cli.test.ts +++ b/integration/expo-native/test/cli.test.ts @@ -148,8 +148,8 @@ describe('run output', () => { next: 'next', record: { results: [ - { spec: { kind: 'golden', path: 'specs/golden/auth-start/opens.e2e.ts', feature: null }, title: 'opens', platform: 'ios', status: 'passed', seconds: 5, attempts: 1, error: null, skipReason: null, skippedBy: null, tags: [], failurePage: null, failureScreen: null, failureScreenshot: null }, - { spec: { kind: 'golden', path: 'specs/golden/sign-up/complete.e2e.ts', feature: null }, title: 'completes', platform: 'ios', status, seconds: 41.5, attempts: 2, error, skipReason: null, skippedBy: null, tags: [], failurePage: '/tmp/failures/complete.md' as EvidencePath, failureScreen: null, failureScreenshot: null }, + { spec: { kind: 'committed', path: 'specs/features/auth-start/opens.e2e.ts', feature: null }, title: 'opens', platform: 'ios', status: 'passed', seconds: 5, attempts: 1, error: null, skipReason: null, skippedBy: null, tags: [], failurePage: null, failureScreen: null, failureScreenshot: null }, + { spec: { kind: 'committed', path: 'specs/features/sign-up/complete.e2e.ts', feature: null }, title: 'completes', platform: 'ios', status, seconds: 41.5, attempts: 2, error, skipReason: null, skippedBy: null, tags: [], failurePage: '/tmp/failures/complete.md' as EvidencePath, failureScreen: null, failureScreenshot: null }, ], videos: [], screenshots: [], diff --git a/integration/expo-native/test/e2e.test.ts b/integration/expo-native/test/e2e.test.ts index 33c0db3cf85..34e68e03fb5 100644 --- a/integration/expo-native/test/e2e.test.ts +++ b/integration/expo-native/test/e2e.test.ts @@ -11,7 +11,7 @@ import { SAMPLE_INPUTS } from '../testing/sample-inputs.ts'; function packageDir(): string { const dir = mkdtempSync(join(tmpdir(), 'verify-specs-')); - for (const file of ['specs/golden/auth-start/opens.e2e.ts', 'specs/golden/sign-up/request-code.e2e.ts', 'specs/explored/probe.e2e.ts', 'specs/fixtures.ts']) { + for (const file of ['specs/features/auth-start/opens.e2e.ts', 'specs/features/sign-up/request-code.e2e.ts', 'specs/explored/probe.e2e.ts', 'specs/fixtures.ts']) { mkdirSync(join(dir, file, '..'), { recursive: true }); writeFileSync(join(dir, file), ''); } @@ -21,10 +21,10 @@ function packageDir(): string { describe('resolveSpecs', () => { it('expands features, feature/spec, paths, and --all', () => { const dir = packageDir(); - assert.deepEqual(resolveSpecs(dir, { selectors: ['auth-start'] }), [{ kind: 'golden', path: 'specs/golden/auth-start/opens.e2e.ts', feature: 'auth-start' }]); - assert.deepEqual(resolveSpecs(dir, { selectors: ['sign-up/request-code'] }).map((s) => s.path), ['specs/golden/sign-up/request-code.e2e.ts']); + assert.deepEqual(resolveSpecs(dir, { selectors: ['auth-start'] }), [{ kind: 'committed', path: 'specs/features/auth-start/opens.e2e.ts', feature: 'auth-start' }]); + assert.deepEqual(resolveSpecs(dir, { selectors: ['sign-up/request-code'] }).map((s) => s.path), ['specs/features/sign-up/request-code.e2e.ts']); assert.deepEqual(resolveSpecs(dir, { selectors: ['specs/explored/probe.e2e.ts'] }, dir), [{ kind: 'explored', path: 'specs/explored/probe.e2e.ts', feature: null }]); - assert.deepEqual(resolveSpecs(dir, { all: true }).map((s) => s.path), ['specs/golden/auth-start/opens.e2e.ts', 'specs/golden/sign-up/request-code.e2e.ts']); + assert.deepEqual(resolveSpecs(dir, { all: true }).map((s) => s.path), ['specs/features/auth-start/opens.e2e.ts', 'specs/features/sign-up/request-code.e2e.ts']); }); it('lists every feature for an unknown selector', () => { @@ -34,9 +34,9 @@ describe('resolveSpecs', () => { describe('planE2E', () => { it('passes selection flags through and points output inside the run', () => { - const plan = planE2E(SAMPLE_INPUTS, [{ kind: 'golden', path: 'specs/golden/a/b.e2e.ts', feature: null }], { verb: 'run', selection: { all: true }, grep: 'x', video: true, retries: 1, githubReport: false, waitSeconds: 0 }, '/package', e2eOutputDir('/package/.verify/runs/r20261002-141210-7c1e' as EvidencePath, 0)); + const plan = planE2E(SAMPLE_INPUTS, [{ kind: 'committed', path: 'specs/features/a/b.e2e.ts', feature: null }], { verb: 'run', selection: { all: true }, grep: 'x', video: true, retries: 1, githubReport: false, waitSeconds: 0 }, '/package', e2eOutputDir('/package/.verify/runs/r20261002-141210-7c1e' as EvidencePath, 0)); assert.deepEqual(plan.args, [ - 'run', 'specs/golden/a/b.e2e.ts', '--config', 'e2e.config.ts', '--target', 'ios', + 'run', 'specs/features/a/b.e2e.ts', '--config', 'e2e.config.ts', '--target', 'ios', '--output', '.verify/runs/r20261002-141210-7c1e/e2e', '--reporter', 'list,markdown,junit', '--retries', '1', '--grep', 'x', '--pass-with-no-tests', ]); @@ -64,19 +64,19 @@ describe('parseE2EReport', () => { run: { results: [ { - id: 'aaaaaaaa11', kind: 'test', titlePath: ['opens'], file: 'specs/golden/auth-start/opens.e2e.ts', platform: 'ios', tags: [], status: 'passed', + id: 'aaaaaaaa11', kind: 'test', titlePath: ['opens'], file: 'specs/features/auth-start/opens.e2e.ts', platform: 'ios', tags: [], status: 'passed', attempts: [{ status: 'passed', durationMs: 9100, artifacts: [{ kind: 'screenshot', path: 'ios/x/attempt-0/screenshots/001-auth.png', producer: { kind: 'step', stepId: 's1' } }], steps: [{ id: 's1', api: 'app.screenshot', label: 'auth' }] }], }, { - id: 'bbbbbbbb22', kind: 'test', titlePath: ['completes'], file: 'specs/golden/sign-up/complete.e2e.ts', platform: 'ios', tags: [], status: 'skipped', + id: 'bbbbbbbb22', kind: 'test', titlePath: ['completes'], file: 'specs/features/sign-up/complete.e2e.ts', platform: 'ios', tags: [], status: 'skipped', skip: { cause: 'filtered', reason: 'title does not match --grep' }, attempts: [], }, { - id: 'ffffffff66', kind: 'test', titlePath: ['ios-only screen'], file: 'specs/golden/sign-up/request-code.e2e.ts', platform: 'android', tags: [], status: 'skipped', + id: 'ffffffff66', kind: 'test', titlePath: ['ios-only screen'], file: 'specs/features/sign-up/request-code.e2e.ts', platform: 'android', tags: [], status: 'skipped', skip: { cause: 'platform-unavailable', reason: 'test declares platforms [ios]' }, attempts: [], }, { - id: 'dddddddd44', kind: 'test', titlePath: ['never ran'], file: 'specs/golden/auth-start/opens.e2e.ts', platform: 'ios', tags: [], status: 'skipped', + id: 'dddddddd44', kind: 'test', titlePath: ['never ran'], file: 'specs/features/auth-start/opens.e2e.ts', platform: 'ios', tags: [], status: 'skipped', skip: { cause: 'infrastructure-unavailable', reason: 'the device could not be opened' }, attempts: [], }, { @@ -90,7 +90,7 @@ describe('parseE2EReport', () => { it('keeps the failed attempt of a test that passed on a retry: its error, its failure page, and its screenshot', () => { const dir = e2eOutputDir(mkdtempSync(join(tmpdir(), 'verify-report-')) as EvidencePath, 0); mkdirSync(join(dir, 'failures'), { recursive: true }); - writeFileSync(join(dir, 'failures', 'specs_golden_auth-start_opens-opens-eeeeeeee.md'), ''); + writeFileSync(join(dir, 'failures', 'specs_features_auth-start_opens-opens-eeeeeeee.md'), ''); mkdirSync(join(dir, 'artifacts', 'ios/o/attempt-0'), { recursive: true }); writeFileSync(join(dir, 'artifacts', 'ios/o/attempt-0/screen.txt'), ''); const attempt = (path: string) => ({ failure: { screen: 'screen', screenshot: 'shot' }, artifacts: [{ id: 'screen', kind: 'other', path: `${path}/screen.txt` }, { id: 'shot', kind: 'screenshot', path: `${path}/screenshots/001-failure.png` }] }); @@ -99,20 +99,20 @@ describe('parseE2EReport', () => { run: { results: [ { - id: 'eeeeeeee55', kind: 'test', titlePath: ['opens'], file: 'specs/golden/auth-start/opens.e2e.ts', platform: 'ios', tags: [], status, + id: 'eeeeeeee55', kind: 'test', titlePath: ['opens'], file: 'specs/features/auth-start/opens.e2e.ts', platform: 'ios', tags: [], status, attempts: [{ status: 'failed', durationMs: 4000, error: { message: 'tap failed\nthe runner session ended' }, ...attempt('ios/o/attempt-0') }, { durationMs: 6000, ...second }], }, ], }, }); - const opens: readonly SpecRef[] = [{ kind: 'golden', path: 'specs/golden/auth-start/opens.e2e.ts', feature: null }]; + const opens: readonly SpecRef[] = [{ kind: 'committed', path: 'specs/features/auth-start/opens.e2e.ts', feature: null }]; const [flaky] = parseE2EReport(retried('flaky', { status: 'passed' }), opens, dir); assert.equal(flaky!.status, 'flaky'); assert.equal(flaky!.attempts, 2); assert.equal(flaky!.seconds, 10); assert.equal(flaky!.error, 'tap failed; the runner session ended'); - assert.equal(flaky!.failurePage, join(dir, 'failures', 'specs_golden_auth-start_opens-opens-eeeeeeee.md')); + assert.equal(flaky!.failurePage, join(dir, 'failures', 'specs_features_auth-start_opens-opens-eeeeeeee.md')); assert.equal(flaky!.failureScreen, join(dir, 'artifacts', 'ios/o/attempt-0/screen.txt')); assert.equal(flaky!.failureScreenshot, join(dir, 'artifacts', 'ios/o/attempt-0/screenshots/001-failure.png')); @@ -131,7 +131,7 @@ describe('parseE2EReport', () => { writeFileSync(join(dir, 'failures', 'specs_explored_probe-fails-cccccccc.md'), ''); mkdirSync(join(dir, 'artifacts', 'ios/p/attempt-0'), { recursive: true }); writeFileSync(join(dir, 'artifacts', 'ios/p/attempt-0/screen.txt'), ''); - const results = parseE2EReport(report, ['specs/golden/auth-start/opens.e2e.ts', 'specs/golden/sign-up/complete.e2e.ts', 'specs/golden/sign-up/request-code.e2e.ts', 'specs/explored/probe.e2e.ts'].map((path) => ({ kind: 'golden', path, feature: null })), dir); + const results = parseE2EReport(report, ['specs/features/auth-start/opens.e2e.ts', 'specs/features/sign-up/complete.e2e.ts', 'specs/features/sign-up/request-code.e2e.ts', 'specs/explored/probe.e2e.ts'].map((path) => ({ kind: 'committed', path, feature: null })), dir); assert.deepEqual(results.map((r) => r.status), ['passed', 'skipped', 'skipped', 'failed', 'failed']); assert.equal(results[3]!.error, 'not run: infrastructure-unavailable the device could not be opened'); assert.equal(results[0]!.seconds, 9.1); @@ -143,8 +143,8 @@ describe('parseE2EReport', () => { assert.equal(results[4]!.error, 'expect.toBeVisible failed; observed: no node'); assert.equal(results[4]!.failureScreen, join(dir, 'artifacts', 'ios/p/attempt-0/screen.txt')); assert.equal(results[4]!.failureScreenshot, join(dir, 'artifacts', 'ios/p/attempt-0/screenshots/001-failure.png'), 'the full path, never truncated'); - const selected = parseE2EReport(report, [{ kind: 'golden', path: 'specs/golden/auth-start/opens.e2e.ts', feature: null }], dir); - assert.deepEqual(selected.map((r) => r.spec.path), ['specs/golden/auth-start/opens.e2e.ts', 'specs/golden/auth-start/opens.e2e.ts'], 'files e2e lists but the run did not select are dropped'); + const selected = parseE2EReport(report, [{ kind: 'committed', path: 'specs/features/auth-start/opens.e2e.ts', feature: null }], dir); + assert.deepEqual(selected.map((r) => r.spec.path), ['specs/features/auth-start/opens.e2e.ts', 'specs/features/auth-start/opens.e2e.ts'], 'files e2e lists but the run did not select are dropped'); assert.equal(results[4]!.failurePage, join(dir, 'failures', 'specs_explored_probe-fails-cccccccc.md')); }); diff --git a/integration/expo-native/test/github-report.test.ts b/integration/expo-native/test/github-report.test.ts index 1f07bbf7eb6..70d9ae4ad7f 100644 --- a/integration/expo-native/test/github-report.test.ts +++ b/integration/expo-native/test/github-report.test.ts @@ -18,9 +18,9 @@ interface Row { } const ALL: readonly Row[] = [ - { file: 'specs/golden/auth-start/opens.e2e.ts', title: 'opens', status: 'passed' }, - { file: 'specs/golden/sign-up/complete.e2e.ts', title: 'completes', status: 'flaky' }, - { file: 'specs/golden/session-tasks/setup-mfa.e2e.ts', title: 'stops on the task', status: 'passed' }, + { file: 'specs/features/auth-start/opens.e2e.ts', title: 'opens', status: 'passed' }, + { file: 'specs/features/sign-up/complete.e2e.ts', title: 'completes', status: 'flaky' }, + { file: 'specs/features/session-tasks/setup-mfa.e2e.ts', title: 'stops on the task', status: 'passed' }, ]; function attempt(index: number, status: 'passed' | 'failed') { diff --git a/integration/expo-native/test/host.test.ts b/integration/expo-native/test/host.test.ts index a4086d3c6b1..db0c8721e16 100644 --- a/integration/expo-native/test/host.test.ts +++ b/integration/expo-native/test/host.test.ts @@ -158,19 +158,19 @@ describe('the clerk-expo host', () => { }); }); -describe('the golden specs of this repository', () => { +describe('the committed specs of this repository', () => { const dir = join(import.meta.dirname, '..'); - const golden = resolveSpecs(dir, { all: true }).map(spec => ({ spec, ...readSpecText(dir, spec.path) })); + const committed = resolveSpecs(dir, { all: true }).map(spec => ({ spec, ...readSpecText(dir, spec.path) })); it('declare settings in one spec file, so a run of all of them is the standard group and then the biometric sign-in group', () => { - const declaring = golden + const declaring = committed .filter(({ spec, ...text }) => declaredIn(text, spec.path) !== null) .map(({ spec }) => spec.path); - assert.deepEqual(declaring, ['specs/golden/native-modules/biometric-availability.e2e.ts']); - const plan = planGroups(golden, STANDARD.key); + assert.deepEqual(declaring, ['specs/features/native-modules/biometric-availability.e2e.ts']); + const plan = planGroups(committed, STANDARD.key); assert.equal(plan.length, 2); assert.equal(plan[0]!.settings, STANDARD); - assert.equal(plan[0]!.specs.length, golden.length - 1); + assert.equal(plan[0]!.specs.length, committed.length - 1); assert.deepEqual( plan[1]!.specs.map(spec => spec.path), declaring, diff --git a/integration/expo-native/test/instances.test.ts b/integration/expo-native/test/instances.test.ts index f28d6521753..36011e2bb4d 100644 --- a/integration/expo-native/test/instances.test.ts +++ b/integration/expo-native/test/instances.test.ts @@ -32,13 +32,13 @@ const FORCED_ORG: InstanceSettings = { config: { organization_settings: { force_ const MFA_LABEL = 'auth_multi_factor.required_for_sign_up=true'; const ORG_LABEL = 'organization_settings.force_organization_selection=true'; const MFA_BODY = { ...standardFile().config, auth_multi_factor: { ...(standardFile().config.auth_multi_factor as object), required_for_sign_up: true } }; -const MFA_SPEC = 'specs/golden/session-tasks/setup-mfa.e2e.ts'; -const MFA_FILE = 'specs/golden/session-tasks/setup-mfa.settings.json'; -const ORG_SPEC = 'specs/golden/session-tasks/choose-organization.e2e.ts'; -const STANDARD_SPEC = 'specs/golden/auth-start/auth-start.e2e.ts'; +const MFA_SPEC = 'specs/features/session-tasks/setup-mfa.e2e.ts'; +const MFA_FILE = 'specs/features/session-tasks/setup-mfa.settings.json'; +const ORG_SPEC = 'specs/features/session-tasks/choose-organization.e2e.ts'; +const STANDARD_SPEC = 'specs/features/auth-start/auth-start.e2e.ts'; const groupOf = (declared: InstanceSettings | null, path: string): SettingsGroup => - planGroups([{ spec: { kind: 'golden', path, feature: null }, source: "test('x', () => {});\n", declaration: declared === null ? null : JSON.stringify(declared) }], null)[0]!; + planGroups([{ spec: { kind: 'committed', path, feature: null }, source: "test('x', () => {});\n", declaration: declared === null ? null : JSON.stringify(declared) }], null)[0]!; const keyRefused = (error: VerifyFailure): boolean => error.code === 'NOT_READY' && error.message === "api.clerk.com answered 401 to the instance's own secret key; the likely cause is a cloud environment whose API credential for api.clerk.com has no path prefix, so it replaces the key on every request to that host" && diff --git a/integration/expo-native/test/run-groups.test.ts b/integration/expo-native/test/run-groups.test.ts index f0edd6fbdc7..d23e7ebfbf8 100644 --- a/integration/expo-native/test/run-groups.test.ts +++ b/integration/expo-native/test/run-groups.test.ts @@ -22,12 +22,12 @@ const MFA: InstanceSettings = { config: { auth_multi_factor: { required_for_sign const FORCED_ORG: InstanceSettings = { config: { organization_settings: { force_organization_selection: true } }, environment: { 'organization_settings.force_organization_selection': true } }; const MFA_LABEL = 'auth_multi_factor.required_for_sign_up=true'; const ORG_LABEL = 'organization_settings.force_organization_selection=true'; -const AUTH_START = 'specs/golden/auth-start/auth-start.e2e.ts'; -const CHOOSE_ORG = 'specs/golden/session-tasks/choose-organization.e2e.ts'; -const COMPLETE_MFA = 'specs/golden/session-tasks/complete-setup-mfa.e2e.ts'; -const SETUP_MFA = 'specs/golden/session-tasks/setup-mfa.e2e.ts'; -const SIGN_UP = 'specs/golden/sign-up/complete.e2e.ts'; -const GOLDEN: Readonly> = { [AUTH_START]: null, [CHOOSE_ORG]: FORCED_ORG, [COMPLETE_MFA]: MFA, [SETUP_MFA]: MFA, [SIGN_UP]: null }; +const AUTH_START = 'specs/features/auth-start/auth-start.e2e.ts'; +const CHOOSE_ORG = 'specs/features/session-tasks/choose-organization.e2e.ts'; +const COMPLETE_MFA = 'specs/features/session-tasks/complete-setup-mfa.e2e.ts'; +const SETUP_MFA = 'specs/features/session-tasks/setup-mfa.e2e.ts'; +const SIGN_UP = 'specs/features/sign-up/complete.e2e.ts'; +const COMMITTED: Readonly> = { [AUTH_START]: null, [CHOOSE_ORG]: FORCED_ORG, [COMPLETE_MFA]: MFA, [SETUP_MFA]: MFA, [SIGN_UP]: null }; const FAKE_E2E = `#!${process.execPath} const fs = require('node:fs'); @@ -150,7 +150,7 @@ function world(options: { readonly specs?: Readonly { it('refuses to run while a JSON file under specs is not the settings file of a spec, before it leases anything', async () => { const w = world(); - writeFileSync(join(w.dir, 'specs/golden/session-tasks/setup-mfa.e2e.settings.json'), settingsText(MFA)); - await assert.rejects(runVerb(w.deps(), RUN_ALL), (error: VerifyFailure) => error.code === 'USAGE' && error.message === 'specs/golden/session-tasks/setup-mfa.e2e.settings.json is not the settings file of a spec, so no run reads it' && error.fix.includes('.settings.json in place of .e2e.ts')); + writeFileSync(join(w.dir, 'specs/features/session-tasks/setup-mfa.e2e.settings.json'), settingsText(MFA)); + await assert.rejects(runVerb(w.deps(), RUN_ALL), (error: VerifyFailure) => error.code === 'USAGE' && error.message === 'specs/features/session-tasks/setup-mfa.e2e.settings.json is not the settings file of a spec, so no run reads it' && error.fix.includes('.settings.json in place of .e2e.ts')); assert.equal(w.lines.includes('device leased'), false); }); diff --git a/integration/expo-native/test/seam.test.ts b/integration/expo-native/test/seam.test.ts index 735368e0cc7..da8c4b6ea14 100644 --- a/integration/expo-native/test/seam.test.ts +++ b/integration/expo-native/test/seam.test.ts @@ -43,8 +43,8 @@ describe('the tests are an e2e project of their own', () => { 'e2e.config.ts': "import { app } from './specs/app.ts';\nimport './src/core/keys.ts';\n", 'specs/app.ts': "export { app } from './support/inputs.ts';\n", 'specs/support/inputs.cts': "import keys = require('../../src/core/keys.ts');\n", - 'specs/golden/sign-in/complete.e2e.tsx': "import '../../fixtures.ts';\nexport const load = (name: string) => import(`../../../testing/${name}.ts`);\n", - 'specs/golden/sign-in/complete.settings.json': '{ "extends": "../../../src/core/instances/base.json" }\n', + 'specs/features/sign-in/complete.e2e.tsx': "import '../../fixtures.ts';\nexport const load = (name: string) => import(`../../../testing/${name}.ts`);\n", + 'specs/features/sign-in/complete.settings.json': '{ "extends": "../../../src/core/instances/base.json" }\n', 'src/core/keys.ts': "import '../../specs/support/inputs.cts';\n", }; for (const [file, text] of Object.entries(files)) { @@ -53,8 +53,8 @@ describe('the tests are an e2e project of their own', () => { } assert.deepEqual([...pathsThatLeaveTheTests(packageDir)].sort(), [ 'e2e.config.ts names src/core/keys.ts', - 'specs/golden/sign-in/complete.e2e.tsx names testing/${name}.ts', - 'specs/golden/sign-in/complete.settings.json names src/core/instances/base.json', + 'specs/features/sign-in/complete.e2e.tsx names testing/${name}.ts', + 'specs/features/sign-in/complete.settings.json names src/core/instances/base.json', 'specs/support/inputs.cts names src/core/keys.ts', ]); }); diff --git a/integration/expo-native/test/secrets.test.ts b/integration/expo-native/test/secrets.test.ts index 9f4814cb156..32dd0b210f3 100644 --- a/integration/expo-native/test/secrets.test.ts +++ b/integration/expo-native/test/secrets.test.ts @@ -48,7 +48,7 @@ describe('redact', () => { }); it('leaves text alone that only looks a little like a JWT', () => { - for (const text of ['eyJhbGciOiJSUzI1NiJ9', 'eyJhbGciOiJSUzI1NiJ9.notapayloadatall.signaturesignature', 'a.b.c', 'pk_test_ZmFrZS0xLmNsZXJrLmFjY291bnRzLmRldiQ', 'specs/golden/sign-up/complete.e2e.ts']) { + for (const text of ['eyJhbGciOiJSUzI1NiJ9', 'eyJhbGciOiJSUzI1NiJ9.notapayloadatall.signaturesignature', 'a.b.c', 'pk_test_ZmFrZS0xLmNsZXJrLmFjY291bnRzLmRldiQ', 'specs/features/sign-up/complete.e2e.ts']) { assert.equal(redact(text), text); } }); diff --git a/integration/expo-native/test/settings.test.ts b/integration/expo-native/test/settings.test.ts index a91ddbab34f..ace9f5d6d77 100644 --- a/integration/expo-native/test/settings.test.ts +++ b/integration/expo-native/test/settings.test.ts @@ -11,7 +11,7 @@ const MFA: InstanceSettings = { config: { auth_multi_factor: { required_for_sign const FORCED_ORG: InstanceSettings = { config: { organization_settings: { force_organization_selection: true } }, environment: { 'organization_settings.force_organization_selection': true } }; const SPEC_SOURCE = "import { test } from '../../fixtures.ts';\n\ntest('x', async ({ host }) => { await host.launch({}); });\n"; -const spec = (path: string): SpecRef => ({ kind: 'golden', path, feature: null }); +const spec = (path: string): SpecRef => ({ kind: 'committed', path, feature: null }); const text = (declaration: InstanceSettings | string | null, source = SPEC_SOURCE) => ({ source, declaration: declaration === null || typeof declaration === 'string' ? declaration : JSON.stringify(declaration) }); const file = (path: string, declared: InstanceSettings | string | null = null) => ({ spec: spec(path), ...text(declared) }); const usage = (pattern: RegExp) => (error: VerifyFailure) => error instanceof VerifyFailure && error.code === 'USAGE' && pattern.test(error.message); @@ -28,7 +28,7 @@ const refuses = (declarations: Readonly>, why: RegExp = / }; describe('the standard file', () => { - it('pins a standard value for every setting the two golden declarations change', () => { + it('pins a standard value for every setting the two committed declarations change', () => { const pinned = configLeaves(standardFile().config); assert.equal(pinned['auth_multi_factor.required_for_sign_up'], false); assert.equal(pinned['organization_settings.force_organization_selection'], false); @@ -38,7 +38,7 @@ describe('the standard file', () => { describe('reading a declaration from the settings file beside a spec', () => { it('names the settings file after the spec file', () => { - assert.equal(settingsFileOf('specs/golden/session-tasks/setup-mfa.e2e.ts'), 'specs/golden/session-tasks/setup-mfa.settings.json'); + assert.equal(settingsFileOf('specs/features/session-tasks/setup-mfa.e2e.ts'), 'specs/features/session-tasks/setup-mfa.settings.json'); assert.equal(settingsFileOf(ODD), ODD_FILE); }); @@ -88,13 +88,13 @@ describe('reading a declaration from the settings file beside a spec', () => { it('finds a JSON file under specs that is not the settings file of a spec beside it, and none when each has its spec', () => { const dir = mkdtempSync(join(tmpdir(), 'verify-settings-')); assert.equal(straySettingsFile(dir), null, 'a package with no specs directory has none'); - mkdirSync(join(dir, 'specs/golden/session-tasks'), { recursive: true }); - for (const name of ['setup-mfa.e2e.ts', 'setup-mfa.settings.json']) writeFileSync(join(dir, 'specs/golden/session-tasks', name), ''); + mkdirSync(join(dir, 'specs/features/session-tasks'), { recursive: true }); + for (const name of ['setup-mfa.e2e.ts', 'setup-mfa.settings.json']) writeFileSync(join(dir, 'specs/features/session-tasks', name), ''); assert.equal(straySettingsFile(dir), null); - writeFileSync(join(dir, 'specs/golden/session-tasks/setup-mfa.e2e.settings.json'), ''); + writeFileSync(join(dir, 'specs/features/session-tasks/setup-mfa.e2e.settings.json'), ''); const stray = straySettingsFile(dir); assert.equal(stray?.code, 'USAGE'); - assert.equal(stray?.message, 'specs/golden/session-tasks/setup-mfa.e2e.settings.json is not the settings file of a spec, so no run reads it'); + assert.equal(stray?.message, 'specs/features/session-tasks/setup-mfa.e2e.settings.json is not the settings file of a spec, so no run reads it'); assert.equal(stray?.fix, 'name a settings file after its spec file, with .settings.json in place of .e2e.ts, or delete it'); for (const misnamed of ['setup-mfa.setting.json', 'setup-mfa-settings.json', 'settings.json', 'setup-mfa.settings.jsonc']) { const other = mkdtempSync(join(tmpdir(), 'verify-settings-')); @@ -204,10 +204,10 @@ describe('checking a declaration against the standard file', () => { }); describe('planning a run in groups', () => { - const golden = [file('specs/golden/auth-start/auth-start.e2e.ts'), file('specs/golden/session-tasks/choose-organization.e2e.ts', FORCED_ORG), file('specs/golden/session-tasks/complete-setup-mfa.e2e.ts', MFA), file('specs/golden/session-tasks/setup-mfa.e2e.ts', MFA), file('specs/golden/sign-up/complete.e2e.ts')]; + const committed = [file('specs/features/auth-start/auth-start.e2e.ts'), file('specs/features/session-tasks/choose-organization.e2e.ts', FORCED_ORG), file('specs/features/session-tasks/complete-setup-mfa.e2e.ts', MFA), file('specs/features/session-tasks/setup-mfa.e2e.ts', MFA), file('specs/features/sign-up/complete.e2e.ts')]; const mfaKey = settingsOf(MFA, 'x').key; const orgKey = settingsOf(FORCED_ORG, 'x').key; - const order = (applied: string | null, specs = golden) => planGroups(specs, applied).map((group) => group.settings.label); + const order = (applied: string | null, specs = committed) => planGroups(specs, applied).map((group) => group.settings.label); it('puts the group already applied first, so a run changes the instance once less than it has groups', () => { assert.deepEqual(order(STANDARD.key), ['standard', 'organization_settings.force_organization_selection=true', 'auth_multi_factor.required_for_sign_up=true']); @@ -220,10 +220,10 @@ describe('planning a run in groups', () => { }); it('is stable: the same files in the same order plan the same groups, each with its files in order', () => { - const plan = planGroups(golden, STANDARD.key); - assert.deepEqual(plan, planGroups(golden, STANDARD.key)); + const plan = planGroups(committed, STANDARD.key); + assert.deepEqual(plan, planGroups(committed, STANDARD.key)); assert.deepEqual(plan.map((group) => group.specs.map((s) => s.path.split('/').at(-1))), [['auth-start.e2e.ts', 'complete.e2e.ts'], ['choose-organization.e2e.ts'], ['complete-setup-mfa.e2e.ts', 'setup-mfa.e2e.ts']]); - assert.equal(plan[2]!.settings.askedBy, 'specs/golden/session-tasks/complete-setup-mfa.e2e.ts', 'the first spec of a group is the one that asked'); + assert.equal(plan[2]!.settings.askedBy, 'specs/features/session-tasks/complete-setup-mfa.e2e.ts', 'the first spec of a group is the one that asked'); assert.equal(plan[0]!.settings, STANDARD); assert.match(plan[0]!.specs[0]!.sourceHash, /^[0-9a-f]{64}$/); assert.notEqual(plan[0]!.specs[0]!.sourceHash, plan[1]!.specs[0]!.sourceHash); diff --git a/integration/expo-native/test/something-ran.test.ts b/integration/expo-native/test/something-ran.test.ts index 84f764f90bb..d2d6d099ce2 100644 --- a/integration/expo-native/test/something-ran.test.ts +++ b/integration/expo-native/test/something-ran.test.ts @@ -14,9 +14,9 @@ const dir = mkdtempSync(join(tmpdir(), 'verify-ran-')) as EvidencePath; function results(rows: readonly { tags?: string[]; status: string; skip?: { cause: string; reason: string } }[]) { const report = { schemaVersion: 'report-1', - run: { results: rows.map((row, i) => ({ id: `0000000${i}aa`, kind: 'test', titlePath: [`t${i}`], file: 'specs/golden/auth-start/a.e2e.ts', platform: 'ios', tags: row.tags ?? [], status: row.status, skip: row.skip, attempts: [] })) }, + run: { results: rows.map((row, i) => ({ id: `0000000${i}aa`, kind: 'test', titlePath: [`t${i}`], file: 'specs/features/auth-start/a.e2e.ts', platform: 'ios', tags: row.tags ?? [], status: row.status, skip: row.skip, attempts: [] })) }, }; - return parseE2EReport(report, [{ kind: 'golden', path: 'specs/golden/auth-start/a.e2e.ts', feature: null }], dir); + return parseE2EReport(report, [{ kind: 'committed', path: 'specs/features/auth-start/a.e2e.ts', feature: null }], dir); } describe('a run in which nothing executed', () => { diff --git a/integration/expo-native/test/test-users.test.ts b/integration/expo-native/test/test-users.test.ts index ed74d51e9b2..77eee443fb3 100644 --- a/integration/expo-native/test/test-users.test.ts +++ b/integration/expo-native/test/test-users.test.ts @@ -124,9 +124,9 @@ describe('the users a test asks for', () => { assert.equal(redact(`input text ${password.slice(0, 16)} then ${password.slice(16)}`), 'input text then '); }); - it('type one text made from the run id in the golden tests, the password the run knows', () => { - const golden = join(import.meta.dirname, '..', 'specs', 'golden'); - const sources = readdirSync(golden, { recursive: true, encoding: 'utf8' }).filter((file) => file.endsWith('.e2e.ts')).map((file) => readFileSync(join(golden, file), 'utf8')); + it('type one text made from the run id in the e2e tests, the password the run knows', () => { + const committed = join(import.meta.dirname, '..', 'specs', 'features'); + const sources = readdirSync(committed, { recursive: true, encoding: 'utf8' }).filter((file) => file.endsWith('.e2e.ts')).map((file) => readFileSync(join(committed, file), 'utf8')); const filled = sources.flatMap((source) => [...source.matchAll(/host\.fill\([^;]*?(`[^`]*\$\{host\.runId\}[^`]*`)\);/g)].map((match) => match[1])); assert.deepEqual([...new Set(filled)], ['`Verify-${host.runId}-Pw1!`']); assert.equal(runPassword(SAMPLE_RUN).use('device-input', (plain) => plain), `Verify-${SAMPLE_RUN}-Pw1!`);